From bd38d14252e06b1b38faa0a5552a4b91c2238ec5 Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Thu, 18 Sep 2025 21:47:01 +0800 Subject: [PATCH 1/2] Add compatibility class/object for org.apache.spark.sql.execution.streaming --- .../sql/execution/streaming/legacy.scala | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/legacy.scala diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/legacy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/legacy.scala new file mode 100644 index 0000000000000..47cbc92aeb50b --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/legacy.scala @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.streaming + +import scala.reflect.ClassTag + +import org.apache.hadoop.conf.Configuration + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.execution.streaming.checkpointing.{HDFSMetadataLog => ActualHDFSMetadataLog} +import org.apache.spark.sql.execution.streaming.runtime.{SerializedOffset => ActualSerializedOffset} + +@deprecated("use org.apache.spark.sql.execution.streaming.checkpointing.HDFSMetadataLog") +class HDFSMetadataLog[T <: AnyRef: ClassTag](conf: Configuration, path: String) + extends ActualHDFSMetadataLog[T](conf, path) { + def this(spark: SparkSession, path: String) = { + this(spark.sessionState.newHadoopConf(), path) + } +} + +@deprecated("use org.apache.spark.sql.execution.streaming.runtime.SerializedOffset") +object SerializedOffset { + def apply(offset: String): ActualSerializedOffset = ActualSerializedOffset(offset) +} From 3206847d8d51dbee626e1c45980fbf1be6d296e5 Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Fri, 19 Sep 2025 08:14:37 +0800 Subject: [PATCH 2/2] Update legacy.scala --- .../apache/spark/sql/execution/streaming/legacy.scala | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/legacy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/legacy.scala index 47cbc92aeb50b..f3ce006ee8116 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/legacy.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/legacy.scala @@ -19,19 +19,13 @@ package org.apache.spark.sql.execution.streaming import scala.reflect.ClassTag -import org.apache.hadoop.conf.Configuration - import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.streaming.checkpointing.{HDFSMetadataLog => ActualHDFSMetadataLog} import org.apache.spark.sql.execution.streaming.runtime.{SerializedOffset => ActualSerializedOffset} @deprecated("use org.apache.spark.sql.execution.streaming.checkpointing.HDFSMetadataLog") -class HDFSMetadataLog[T <: AnyRef: ClassTag](conf: Configuration, path: String) - extends ActualHDFSMetadataLog[T](conf, path) { - def this(spark: SparkSession, path: String) = { - this(spark.sessionState.newHadoopConf(), path) - } -} +class HDFSMetadataLog[T <: AnyRef: ClassTag](sparkSession: SparkSession, path: String) + extends ActualHDFSMetadataLog[T](sparkSession, path) @deprecated("use org.apache.spark.sql.execution.streaming.runtime.SerializedOffset") object SerializedOffset {