diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index c1ae6f218458..b6023e1b8be6 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -551,7 +551,8 @@ std::unordered_map WholeStageResultIterator::getQueryC configs[velox::core::QueryConfig::kSparkPartitionId] = std::to_string(taskInfo_.partitionId); - // Enable Spark legacy date formatter if spark.sql.legacy.timeParserPolicy is set to 'LEGACY'. + // Enable Spark legacy date formatter if spark.sql.legacy.timeParserPolicy is set to 'LEGACY' + // or 'legacy' if (veloxCfg_->get(kSparkLegacyTimeParserPolicy, "") == "LEGACY") { configs[velox::core::QueryConfig::kSparkLegacyDateFormatter] = "true"; } else { diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 62ab86836301..0a4d7f9deda0 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -261,8 +261,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("to_unix_timestamp") // Unsupported datetime format: specifier X is not supported by velox. .exclude("to_timestamp with microseconds precision") - // Replaced by another test. - .exclude("to_timestamp") // Legacy mode is not supported, assuming this mode is not commonly used. .exclude("SPARK-30668: use legacy timestamp parser in to_timestamp") // Legacy mode is not supported and velox getTimestamp function does not throw diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala index 5ddfe6fc1ff3..aa94dc50c4a5 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala @@ -114,19 +114,9 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), Seq(Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null))) - // legacyParserPolicy is not respected by Gluten. // invalid format - // val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')") - // if (legacyParserPolicy == "legacy") { - // checkAnswer(invalid, - // Seq(Row(null), Row(null), Row(null), Row(null))) - // } else { - // val e = intercept[SparkUpgradeException](invalid.collect()) - // assert(e.getCause.isInstanceOf[IllegalArgumentException]) - // assert(e.getMessage.contains( - // "You may get a different result due to the upgrading to Spark")) - // } - + val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')") + checkAnswer(invalid, Seq(Row(null), Row(null), Row(null), Row(null))) // February val y1 = "2016-02-29" val y2 = "2017-02-29" @@ -198,53 +188,8 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra df2.select(unix_timestamp(col("y"), "yyyy-MM-dd")), Seq(Row(secs(ts5.getTime)), Row(null))) - // Not consistent behavior with gluten + velox. - // invalid format - // val invalid = df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')") - // val e = intercept[IllegalArgumentException](invalid.collect()) - // assert(e.getMessage.contains('b')) - } - } - } - - // Ported from spark with a test case for legacy mode removed. - testGluten("to_timestamp") { - Seq("legacy", "corrected").foreach { - legacyParserPolicy => - withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) { - val date1 = Date.valueOf("2015-07-24") - val date2 = Date.valueOf("2015-07-25") - val ts_date1 = Timestamp.valueOf("2015-07-24 00:00:00") - val ts_date2 = Timestamp.valueOf("2015-07-25 00:00:00") - val ts1 = Timestamp.valueOf("2015-07-24 10:00:00") - val ts2 = Timestamp.valueOf("2015-07-25 02:02:02") - val s1 = "2015/07/24 10:00:00.5" - val s2 = "2015/07/25 02:02:02.6" - val ts1m = Timestamp.valueOf("2015-07-24 10:00:00.5") - val ts2m = Timestamp.valueOf("2015-07-25 02:02:02.6") - val ss1 = "2015-07-24 10:00:00" - val ss2 = "2015-07-25 02:02:02" - val fmt = "yyyy/MM/dd HH:mm:ss.S" - val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss") - - checkAnswer( - df.select(to_timestamp(col("ss"))), - df.select(timestamp_seconds(unix_timestamp(col("ss"))))) - checkAnswer(df.select(to_timestamp(col("ss"))), Seq(Row(ts1), Row(ts2))) - if (legacyParserPolicy == "legacy") { - // In Spark 2.4 and earlier, to_timestamp() parses in seconds precision and cuts off - // the fractional part of seconds. The behavior was changed by SPARK-27438. - // Ignore this test case. Velox returns null for such case. - // val legacyFmt = "yyyy/MM/dd HH:mm:ss" - // checkAnswer(df.select(to_timestamp(col("s"), legacyFmt)), Seq( - // Row(ts1), Row(ts2))) - } else { - checkAnswer(df.select(to_timestamp(col("s"), fmt)), Seq(Row(ts1m), Row(ts2m))) - } - checkAnswer(df.select(to_timestamp(col("ts"), fmt)), Seq(Row(ts1), Row(ts2))) - checkAnswer( - df.select(to_timestamp(col("d"), "yyyy-MM-dd")), - Seq(Row(ts_date1), Row(ts_date2))) + val invalid = df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')") + checkAnswer(invalid, Seq(Row(null), Row(null), Row(null), Row(null))) } } } diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 72b77ae1f95b..adba37f162cf 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1070,8 +1070,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("to_unix_timestamp") // Unsupported datetime format: specifier X is not supported by velox. .exclude("to_timestamp with microseconds precision") - // Replaced by another test. - .exclude("to_timestamp") // Legacy mode is not supported, assuming this mode is not commonly used. .exclude("SPARK-30668: use legacy timestamp parser in to_timestamp") // Legacy mode is not supported and velox getTimestamp function does not throw diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala index ae86c9d06e81..f9c5995cafdd 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala @@ -112,18 +112,9 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), Seq(Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null))) - // legacyParserPolicy is not respected by Gluten. // invalid format - // val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')") - // if (legacyParserPolicy == "legacy") { - // checkAnswer(invalid, - // Seq(Row(null), Row(null), Row(null), Row(null))) - // } else { - // val e = intercept[SparkUpgradeException](invalid.collect()) - // assert(e.getCause.isInstanceOf[IllegalArgumentException]) - // assert( e.getMessage.contains( - // "You may get a different result due to the upgrading to Spark")) - // } + val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')") + checkAnswer(invalid, Seq(Row(null), Row(null), Row(null), Row(null))) // February val y1 = "2016-02-29" @@ -196,53 +187,8 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra df2.select(unix_timestamp(col("y"), "yyyy-MM-dd")), Seq(Row(secs(ts5.getTime)), Row(null))) - // Not consistent behavior with gluten + velox. - // invalid format - // val invalid = df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')") - // val e = intercept[IllegalArgumentException](invalid.collect()) - // assert(e.getMessage.contains('b')) - } - } - } - - // Ported from spark with a test case for legacy mode removed. - testGluten("to_timestamp") { - Seq("legacy", "corrected").foreach { - legacyParserPolicy => - withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) { - val date1 = Date.valueOf("2015-07-24") - val date2 = Date.valueOf("2015-07-25") - val ts_date1 = Timestamp.valueOf("2015-07-24 00:00:00") - val ts_date2 = Timestamp.valueOf("2015-07-25 00:00:00") - val ts1 = Timestamp.valueOf("2015-07-24 10:00:00") - val ts2 = Timestamp.valueOf("2015-07-25 02:02:02") - val s1 = "2015/07/24 10:00:00.5" - val s2 = "2015/07/25 02:02:02.6" - val ts1m = Timestamp.valueOf("2015-07-24 10:00:00.5") - val ts2m = Timestamp.valueOf("2015-07-25 02:02:02.6") - val ss1 = "2015-07-24 10:00:00" - val ss2 = "2015-07-25 02:02:02" - val fmt = "yyyy/MM/dd HH:mm:ss.S" - val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss") - - checkAnswer( - df.select(to_timestamp(col("ss"))), - df.select(timestamp_seconds(unix_timestamp(col("ss"))))) - checkAnswer(df.select(to_timestamp(col("ss"))), Seq(Row(ts1), Row(ts2))) - if (legacyParserPolicy == "legacy") { - // In Spark 2.4 and earlier, to_timestamp() parses in seconds precision and cuts off - // the fractional part of seconds. The behavior was changed by SPARK-27438. - // Ignore this test case. Velox returns null for such case. - // val legacyFmt = "yyyy/MM/dd HH:mm:ss" - // checkAnswer(df.select(to_timestamp(col("s"), legacyFmt)), Seq( - // Row(ts1), Row(ts2))) - } else { - checkAnswer(df.select(to_timestamp(col("s"), fmt)), Seq(Row(ts1m), Row(ts2m))) - } - checkAnswer(df.select(to_timestamp(col("ts"), fmt)), Seq(Row(ts1), Row(ts2))) - checkAnswer( - df.select(to_timestamp(col("d"), "yyyy-MM-dd")), - Seq(Row(ts_date1), Row(ts_date2))) + val invalid = df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')") + checkAnswer(invalid, Seq(Row(null), Row(null), Row(null), Row(null))) } } } diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 94d3a1f6e890..d58319667147 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1100,8 +1100,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("to_unix_timestamp") // Unsupported datetime format: specifier X is not supported by velox. .exclude("to_timestamp with microseconds precision") - // Replaced by another test. - .exclude("to_timestamp") // Legacy mode is not supported, assuming this mode is not commonly used. .exclude("SPARK-30668: use legacy timestamp parser in to_timestamp") // Legacy mode is not supported and velox getTimestamp function does not throw diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala index ae86c9d06e81..d66f26d7c09e 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala @@ -112,18 +112,9 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), Seq(Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null))) - // legacyParserPolicy is not respected by Gluten. // invalid format - // val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')") - // if (legacyParserPolicy == "legacy") { - // checkAnswer(invalid, - // Seq(Row(null), Row(null), Row(null), Row(null))) - // } else { - // val e = intercept[SparkUpgradeException](invalid.collect()) - // assert(e.getCause.isInstanceOf[IllegalArgumentException]) - // assert( e.getMessage.contains( - // "You may get a different result due to the upgrading to Spark")) - // } + val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')") + checkAnswer(invalid, Seq(Row(null), Row(null), Row(null), Row(null))) // February val y1 = "2016-02-29" @@ -196,53 +187,9 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra df2.select(unix_timestamp(col("y"), "yyyy-MM-dd")), Seq(Row(secs(ts5.getTime)), Row(null))) - // Not consistent behavior with gluten + velox. // invalid format - // val invalid = df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')") - // val e = intercept[IllegalArgumentException](invalid.collect()) - // assert(e.getMessage.contains('b')) - } - } - } - - // Ported from spark with a test case for legacy mode removed. - testGluten("to_timestamp") { - Seq("legacy", "corrected").foreach { - legacyParserPolicy => - withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) { - val date1 = Date.valueOf("2015-07-24") - val date2 = Date.valueOf("2015-07-25") - val ts_date1 = Timestamp.valueOf("2015-07-24 00:00:00") - val ts_date2 = Timestamp.valueOf("2015-07-25 00:00:00") - val ts1 = Timestamp.valueOf("2015-07-24 10:00:00") - val ts2 = Timestamp.valueOf("2015-07-25 02:02:02") - val s1 = "2015/07/24 10:00:00.5" - val s2 = "2015/07/25 02:02:02.6" - val ts1m = Timestamp.valueOf("2015-07-24 10:00:00.5") - val ts2m = Timestamp.valueOf("2015-07-25 02:02:02.6") - val ss1 = "2015-07-24 10:00:00" - val ss2 = "2015-07-25 02:02:02" - val fmt = "yyyy/MM/dd HH:mm:ss.S" - val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss") - - checkAnswer( - df.select(to_timestamp(col("ss"))), - df.select(timestamp_seconds(unix_timestamp(col("ss"))))) - checkAnswer(df.select(to_timestamp(col("ss"))), Seq(Row(ts1), Row(ts2))) - if (legacyParserPolicy == "legacy") { - // In Spark 2.4 and earlier, to_timestamp() parses in seconds precision and cuts off - // the fractional part of seconds. The behavior was changed by SPARK-27438. - // Ignore this test case. Velox returns null for such case. - // val legacyFmt = "yyyy/MM/dd HH:mm:ss" - // checkAnswer(df.select(to_timestamp(col("s"), legacyFmt)), Seq( - // Row(ts1), Row(ts2))) - } else { - checkAnswer(df.select(to_timestamp(col("s"), fmt)), Seq(Row(ts1m), Row(ts2m))) - } - checkAnswer(df.select(to_timestamp(col("ts"), fmt)), Seq(Row(ts1), Row(ts2))) - checkAnswer( - df.select(to_timestamp(col("d"), "yyyy-MM-dd")), - Seq(Row(ts_date1), Row(ts_date2))) + val invalid = df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')") + checkAnswer(invalid, Seq(Row(null), Row(null), Row(null), Row(null))) } } } diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 73c4d43ced53..2bb567dd68c4 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -1122,8 +1122,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("to_unix_timestamp") // Unsupported datetime format: specifier X is not supported by velox. .exclude("to_timestamp with microseconds precision") - // Replaced by another test. - .exclude("to_timestamp") // Legacy mode is not supported, assuming this mode is not commonly used. .exclude("SPARK-30668: use legacy timestamp parser in to_timestamp") // Legacy mode is not supported and velox getTimestamp function does not throw diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala index ae86c9d06e81..f9c5995cafdd 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala @@ -112,18 +112,9 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd mm:HH:ss')"), Seq(Row(secs(ts4.getTime)), Row(null), Row(secs(ts3.getTime)), Row(null))) - // legacyParserPolicy is not respected by Gluten. // invalid format - // val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')") - // if (legacyParserPolicy == "legacy") { - // checkAnswer(invalid, - // Seq(Row(null), Row(null), Row(null), Row(null))) - // } else { - // val e = intercept[SparkUpgradeException](invalid.collect()) - // assert(e.getCause.isInstanceOf[IllegalArgumentException]) - // assert( e.getMessage.contains( - // "You may get a different result due to the upgrading to Spark")) - // } + val invalid = df1.selectExpr(s"unix_timestamp(x, 'yyyy-MM-dd aa:HH:ss')") + checkAnswer(invalid, Seq(Row(null), Row(null), Row(null), Row(null))) // February val y1 = "2016-02-29" @@ -196,53 +187,8 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra df2.select(unix_timestamp(col("y"), "yyyy-MM-dd")), Seq(Row(secs(ts5.getTime)), Row(null))) - // Not consistent behavior with gluten + velox. - // invalid format - // val invalid = df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')") - // val e = intercept[IllegalArgumentException](invalid.collect()) - // assert(e.getMessage.contains('b')) - } - } - } - - // Ported from spark with a test case for legacy mode removed. - testGluten("to_timestamp") { - Seq("legacy", "corrected").foreach { - legacyParserPolicy => - withSQLConf(SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy) { - val date1 = Date.valueOf("2015-07-24") - val date2 = Date.valueOf("2015-07-25") - val ts_date1 = Timestamp.valueOf("2015-07-24 00:00:00") - val ts_date2 = Timestamp.valueOf("2015-07-25 00:00:00") - val ts1 = Timestamp.valueOf("2015-07-24 10:00:00") - val ts2 = Timestamp.valueOf("2015-07-25 02:02:02") - val s1 = "2015/07/24 10:00:00.5" - val s2 = "2015/07/25 02:02:02.6" - val ts1m = Timestamp.valueOf("2015-07-24 10:00:00.5") - val ts2m = Timestamp.valueOf("2015-07-25 02:02:02.6") - val ss1 = "2015-07-24 10:00:00" - val ss2 = "2015-07-25 02:02:02" - val fmt = "yyyy/MM/dd HH:mm:ss.S" - val df = Seq((date1, ts1, s1, ss1), (date2, ts2, s2, ss2)).toDF("d", "ts", "s", "ss") - - checkAnswer( - df.select(to_timestamp(col("ss"))), - df.select(timestamp_seconds(unix_timestamp(col("ss"))))) - checkAnswer(df.select(to_timestamp(col("ss"))), Seq(Row(ts1), Row(ts2))) - if (legacyParserPolicy == "legacy") { - // In Spark 2.4 and earlier, to_timestamp() parses in seconds precision and cuts off - // the fractional part of seconds. The behavior was changed by SPARK-27438. - // Ignore this test case. Velox returns null for such case. - // val legacyFmt = "yyyy/MM/dd HH:mm:ss" - // checkAnswer(df.select(to_timestamp(col("s"), legacyFmt)), Seq( - // Row(ts1), Row(ts2))) - } else { - checkAnswer(df.select(to_timestamp(col("s"), fmt)), Seq(Row(ts1m), Row(ts2m))) - } - checkAnswer(df.select(to_timestamp(col("ts"), fmt)), Seq(Row(ts1), Row(ts2))) - checkAnswer( - df.select(to_timestamp(col("d"), "yyyy-MM-dd")), - Seq(Row(ts_date1), Row(ts_date2))) + val invalid = df1.selectExpr(s"to_unix_timestamp(x, 'yyyy-MM-dd bb:HH:ss')") + checkAnswer(invalid, Seq(Row(null), Row(null), Row(null), Row(null))) } } } diff --git a/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala index 94a95ae515a3..4eb5b388ac46 100644 --- a/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala @@ -656,6 +656,13 @@ object GlutenConfig { SPARK_SHUFFLE_FILE_BUFFER, (JavaUtils.byteStringAs(v, ByteUnit.KiB) * 1024).toString)) + conf + .get(LEGACY_TIME_PARSER_POLICY.key) + .foreach( + v => + nativeConfMap + .put(LEGACY_TIME_PARSER_POLICY.key, v.toUpperCase(Locale.ROOT))) + // Backend's dynamic session conf only. val confPrefix = prefixOf(backendName) conf