From 3e51c44a4263142ca0822bc8dfdfaa16a297293a Mon Sep 17 00:00:00 2001 From: zml1206 Date: Mon, 7 Apr 2025 21:08:35 +0800 Subject: [PATCH 1/4] [CORE] Native runtime keeps the same timezone as JVM when SESSION_LOCAL_TIMEZONE is not set --- .../org/apache/gluten/runtime/Runtime.scala | 8 ++- .../utils/velox/VeloxTestSettings.scala | 10 ++-- .../GlutenDateExpressionsSuite.scala | 58 ------------------- .../utils/velox/VeloxTestSettings.scala | 10 ++-- .../GlutenDateExpressionsSuite.scala | 58 ------------------- .../utils/velox/VeloxTestSettings.scala | 10 ++-- .../GlutenDateExpressionsSuite.scala | 58 ------------------- .../utils/velox/VeloxTestSettings.scala | 10 ++-- .../GlutenDateExpressionsSuite.scala | 58 ------------------- 9 files changed, 23 insertions(+), 257 deletions(-) diff --git a/gluten-arrow/src/main/scala/org/apache/gluten/runtime/Runtime.scala b/gluten-arrow/src/main/scala/org/apache/gluten/runtime/Runtime.scala index f4fb2fa42c19..1a4290893d32 100644 --- a/gluten-arrow/src/main/scala/org/apache/gluten/runtime/Runtime.scala +++ b/gluten-arrow/src/main/scala/org/apache/gluten/runtime/Runtime.scala @@ -47,7 +47,7 @@ object Runtime { nmm.getHandle(), ConfigUtil.serialize( GlutenConfig - .getNativeSessionConf(backendName, GlutenConfigUtil.parseConfig(SQLConf.get.getAllConfs))) + .getNativeSessionConf(backendName, GlutenConfigUtil.parseConfig(getConf))) ) private val released: AtomicBoolean = new AtomicBoolean(false) @@ -68,5 +68,11 @@ object Runtime { override def priority(): Int = 20 override def resourceName(): String = s"runtime" + + private def getConf: Map[String, String] = { + val sqlConf = SQLConf.get + sqlConf.getAllConfs + (SQLConf.SESSION_LOCAL_TIMEZONE.key -> sqlConf.getConf( + SQLConf.SESSION_LOCAL_TIMEZONE)) + } } } diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 66e2d3159ea9..3f89f406b758 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -210,19 +210,17 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDateExpressionsSuite] // Rewrite because Spark collect causes long overflow. .exclude("TIMESTAMP_MICROS") - // Replaced by a gluten test to pass timezone through config. + // Velox doesn't support timezones like UTC. .exclude("unix_timestamp") - // Replaced by a gluten test to pass timezone through config. + // Velox doesn't support timezones like UTC. .exclude("to_unix_timestamp") - // Replaced by a gluten test to pass timezone through config. + // Velox doesn't support timezones like UTC. .exclude("Hour") // Unsupported format: yyyy-MM-dd HH:mm:ss.SSS .exclude("SPARK-33498: GetTimestamp,UnixTimestamp,ToUnixTimestamp with parseError") - // Replaced by a gluten test to pass timezone through config. - .exclude("DateFormat") // Legacy mode is not supported, assuming this mode is not commonly used. .exclude("to_timestamp exception mode") - // Replaced by a gluten test to pass timezone through config. + // Velox doesn't support timezones like UTC. .exclude("from_unixtime") // https://github.com/facebookincubator/velox/pull/10563/files#diff-140dc50e6dac735f72d29014da44b045509df0dd1737f458de1fe8cfd33d8145 .excludeGlutenTest("from_unixtime") diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala index 44d4502aedac..3a0befd100ff 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala @@ -286,64 +286,6 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr ToUnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil) } - // Modified based on vanilla spark to explicitly set timezone in config. - testGluten("DateFormat") { - val PST_OPT = Option("America/Los_Angeles") - val JST_OPT = Option("Asia/Tokyo") - - Seq("legacy", "corrected").foreach { - legacyParserPolicy => - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get) { - checkEvaluation( - DateFormatClass(Literal.create(null, TimestampType), Literal("y"), UTC_OPT), - null) - checkEvaluation( - DateFormatClass( - Cast(Literal(d), TimestampType, UTC_OPT), - Literal.create(null, StringType), - UTC_OPT), - null) - - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("y"), UTC_OPT), - "2015") - checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), UTC_OPT), "2013") - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("H"), UTC_OPT), - "0") - checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), UTC_OPT), "13") - } - - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) { - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("y"), PST_OPT), - "2015") - checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), PST_OPT), "2013") - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("H"), PST_OPT), - "0") - checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), PST_OPT), "5") - } - - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> JST_OPT.get) { - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("y"), JST_OPT), - "2015") - checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), JST_OPT), "2013") - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("H"), JST_OPT), - "0") - checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), JST_OPT), "22") - } - } - } - testGluten("from_unixtime") { val outstandingTimezonesIds: Seq[String] = Seq( // Velox doesn't support timezones like "UTC". diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 8f0d562f9582..76254578d4f6 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -126,19 +126,17 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDateExpressionsSuite] // Has exception in fallback execution when we use resultDF.collect in evaluation. .exclude("TIMESTAMP_MICROS") - // Replaced by a gluten test to pass timezone through config. + // Velox doesn't support timezones like UTC. .exclude("unix_timestamp") - // Replaced by a gluten test to pass timezone through config. + // Velox doesn't support timezones like UTC. .exclude("to_unix_timestamp") - // Replaced by a gluten test to pass timezone through config. + // Velox doesn't support timezones like UTC. .exclude("Hour") // Unsupported format: yyyy-MM-dd HH:mm:ss.SSS .exclude("SPARK-33498: GetTimestamp,UnixTimestamp,ToUnixTimestamp with parseError") - // Replaced by a gluten test to pass timezone through config. - .exclude("DateFormat") // Legacy mode is not supported, assuming this mode is not commonly used. .exclude("to_timestamp exception mode") - // Replaced by a gluten test to pass timezone through config. + // Velox doesn't support timezones like UTC. .exclude("from_unixtime") .exclude("test timestamp add") // https://github.com/facebookincubator/velox/pull/10563/files#diff-140dc50e6dac735f72d29014da44b045509df0dd1737f458de1fe8cfd33d8145 diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala index 234537feef8a..e022e6a9b92c 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala @@ -284,64 +284,6 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr ToUnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil) } - // Modified based on vanilla spark to explicitly set timezone in config. - testGluten("DateFormat") { - val PST_OPT = Option("America/Los_Angeles") - val JST_OPT = Option("Asia/Tokyo") - - Seq("legacy", "corrected").foreach { - legacyParserPolicy => - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get) { - checkEvaluation( - DateFormatClass(Literal.create(null, TimestampType), Literal("y"), UTC_OPT), - null) - checkEvaluation( - DateFormatClass( - Cast(Literal(d), TimestampType, UTC_OPT), - Literal.create(null, StringType), - UTC_OPT), - null) - - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("y"), UTC_OPT), - "2015") - checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), UTC_OPT), "2013") - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("H"), UTC_OPT), - "0") - checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), UTC_OPT), "13") - } - - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) { - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("y"), PST_OPT), - "2015") - checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), PST_OPT), "2013") - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("H"), PST_OPT), - "0") - checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), PST_OPT), "5") - } - - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> JST_OPT.get) { - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("y"), JST_OPT), - "2015") - checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), JST_OPT), "2013") - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("H"), JST_OPT), - "0") - checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), JST_OPT), "22") - } - } - } - testGluten("from_unixtime") { val outstandingTimezonesIds: Seq[String] = Seq( // Velox doesn't support timezones like "UTC". diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index d688cc12724e..62ba74a43dff 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -121,19 +121,17 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDateExpressionsSuite] // Has exception in fallback execution when we use resultDF.collect in evaluation. .exclude("TIMESTAMP_MICROS") - // Replaced by a gluten test to pass timezone through config. + // Velox doesn't support timezones like UTC. .exclude("unix_timestamp") - // Replaced by a gluten test to pass timezone through config. + // Velox doesn't support timezones like UTC. .exclude("to_unix_timestamp") - // Replaced by a gluten test to pass timezone through config. + // Velox doesn't support timezones like UTC. .exclude("Hour") // Unsupported format: yyyy-MM-dd HH:mm:ss.SSS .exclude("SPARK-33498: GetTimestamp,UnixTimestamp,ToUnixTimestamp with parseError") - // Replaced by a gluten test to pass timezone through config. - .exclude("DateFormat") // Legacy mode is not supported, assuming this mode is not commonly used. .exclude("to_timestamp exception mode") - // Replaced by a gluten test to pass timezone through config. + // Velox doesn't support timezones like UTC. .exclude("from_unixtime") // https://github.com/facebookincubator/velox/pull/10563/files#diff-140dc50e6dac735f72d29014da44b045509df0dd1737f458de1fe8cfd33d8145 .excludeGlutenTest("from_unixtime") diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala index f4a9dfd56a92..9b103e46ab6d 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala @@ -284,64 +284,6 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr ToUnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil) } - // Modified based on vanilla spark to explicitly set timezone in config. - testGluten("DateFormat") { - val PST_OPT = Option("America/Los_Angeles") - val JST_OPT = Option("Asia/Tokyo") - - Seq("legacy", "corrected").foreach { - legacyParserPolicy => - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get) { - checkEvaluation( - DateFormatClass(Literal.create(null, TimestampType), Literal("y"), UTC_OPT), - null) - checkEvaluation( - DateFormatClass( - Cast(Literal(d), TimestampType, UTC_OPT), - Literal.create(null, StringType), - UTC_OPT), - null) - - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("y"), UTC_OPT), - "2015") - checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), UTC_OPT), "2013") - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("H"), UTC_OPT), - "0") - checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), UTC_OPT), "13") - } - - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) { - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("y"), PST_OPT), - "2015") - checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), PST_OPT), "2013") - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("H"), PST_OPT), - "0") - checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), PST_OPT), "5") - } - - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> JST_OPT.get) { - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("y"), JST_OPT), - "2015") - checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), JST_OPT), "2013") - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("H"), JST_OPT), - "0") - checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), JST_OPT), "22") - } - } - } - testGluten("from_unixtime") { val outstandingTimezonesIds: Seq[String] = Seq( // Velox doesn't support timezones like "UTC". diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index a069724fa9a2..39ff8ce46cfc 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -122,19 +122,17 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDateExpressionsSuite] // Has exception in fallback execution when we use resultDF.collect in evaluation. .exclude("TIMESTAMP_MICROS") - // Replaced by a gluten test to pass timezone through config. + // Velox doesn't support timezones like UTC. .exclude("unix_timestamp") - // Replaced by a gluten test to pass timezone through config. + // Velox doesn't support timezones like UTC. .exclude("to_unix_timestamp") - // Replaced by a gluten test to pass timezone through config. + // Velox doesn't support timezones like UTC. .exclude("Hour") // Unsupported format: yyyy-MM-dd HH:mm:ss.SSS .exclude("SPARK-33498: GetTimestamp,UnixTimestamp,ToUnixTimestamp with parseError") // Replaced by a gluten test to pass timezone through config. - .exclude("DateFormat") - // Legacy mode is not supported, assuming this mode is not commonly used. .exclude("to_timestamp exception mode") - // Replaced by a gluten test to pass timezone through config. + // Velox doesn't support timezones like UTC. .exclude("from_unixtime") // https://github.com/facebookincubator/velox/pull/10563/files#diff-140dc50e6dac735f72d29014da44b045509df0dd1737f458de1fe8cfd33d8145 .excludeGlutenTest("from_unixtime") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala index f4a9dfd56a92..9b103e46ab6d 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala @@ -284,64 +284,6 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr ToUnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil) } - // Modified based on vanilla spark to explicitly set timezone in config. - testGluten("DateFormat") { - val PST_OPT = Option("America/Los_Angeles") - val JST_OPT = Option("Asia/Tokyo") - - Seq("legacy", "corrected").foreach { - legacyParserPolicy => - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get) { - checkEvaluation( - DateFormatClass(Literal.create(null, TimestampType), Literal("y"), UTC_OPT), - null) - checkEvaluation( - DateFormatClass( - Cast(Literal(d), TimestampType, UTC_OPT), - Literal.create(null, StringType), - UTC_OPT), - null) - - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("y"), UTC_OPT), - "2015") - checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), UTC_OPT), "2013") - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("H"), UTC_OPT), - "0") - checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), UTC_OPT), "13") - } - - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) { - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("y"), PST_OPT), - "2015") - checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), PST_OPT), "2013") - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("H"), PST_OPT), - "0") - checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), PST_OPT), "5") - } - - withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> JST_OPT.get) { - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("y"), JST_OPT), - "2015") - checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), JST_OPT), "2013") - checkEvaluation( - DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("H"), JST_OPT), - "0") - checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), JST_OPT), "22") - } - } - } - testGluten("from_unixtime") { val outstandingTimezonesIds: Seq[String] = Seq( // Velox doesn't support timezones like "UTC". From bdfe61419b3abc8ad680c236fc35582f82632190 Mon Sep 17 00:00:00 2001 From: zml1206 Date: Tue, 8 Apr 2025 00:09:15 +0800 Subject: [PATCH 2/4] fix --- .../utils/velox/VeloxTestSettings.scala | 10 +-- .../GlutenDateExpressionsSuite.scala | 61 ++++++++++++++++++- .../utils/velox/VeloxTestSettings.scala | 10 +-- .../GlutenDateExpressionsSuite.scala | 61 ++++++++++++++++++- .../utils/velox/VeloxTestSettings.scala | 10 +-- .../GlutenDateExpressionsSuite.scala | 61 ++++++++++++++++++- .../utils/velox/VeloxTestSettings.scala | 10 +-- .../GlutenDateExpressionsSuite.scala | 61 ++++++++++++++++++- 8 files changed, 260 insertions(+), 24 deletions(-) diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 3f89f406b758..66e2d3159ea9 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -210,17 +210,19 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDateExpressionsSuite] // Rewrite because Spark collect causes long overflow. .exclude("TIMESTAMP_MICROS") - // Velox doesn't support timezones like UTC. + // Replaced by a gluten test to pass timezone through config. .exclude("unix_timestamp") - // Velox doesn't support timezones like UTC. + // Replaced by a gluten test to pass timezone through config. .exclude("to_unix_timestamp") - // Velox doesn't support timezones like UTC. + // Replaced by a gluten test to pass timezone through config. .exclude("Hour") // Unsupported format: yyyy-MM-dd HH:mm:ss.SSS .exclude("SPARK-33498: GetTimestamp,UnixTimestamp,ToUnixTimestamp with parseError") + // Replaced by a gluten test to pass timezone through config. + .exclude("DateFormat") // Legacy mode is not supported, assuming this mode is not commonly used. .exclude("to_timestamp exception mode") - // Velox doesn't support timezones like UTC. + // Replaced by a gluten test to pass timezone through config. .exclude("from_unixtime") // https://github.com/facebookincubator/velox/pull/10563/files#diff-140dc50e6dac735f72d29014da44b045509df0dd1737f458de1fe8cfd33d8145 .excludeGlutenTest("from_unixtime") diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala index 3a0befd100ff..dfa364067943 100644 --- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala +++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala @@ -286,6 +286,64 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr ToUnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil) } + // Modified based on vanilla spark to explicitly set timezone in config. + testGluten("DateFormat") { + val PST_OPT = Option("America/Los_Angeles") + val JST_OPT = Option("Asia/Tokyo") + + Seq("legacy", "corrected").foreach { + legacyParserPolicy => + withSQLConf( + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, + SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get) { + checkEvaluation( + DateFormatClass(Literal.create(null, TimestampType), Literal("y"), UTC_OPT), + null) + checkEvaluation( + DateFormatClass( + Cast(Literal(d), TimestampType, UTC_OPT), + Literal.create(null, StringType), + UTC_OPT), + null) + + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("y"), UTC_OPT), + "2015") + checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), UTC_OPT), "2013") + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("H"), UTC_OPT), + "0") + checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), UTC_OPT), "13") + } + + withSQLConf( + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, + SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) { + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("y"), PST_OPT), + "2015") + checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), PST_OPT), "2013") + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("H"), PST_OPT), + "0") + checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), PST_OPT), "5") + } + + withSQLConf( + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, + SQLConf.SESSION_LOCAL_TIMEZONE.key -> JST_OPT.get) { + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("y"), JST_OPT), + "2015") + checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), JST_OPT), "2013") + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("H"), JST_OPT), + "0") + checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), JST_OPT), "22") + } + } + } + testGluten("from_unixtime") { val outstandingTimezonesIds: Seq[String] = Seq( // Velox doesn't support timezones like "UTC". @@ -374,8 +432,7 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr Seq("legacy", "corrected").foreach { legacyParserPolicy => withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy ) { assert(Hour(Literal.create(null, DateType), UTC_OPT).resolved === false) assert(Hour(Literal(ts), UTC_OPT).resolved) diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 76254578d4f6..8f0d562f9582 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -126,17 +126,19 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDateExpressionsSuite] // Has exception in fallback execution when we use resultDF.collect in evaluation. .exclude("TIMESTAMP_MICROS") - // Velox doesn't support timezones like UTC. + // Replaced by a gluten test to pass timezone through config. .exclude("unix_timestamp") - // Velox doesn't support timezones like UTC. + // Replaced by a gluten test to pass timezone through config. .exclude("to_unix_timestamp") - // Velox doesn't support timezones like UTC. + // Replaced by a gluten test to pass timezone through config. .exclude("Hour") // Unsupported format: yyyy-MM-dd HH:mm:ss.SSS .exclude("SPARK-33498: GetTimestamp,UnixTimestamp,ToUnixTimestamp with parseError") + // Replaced by a gluten test to pass timezone through config. + .exclude("DateFormat") // Legacy mode is not supported, assuming this mode is not commonly used. .exclude("to_timestamp exception mode") - // Velox doesn't support timezones like UTC. + // Replaced by a gluten test to pass timezone through config. .exclude("from_unixtime") .exclude("test timestamp add") // https://github.com/facebookincubator/velox/pull/10563/files#diff-140dc50e6dac735f72d29014da44b045509df0dd1737f458de1fe8cfd33d8145 diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala index e022e6a9b92c..34c2358fe7cb 100644 --- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala +++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala @@ -284,6 +284,64 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr ToUnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil) } + // Modified based on vanilla spark to explicitly set timezone in config. + testGluten("DateFormat") { + val PST_OPT = Option("America/Los_Angeles") + val JST_OPT = Option("Asia/Tokyo") + + Seq("legacy", "corrected").foreach { + legacyParserPolicy => + withSQLConf( + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, + SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get) { + checkEvaluation( + DateFormatClass(Literal.create(null, TimestampType), Literal("y"), UTC_OPT), + null) + checkEvaluation( + DateFormatClass( + Cast(Literal(d), TimestampType, UTC_OPT), + Literal.create(null, StringType), + UTC_OPT), + null) + + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("y"), UTC_OPT), + "2015") + checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), UTC_OPT), "2013") + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("H"), UTC_OPT), + "0") + checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), UTC_OPT), "13") + } + + withSQLConf( + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, + SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) { + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("y"), PST_OPT), + "2015") + checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), PST_OPT), "2013") + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("H"), PST_OPT), + "0") + checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), PST_OPT), "5") + } + + withSQLConf( + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, + SQLConf.SESSION_LOCAL_TIMEZONE.key -> JST_OPT.get) { + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("y"), JST_OPT), + "2015") + checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), JST_OPT), "2013") + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("H"), JST_OPT), + "0") + checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), JST_OPT), "22") + } + } + } + testGluten("from_unixtime") { val outstandingTimezonesIds: Seq[String] = Seq( // Velox doesn't support timezones like "UTC". @@ -372,8 +430,7 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr Seq("legacy", "corrected").foreach { legacyParserPolicy => withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy ) { assert(Hour(Literal.create(null, DateType), UTC_OPT).resolved === false) assert(Hour(Literal(ts), UTC_OPT).resolved) diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 62ba74a43dff..d688cc12724e 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -121,17 +121,19 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDateExpressionsSuite] // Has exception in fallback execution when we use resultDF.collect in evaluation. .exclude("TIMESTAMP_MICROS") - // Velox doesn't support timezones like UTC. + // Replaced by a gluten test to pass timezone through config. .exclude("unix_timestamp") - // Velox doesn't support timezones like UTC. + // Replaced by a gluten test to pass timezone through config. .exclude("to_unix_timestamp") - // Velox doesn't support timezones like UTC. + // Replaced by a gluten test to pass timezone through config. .exclude("Hour") // Unsupported format: yyyy-MM-dd HH:mm:ss.SSS .exclude("SPARK-33498: GetTimestamp,UnixTimestamp,ToUnixTimestamp with parseError") + // Replaced by a gluten test to pass timezone through config. + .exclude("DateFormat") // Legacy mode is not supported, assuming this mode is not commonly used. .exclude("to_timestamp exception mode") - // Velox doesn't support timezones like UTC. + // Replaced by a gluten test to pass timezone through config. .exclude("from_unixtime") // https://github.com/facebookincubator/velox/pull/10563/files#diff-140dc50e6dac735f72d29014da44b045509df0dd1737f458de1fe8cfd33d8145 .excludeGlutenTest("from_unixtime") diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala index 9b103e46ab6d..a76ba8394802 100644 --- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala +++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala @@ -284,6 +284,64 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr ToUnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil) } + // Modified based on vanilla spark to explicitly set timezone in config. + testGluten("DateFormat") { + val PST_OPT = Option("America/Los_Angeles") + val JST_OPT = Option("Asia/Tokyo") + + Seq("legacy", "corrected").foreach { + legacyParserPolicy => + withSQLConf( + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, + SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get) { + checkEvaluation( + DateFormatClass(Literal.create(null, TimestampType), Literal("y"), UTC_OPT), + null) + checkEvaluation( + DateFormatClass( + Cast(Literal(d), TimestampType, UTC_OPT), + Literal.create(null, StringType), + UTC_OPT), + null) + + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("y"), UTC_OPT), + "2015") + checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), UTC_OPT), "2013") + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("H"), UTC_OPT), + "0") + checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), UTC_OPT), "13") + } + + withSQLConf( + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, + SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) { + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("y"), PST_OPT), + "2015") + checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), PST_OPT), "2013") + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("H"), PST_OPT), + "0") + checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), PST_OPT), "5") + } + + withSQLConf( + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, + SQLConf.SESSION_LOCAL_TIMEZONE.key -> JST_OPT.get) { + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("y"), JST_OPT), + "2015") + checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), JST_OPT), "2013") + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("H"), JST_OPT), + "0") + checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), JST_OPT), "22") + } + } + } + testGluten("from_unixtime") { val outstandingTimezonesIds: Seq[String] = Seq( // Velox doesn't support timezones like "UTC". @@ -372,8 +430,7 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr Seq("legacy", "corrected").foreach { legacyParserPolicy => withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy ) { assert(Hour(Literal.create(null, DateType), UTC_OPT).resolved === false) assert(Hour(Literal(ts), UTC_OPT).resolved) diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 39ff8ce46cfc..a069724fa9a2 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -122,17 +122,19 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenDateExpressionsSuite] // Has exception in fallback execution when we use resultDF.collect in evaluation. .exclude("TIMESTAMP_MICROS") - // Velox doesn't support timezones like UTC. + // Replaced by a gluten test to pass timezone through config. .exclude("unix_timestamp") - // Velox doesn't support timezones like UTC. + // Replaced by a gluten test to pass timezone through config. .exclude("to_unix_timestamp") - // Velox doesn't support timezones like UTC. + // Replaced by a gluten test to pass timezone through config. .exclude("Hour") // Unsupported format: yyyy-MM-dd HH:mm:ss.SSS .exclude("SPARK-33498: GetTimestamp,UnixTimestamp,ToUnixTimestamp with parseError") // Replaced by a gluten test to pass timezone through config. + .exclude("DateFormat") + // Legacy mode is not supported, assuming this mode is not commonly used. .exclude("to_timestamp exception mode") - // Velox doesn't support timezones like UTC. + // Replaced by a gluten test to pass timezone through config. .exclude("from_unixtime") // https://github.com/facebookincubator/velox/pull/10563/files#diff-140dc50e6dac735f72d29014da44b045509df0dd1737f458de1fe8cfd33d8145 .excludeGlutenTest("from_unixtime") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala index 9b103e46ab6d..a76ba8394802 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenDateExpressionsSuite.scala @@ -284,6 +284,64 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr ToUnixTimestamp(Literal("2015-07-24"), Literal("\""), UTC_OPT) :: Nil) } + // Modified based on vanilla spark to explicitly set timezone in config. + testGluten("DateFormat") { + val PST_OPT = Option("America/Los_Angeles") + val JST_OPT = Option("Asia/Tokyo") + + Seq("legacy", "corrected").foreach { + legacyParserPolicy => + withSQLConf( + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, + SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get) { + checkEvaluation( + DateFormatClass(Literal.create(null, TimestampType), Literal("y"), UTC_OPT), + null) + checkEvaluation( + DateFormatClass( + Cast(Literal(d), TimestampType, UTC_OPT), + Literal.create(null, StringType), + UTC_OPT), + null) + + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("y"), UTC_OPT), + "2015") + checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), UTC_OPT), "2013") + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, UTC_OPT), Literal("H"), UTC_OPT), + "0") + checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), UTC_OPT), "13") + } + + withSQLConf( + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, + SQLConf.SESSION_LOCAL_TIMEZONE.key -> PST_OPT.get) { + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("y"), PST_OPT), + "2015") + checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), PST_OPT), "2013") + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, PST_OPT), Literal("H"), PST_OPT), + "0") + checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), PST_OPT), "5") + } + + withSQLConf( + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, + SQLConf.SESSION_LOCAL_TIMEZONE.key -> JST_OPT.get) { + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("y"), JST_OPT), + "2015") + checkEvaluation(DateFormatClass(Literal(ts), Literal("y"), JST_OPT), "2013") + checkEvaluation( + DateFormatClass(Cast(Literal(d), TimestampType, JST_OPT), Literal("H"), JST_OPT), + "0") + checkEvaluation(DateFormatClass(Literal(ts), Literal("H"), JST_OPT), "22") + } + } + } + testGluten("from_unixtime") { val outstandingTimezonesIds: Seq[String] = Seq( // Velox doesn't support timezones like "UTC". @@ -372,8 +430,7 @@ class GlutenDateExpressionsSuite extends DateExpressionsSuite with GlutenTestsTr Seq("legacy", "corrected").foreach { legacyParserPolicy => withSQLConf( - SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy, - SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get + SQLConf.LEGACY_TIME_PARSER_POLICY.key -> legacyParserPolicy ) { assert(Hour(Literal.create(null, DateType), UTC_OPT).resolved === false) assert(Hour(Literal(ts), UTC_OPT).resolved) From 72e93b90ea6327638345d414a5720e4e9c8b6228 Mon Sep 17 00:00:00 2001 From: zml1206 Date: Tue, 8 Apr 2025 10:23:25 +0800 Subject: [PATCH 3/4] fix --- .../main/scala/org/apache/gluten/runtime/Runtime.scala | 8 +------- .../scala/org/apache/gluten/config/GlutenConfig.scala | 8 ++++---- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/gluten-arrow/src/main/scala/org/apache/gluten/runtime/Runtime.scala b/gluten-arrow/src/main/scala/org/apache/gluten/runtime/Runtime.scala index 1a4290893d32..f4fb2fa42c19 100644 --- a/gluten-arrow/src/main/scala/org/apache/gluten/runtime/Runtime.scala +++ b/gluten-arrow/src/main/scala/org/apache/gluten/runtime/Runtime.scala @@ -47,7 +47,7 @@ object Runtime { nmm.getHandle(), ConfigUtil.serialize( GlutenConfig - .getNativeSessionConf(backendName, GlutenConfigUtil.parseConfig(getConf))) + .getNativeSessionConf(backendName, GlutenConfigUtil.parseConfig(SQLConf.get.getAllConfs))) ) private val released: AtomicBoolean = new AtomicBoolean(false) @@ -68,11 +68,5 @@ object Runtime { override def priority(): Int = 20 override def resourceName(): String = s"runtime" - - private def getConf: Map[String, String] = { - val sqlConf = SQLConf.get - sqlConf.getAllConfs + (SQLConf.SESSION_LOCAL_TIMEZONE.key -> sqlConf.getConf( - SQLConf.SESSION_LOCAL_TIMEZONE)) - } } } diff --git a/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala index f9920e67323c..d2ffd552a654 100644 --- a/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala @@ -461,7 +461,6 @@ object GlutenConfig { COLUMNAR_TASK_OFFHEAP_SIZE_IN_BYTES.key, COLUMNAR_MAX_BATCH_SIZE.key, SHUFFLE_WRITER_BUFFER_SIZE.key, - SQLConf.SESSION_LOCAL_TIMEZONE.key, GLUTEN_DEFAULT_SESSION_TIMEZONE.key, SQLConf.LEGACY_SIZE_OF_NULL.key, "spark.io.compression.codec", @@ -516,7 +515,8 @@ object GlutenConfig { GLUTEN_COLUMNAR_TO_ROW_MEM_THRESHOLD.key, GLUTEN_COLUMNAR_TO_ROW_MEM_THRESHOLD.defaultValue.get.toString), (SPARK_SHUFFLE_SPILL_COMPRESS, SPARK_SHUFFLE_SPILL_COMPRESS_DEFAULT.toString), - (SQLConf.MAP_KEY_DEDUP_POLICY.key, SQLConf.MAP_KEY_DEDUP_POLICY.defaultValueString) + (SQLConf.MAP_KEY_DEDUP_POLICY.key, SQLConf.MAP_KEY_DEDUP_POLICY.defaultValueString), + (SESSION_LOCAL_TIMEZONE.key, SESSION_LOCAL_TIMEZONE.defaultValueString) ) keyWithDefault.forEach(e => nativeConfMap.put(e._1, conf.getOrElse(e._1, e._2))) GlutenConfigUtil.mapByteConfValue( @@ -595,7 +595,8 @@ object GlutenConfig { ("spark.gluten.sql.columnar.backend.velox.fileHandleCacheEnabled", "false"), ("spark.gluten.velox.awsSdkLogLevel", "FATAL"), ("spark.gluten.velox.s3UseProxyFromEnv", "false"), - ("spark.gluten.velox.s3PayloadSigningPolicy", "Never") + ("spark.gluten.velox.s3PayloadSigningPolicy", "Never"), + (SESSION_LOCAL_TIMEZONE.key, SESSION_LOCAL_TIMEZONE.defaultValueString) ) keyWithDefault.forEach(e => nativeConfMap.put(e._1, conf.getOrElse(e._1, e._2))) @@ -609,7 +610,6 @@ object GlutenConfig { COLUMNAR_OFFHEAP_SIZE_IN_BYTES.key, COLUMNAR_TASK_OFFHEAP_SIZE_IN_BYTES.key, SPARK_OFFHEAP_ENABLED, - SESSION_LOCAL_TIMEZONE.key, DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key, SPARK_REDACTION_REGEX, LEGACY_TIME_PARSER_POLICY.key From bb3a99002b88791a81aa0f552c36553719ffb16d Mon Sep 17 00:00:00 2001 From: zml1206 Date: Wed, 9 Apr 2025 18:29:37 +0800 Subject: [PATCH 4/4] remove GLUTEN_DEFAULT_SESSION_TIMEZONE and kDefaultSessionTimezone --- cpp/core/config/GlutenConfig.h | 2 -- cpp/velox/compute/WholeStageResultIterator.cc | 7 +------ .../src/main/scala/org/apache/gluten/GlutenPlugin.scala | 4 ---- .../scala/org/apache/gluten/config/GlutenConfig.scala | 8 -------- 4 files changed, 1 insertion(+), 20 deletions(-) diff --git a/cpp/core/config/GlutenConfig.h b/cpp/core/config/GlutenConfig.h index 95bfd8751d7a..51efc25b582d 100644 --- a/cpp/core/config/GlutenConfig.h +++ b/cpp/core/config/GlutenConfig.h @@ -37,8 +37,6 @@ const std::string kAllowPrecisionLoss = "spark.sql.decimalOperations.allowPrecis const std::string kIgnoreMissingFiles = "spark.sql.files.ignoreMissingFiles"; -const std::string kDefaultSessionTimezone = "spark.gluten.sql.session.timeZone.default"; - const std::string kSparkOverheadMemory = "spark.gluten.memoryOverhead.size.in.bytes"; const std::string kSparkOffHeapMemory = "spark.gluten.memory.offHeap.size.in.bytes"; diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index 4c3e6cb622ab..c19d319012df 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -477,12 +477,7 @@ std::unordered_map WholeStageResultIterator::getQueryC configs[velox::core::QueryConfig::kMaxOutputBatchRows] = std::to_string(veloxCfg_->get(kSparkBatchSize, 4096)); try { - if (veloxCfg_->valueExists(kDefaultSessionTimezone)) { - configs[velox::core::QueryConfig::kSessionTimezone] = veloxCfg_->get(kDefaultSessionTimezone, ""); - } - if (veloxCfg_->valueExists(kSessionTimezone)) { - configs[velox::core::QueryConfig::kSessionTimezone] = veloxCfg_->get(kSessionTimezone, ""); - } + configs[velox::core::QueryConfig::kSessionTimezone] = veloxCfg_->get(kSessionTimezone, ""); // Adjust timestamp according to the above configured session timezone. configs[velox::core::QueryConfig::kAdjustTimestampToTimezone] = "true"; diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala index 41814492f30c..f06330f9256f 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala @@ -171,10 +171,6 @@ private[gluten] class GlutenDriverPlugin extends DriverPlugin with Logging { s"and set '$SPARK_OFFHEAP_SIZE_KEY' to be greater than $minOffHeapSize") } - // Session's local time zone must be set. If not explicitly set by user, its default - // value (detected for the platform) is used, consistent with spark. - conf.set(GLUTEN_DEFAULT_SESSION_TIMEZONE.key, SQLConf.SESSION_LOCAL_TIMEZONE.defaultValueString) - // Task slots. val taskSlots = SparkResourceUtil.getTaskSlots(conf) conf.set(NUM_TASK_SLOTS_PER_EXECUTOR.key, taskSlots.toString) diff --git a/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala b/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala index d2ffd552a654..029af3229d3c 100644 --- a/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala +++ b/shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala @@ -461,7 +461,6 @@ object GlutenConfig { COLUMNAR_TASK_OFFHEAP_SIZE_IN_BYTES.key, COLUMNAR_MAX_BATCH_SIZE.key, SHUFFLE_WRITER_BUFFER_SIZE.key, - GLUTEN_DEFAULT_SESSION_TIMEZONE.key, SQLConf.LEGACY_SIZE_OF_NULL.key, "spark.io.compression.codec", "spark.sql.decimalOperations.allowPrecisionLoss", @@ -655,13 +654,6 @@ object GlutenConfig { .booleanConf .createWithDefault(true) - val GLUTEN_DEFAULT_SESSION_TIMEZONE = buildConf("spark.gluten.sql.session.timeZone.default") - .doc( - "used to hold default session timezone and will be really used by Gluten only if " + - "`spark.sql.session.timeZone` is not set.") - .stringConf - .createWithDefaultString(SQLConf.SESSION_LOCAL_TIMEZONE.defaultValueString) - val GLUTEN_LOAD_LIB_OS = buildConf("spark.gluten.loadLibOS") .doc("The shared library loader's OS name.")