From 6be4548b06413a22d973d9f3080f9deac5838538 Mon Sep 17 00:00:00 2001 From: Szehon Ho Date: Tue, 1 Jul 2025 12:44:52 -0700 Subject: [PATCH 1/4] [SPARK-52638][SQL] Allow preserving Hive-style column order to be configurable --- .../apache/spark/sql/internal/SQLConf.scala | 10 ++++++ .../spark/sql/hive/HiveExternalCatalog.scala | 21 +++++++----- .../sql/hive/execution/HiveDDLSuite.scala | 33 ++++++++++++++++++- 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 762abfe3b6061..9c4c29c250791 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -5989,6 +5989,16 @@ object SQLConf { .booleanConf .createWithDefault(true) + val HIVE_PRESERVE_LEGACY_COLUMN_ORDER = + buildConf("spark.sql.hive.preserveLegacyColumnOrder.enabled") + .internal() + .doc("When true, tables returned from HiveExternalCatalog preserve Hive-style column order " + + "where the partition columns are at the end. Otherwise, the user-specified column order " + + "is returned.") + .version("4.1.0") + .booleanConf + .createWithDefault(true) + /** * Holds information about keys that have been deprecated. * diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index 5c7a60151c496..802c377fe6ab2 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -48,6 +48,7 @@ import org.apache.spark.sql.execution.command.DDLUtils import org.apache.spark.sql.execution.datasources.{PartitioningUtils, SourceOptions} import org.apache.spark.sql.hive.client.HiveClient import org.apache.spark.sql.internal.HiveSerDe +import org.apache.spark.sql.internal.SQLConf.HIVE_PRESERVE_LEGACY_COLUMN_ORDER import org.apache.spark.sql.internal.StaticSQLConf._ import org.apache.spark.sql.types._ import org.apache.spark.sql.util.SchemaUtils @@ -818,16 +819,20 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat // columns are not put at the end of schema. We need to reorder it when reading the schema // from the table properties. private def reorderSchema(schema: StructType, partColumnNames: Seq[String]): StructType = { - val partitionFields = partColumnNames.map { partCol => - schema.find(_.name == partCol).getOrElse { - throw new AnalysisException( - errorClass = "_LEGACY_ERROR_TEMP_3088", - messageParameters = Map( - "schema" -> schema.catalogString, - "partColumnNames" -> partColumnNames.mkString("[", ", ", "]"))) + if (conf.get(HIVE_PRESERVE_LEGACY_COLUMN_ORDER)) { + schema + } else { + val partitionFields = partColumnNames.map { partCol => + schema.find(_.name == partCol).getOrElse { + throw new AnalysisException( + errorClass = "_LEGACY_ERROR_TEMP_3088", + messageParameters = Map( + "schema" -> schema.catalogString, + "partColumnNames" -> partColumnNames.mkString("[", ", ", "]"))) + } } + StructType(schema.filterNot(partitionFields.contains) ++ partitionFields) } - StructType(schema.filterNot(partitionFields.contains) ++ partitionFields) } private def restoreHiveSerdeTable(table: CatalogTable): CatalogTable = { diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 13e8d3721d81e..b2a3d23e6db3f 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -36,9 +36,10 @@ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException import org.apache.spark.sql.catalyst.catalog._ import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParseException} -import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogV2Util, Identifier, TableChange, TableInfo} +import org.apache.spark.sql.connector.catalog.{CatalogManager, CatalogV2Util, Identifier, TableCatalog, TableChange, TableInfo} import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME import org.apache.spark.sql.connector.catalog.SupportsNamespaces.PROP_OWNER +import org.apache.spark.sql.connector.expressions.Expressions import org.apache.spark.sql.execution.command.{DDLSuite, DDLUtils} import org.apache.spark.sql.execution.datasources.orc.OrcCompressionCodec import org.apache.spark.sql.execution.datasources.parquet.{ParquetCompressionCodec, ParquetFooterReader} @@ -3432,4 +3433,34 @@ class HiveDDLSuite any[String], any[String], any[StructType]) } } + + test("SPARK-52638: Allow preserving Hive-style column order to be configurable") { + val catalog = spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog] + withSQLConf( + SQLConf.HIVE_PRESERVE_LEGACY_COLUMN_ORDER.key -> "false" + ) { + withTable("t1") { + val identifier = Identifier.of(Array("default"), "t1") + val outputSchema = new StructType() + .add("a", IntegerType, true, "comment1") + .add("b", IntegerType, true, "comment2") + .add("c", IntegerType, true, "comment3") + .add("d", IntegerType, true, "comment4") + catalog.createTable( + identifier, + new TableInfo.Builder() + .withProperties(Map.empty.asJava) + .withColumns(CatalogV2Util.structTypeToV2Columns(outputSchema)) + .withPartitions(Array(Expressions.identity("a"))) + .build() + ) + val cols = catalog.loadTable(identifier).columns() + assert(cols.length == 4) + assert(cols(0).name() == "a") + assert(cols(1).name() == "b") + assert(cols(2).name() == "c") + assert(cols(3).name() == "d") + } + } + } } From 41cc8634a82fd1219077c8de9850f270291f6dfb Mon Sep 17 00:00:00 2001 From: Szehon Ho Date: Tue, 1 Jul 2025 16:31:24 -0700 Subject: [PATCH 2/4] fix test --- .../scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index 802c377fe6ab2..46b1875d93efe 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -820,8 +820,6 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat // from the table properties. private def reorderSchema(schema: StructType, partColumnNames: Seq[String]): StructType = { if (conf.get(HIVE_PRESERVE_LEGACY_COLUMN_ORDER)) { - schema - } else { val partitionFields = partColumnNames.map { partCol => schema.find(_.name == partCol).getOrElse { throw new AnalysisException( @@ -832,6 +830,8 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat } } StructType(schema.filterNot(partitionFields.contains) ++ partitionFields) + } else { + schema } } From f0978478740d6cff8373e7e770a60b2b3a6d8600 Mon Sep 17 00:00:00 2001 From: Szehon Ho Date: Wed, 2 Jul 2025 20:19:42 -0700 Subject: [PATCH 3/4] Add table property and only reorder if table property is false --- .../apache/spark/sql/internal/SQLConf.scala | 10 ++-- .../spark/sql/hive/HiveExternalCatalog.scala | 56 +++++++++++++------ .../sql/hive/execution/HiveDDLSuite.scala | 21 ++++++- 3 files changed, 62 insertions(+), 25 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 9c4c29c250791..a2e260c569673 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -5989,12 +5989,12 @@ object SQLConf { .booleanConf .createWithDefault(true) - val HIVE_PRESERVE_LEGACY_COLUMN_ORDER = - buildConf("spark.sql.hive.preserveLegacyColumnOrder.enabled") + val LEGACY_PRESERVE_HIVE_COLUMN_ORDER = + buildConf("spark.sql.legacy.preserveHiveColumnOrder") .internal() - .doc("When true, tables returned from HiveExternalCatalog preserve Hive-style column order " + - "where the partition columns are at the end. Otherwise, the user-specified column order " + - "is returned.") + .doc("When true, tables created by HiveExternalCatalog will maintain Hive-style column " + + "order where the partition columns are at the end. " + + "Otherwise, use the user-specified column order.") .version("4.1.0") .booleanConf .createWithDefault(true) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index 46b1875d93efe..99640d5a13da5 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -48,7 +48,7 @@ import org.apache.spark.sql.execution.command.DDLUtils import org.apache.spark.sql.execution.datasources.{PartitioningUtils, SourceOptions} import org.apache.spark.sql.hive.client.HiveClient import org.apache.spark.sql.internal.HiveSerDe -import org.apache.spark.sql.internal.SQLConf.HIVE_PRESERVE_LEGACY_COLUMN_ORDER +import org.apache.spark.sql.internal.SQLConf.LEGACY_PRESERVE_HIVE_COLUMN_ORDER import org.apache.spark.sql.internal.StaticSQLConf._ import org.apache.spark.sql.types._ import org.apache.spark.sql.util.SchemaUtils @@ -260,6 +260,10 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat tableProperties.put(TABLE_PARTITION_PROVIDER, TABLE_PARTITION_PROVIDER_CATALOG) } + if (!conf.get(LEGACY_PRESERVE_HIVE_COLUMN_ORDER)) { + tableProperties.put(LEGACY_COLUMN_ORDER, "false") + } + // we have to set the table schema here so that the table schema JSON // string in the table properties still uses the original schema val hiveTable = tableDefinition.copy( @@ -819,20 +823,16 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat // columns are not put at the end of schema. We need to reorder it when reading the schema // from the table properties. private def reorderSchema(schema: StructType, partColumnNames: Seq[String]): StructType = { - if (conf.get(HIVE_PRESERVE_LEGACY_COLUMN_ORDER)) { - val partitionFields = partColumnNames.map { partCol => - schema.find(_.name == partCol).getOrElse { - throw new AnalysisException( - errorClass = "_LEGACY_ERROR_TEMP_3088", - messageParameters = Map( - "schema" -> schema.catalogString, - "partColumnNames" -> partColumnNames.mkString("[", ", ", "]"))) - } + val partitionFields = partColumnNames.map { partCol => + schema.find(_.name == partCol).getOrElse { + throw new AnalysisException( + errorClass = "_LEGACY_ERROR_TEMP_3088", + messageParameters = Map( + "schema" -> schema.catalogString, + "partColumnNames" -> partColumnNames.mkString("[", ", ", "]"))) } - StructType(schema.filterNot(partitionFields.contains) ++ partitionFields) - } else { - schema } + StructType(schema.filterNot(partitionFields.contains) ++ partitionFields) } private def restoreHiveSerdeTable(table: CatalogTable): CatalogTable = { @@ -847,12 +847,16 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat if (maybeSchemaFromTableProps.isDefined) { val schemaFromTableProps = maybeSchemaFromTableProps.get val partColumnNames = getPartitionColumnsFromTableProperties(table) - val reorderedSchema = reorderSchema(schema = schemaFromTableProps, partColumnNames) + val schema = if (isLegacyColumnOrder(table)) { + reorderSchema(schema = schemaFromTableProps, partColumnNames) + } else { + schemaFromTableProps + } - if (DataTypeUtils.equalsIgnoreCaseNullabilityAndCollation(reorderedSchema, table.schema) || + if (DataTypeUtils.equalsIgnoreCaseNullabilityAndCollation(schema, table.schema) || options.respectSparkSchema) { hiveTable.copy( - schema = reorderedSchema, + schema = schema, partitionColumnNames = partColumnNames, bucketSpec = getBucketSpecFromTableProperties(table)) } else { @@ -898,12 +902,16 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat val schemaFromTableProps = getSchemaFromTableProperties(table.properties).getOrElse(new StructType()) val partColumnNames = getPartitionColumnsFromTableProperties(table) - val reorderedSchema = reorderSchema(schema = schemaFromTableProps, partColumnNames) + val schema = if (isLegacyColumnOrder(table)) { + reorderSchema(schema = schemaFromTableProps, partColumnNames) + } else { + schemaFromTableProps + } table.copy( provider = Some(provider), storage = storageWithoutHiveGeneratedProperties, - schema = reorderedSchema, + schema = schema, partitionColumnNames = partColumnNames, bucketSpec = getBucketSpecFromTableProperties(table), tracksPartitionsInCatalog = partitionProvider == Some(TABLE_PARTITION_PROVIDER_CATALOG), @@ -1435,6 +1443,8 @@ object HiveExternalCatalog { val EMPTY_DATA_SCHEMA = new StructType() .add("col", "array", nullable = true, comment = "from deserializer") + val LEGACY_COLUMN_ORDER = "legacy_column_order" + private def getColumnNamesByType( props: Map[String, String], colType: String, @@ -1494,4 +1504,14 @@ object HiveExternalCatalog { case st: StringType => st.isUTF8BinaryCollation case _ => true } + + // Whether table was created specifying legacy column order + // lack of the table property means we prserve legacy column order + private def isLegacyColumnOrder(table: CatalogTable): Boolean = { + val legacyColumnOrder = table.properties.get(LEGACY_COLUMN_ORDER) + legacyColumnOrder match { + case Some(l) if l.equalsIgnoreCase("false") => false + case _ => false + } + } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index b2a3d23e6db3f..a1c1c37502123 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -3437,7 +3437,7 @@ class HiveDDLSuite test("SPARK-52638: Allow preserving Hive-style column order to be configurable") { val catalog = spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog] withSQLConf( - SQLConf.HIVE_PRESERVE_LEGACY_COLUMN_ORDER.key -> "false" + SQLConf.LEGACY_PRESERVE_HIVE_COLUMN_ORDER.key -> "false" ) { withTable("t1") { val identifier = Identifier.of(Array("default"), "t1") @@ -3454,12 +3454,29 @@ class HiveDDLSuite .withPartitions(Array(Expressions.identity("a"))) .build() ) - val cols = catalog.loadTable(identifier).columns() + val table1 = catalog.loadTable(identifier) + val cols = table1.columns() assert(cols.length == 4) assert(cols(0).name() == "a") assert(cols(1).name() == "b") assert(cols(2).name() == "c") assert(cols(3).name() == "d") + assert(table1.properties().get("spark.sql.legacy.preserveHiveColumnOrder") == "false") + + catalog.alterTable( + identifier, + TableChange.addColumn(Array("e"), IntegerType) + ) + + val table2 = catalog.loadTable(identifier) + val cols2 = table2.columns() + assert(cols2.length == 5) + assert(cols2(0).name() == "a") + assert(cols2(1).name() == "b") + assert(cols2(2).name() == "c") + assert(cols2(3).name() == "d") + assert(cols2(4).name() == "e") + assert(table2.properties().get("spark.sql.legacy.preserveHiveColumnOrder") == "false") } } } From 250866047c2c272348f48482b7dd7966c8b858a8 Mon Sep 17 00:00:00 2001 From: Szehon Ho Date: Thu, 3 Jul 2025 11:34:05 -0700 Subject: [PATCH 4/4] Review comments --- .../apache/spark/sql/internal/SQLConf.scala | 4 +- .../spark/sql/hive/HiveExternalCatalog.scala | 14 +-- .../sql/hive/execution/HiveDDLSuite.scala | 98 +++++++++++-------- 3 files changed, 65 insertions(+), 51 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index a2e260c569673..71d890805be88 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -5993,8 +5993,8 @@ object SQLConf { buildConf("spark.sql.legacy.preserveHiveColumnOrder") .internal() .doc("When true, tables created by HiveExternalCatalog will maintain Hive-style column " + - "order where the partition columns are at the end. " + - "Otherwise, use the user-specified column order.") + "order where the partition columns are at the end. Otherwise, use the user-specified " + + "column order. Does not affect tables with provider = `hive`") .version("4.1.0") .booleanConf .createWithDefault(true) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index 99640d5a13da5..378b934ae1bda 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -847,16 +847,12 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat if (maybeSchemaFromTableProps.isDefined) { val schemaFromTableProps = maybeSchemaFromTableProps.get val partColumnNames = getPartitionColumnsFromTableProperties(table) - val schema = if (isLegacyColumnOrder(table)) { - reorderSchema(schema = schemaFromTableProps, partColumnNames) - } else { - schemaFromTableProps - } + val reorderedSchema = reorderSchema(schema = schemaFromTableProps, partColumnNames) - if (DataTypeUtils.equalsIgnoreCaseNullabilityAndCollation(schema, table.schema) || + if (DataTypeUtils.equalsIgnoreCaseNullabilityAndCollation(reorderedSchema, table.schema) || options.respectSparkSchema) { hiveTable.copy( - schema = schema, + schema = reorderedSchema, partitionColumnNames = partColumnNames, bucketSpec = getBucketSpecFromTableProperties(table)) } else { @@ -1506,12 +1502,12 @@ object HiveExternalCatalog { } // Whether table was created specifying legacy column order - // lack of the table property means we prserve legacy column order + // lack of the table property means we preserve legacy column order private def isLegacyColumnOrder(table: CatalogTable): Boolean = { val legacyColumnOrder = table.properties.get(LEGACY_COLUMN_ORDER) legacyColumnOrder match { case Some(l) if l.equalsIgnoreCase("false") => false - case _ => false + case _ => true } } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index a1c1c37502123..1324a9aa2dde3 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -3436,47 +3436,65 @@ class HiveDDLSuite test("SPARK-52638: Allow preserving Hive-style column order to be configurable") { val catalog = spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog] - withSQLConf( - SQLConf.LEGACY_PRESERVE_HIVE_COLUMN_ORDER.key -> "false" - ) { - withTable("t1") { - val identifier = Identifier.of(Array("default"), "t1") - val outputSchema = new StructType() - .add("a", IntegerType, true, "comment1") - .add("b", IntegerType, true, "comment2") - .add("c", IntegerType, true, "comment3") - .add("d", IntegerType, true, "comment4") - catalog.createTable( - identifier, - new TableInfo.Builder() - .withProperties(Map.empty.asJava) - .withColumns(CatalogV2Util.structTypeToV2Columns(outputSchema)) - .withPartitions(Array(Expressions.identity("a"))) - .build() - ) - val table1 = catalog.loadTable(identifier) - val cols = table1.columns() - assert(cols.length == 4) - assert(cols(0).name() == "a") - assert(cols(1).name() == "b") - assert(cols(2).name() == "c") - assert(cols(3).name() == "d") - assert(table1.properties().get("spark.sql.legacy.preserveHiveColumnOrder") == "false") - - catalog.alterTable( - identifier, - TableChange.addColumn(Array("e"), IntegerType) - ) + Seq(true, false).map { preserveOrder => { + withSQLConf(SQLConf.LEGACY_PRESERVE_HIVE_COLUMN_ORDER.key -> preserveOrder.toString) { + withTable("t1") { + val identifier = Identifier.of(Array("default"), "t1") + val outputSchema = new StructType() + .add("a", IntegerType, true, "comment1") + .add("b", IntegerType, true, "comment2") + .add("c", IntegerType, true, "comment3") + .add("d", IntegerType, true, "comment4") + catalog.createTable( + identifier, + new TableInfo.Builder() + .withProperties(Map.empty.asJava) + .withColumns(CatalogV2Util.structTypeToV2Columns(outputSchema)) + .withPartitions(Array(Expressions.identity("a"))) + .build() + ) + val table1 = catalog.loadTable(identifier) + val cols = table1.columns() + + if (preserveOrder) { + assert(cols.length == 4) + assert(cols(0).name() == "a") + assert(cols(1).name() == "b") + assert(cols(2).name() == "c") + assert(cols(3).name() == "d") + assert(table1.properties().get("spark.sql.legacy.preserveHiveColumnOrder") == "false") + } else { + assert(cols.length == 4) + assert(cols(0).name() == "b") + assert(cols(1).name() == "c") + assert(cols(2).name() == "d") + assert(cols(3).name() == "a") + } - val table2 = catalog.loadTable(identifier) - val cols2 = table2.columns() - assert(cols2.length == 5) - assert(cols2(0).name() == "a") - assert(cols2(1).name() == "b") - assert(cols2(2).name() == "c") - assert(cols2(3).name() == "d") - assert(cols2(4).name() == "e") - assert(table2.properties().get("spark.sql.legacy.preserveHiveColumnOrder") == "false") + catalog.alterTable( + identifier, + TableChange.addColumn(Array("e"), IntegerType) + ) + + val table2 = catalog.loadTable(identifier) + val cols2 = table2.columns() + if (preserveOrder) { + assert(cols2.length == 5) + assert(cols2(0).name() == "a") + assert(cols2(1).name() == "b") + assert(cols2(2).name() == "c") + assert(cols2(3).name() == "d") + assert(cols2(4).name() == "e") + assert(table2.properties().get("spark.sql.legacy.preserveHiveColumnOrder") == "false") + } else { + assert(cols2(0).name() == "b") + assert(cols2(1).name() == "c") + assert(cols2(2).name() == "d") + assert(cols2(3).name() == "e") + assert(cols2(4).name() == "a") + } + } + } } } }