[GLUTEN-11108][CORE] Upgrade iceberg version to 1.10.0 for spark 3.4 and 3.5

.github/workflows/velox_backend_enhanced.yml

@@ -127,7 +127,7 @@ jobs:
           java -version
           export SPARK_HOME=/opt/shims/spark34/spark_home/
           ls -l $SPARK_HOME
-          $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi \
+          $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi \
           -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest \
           -DargLine="-Dspark.test.home=$SPARK_HOME"
       - name: Upload test report

.github/workflows/velox_backend_x86.yml

@@ -829,7 +829,7 @@ jobs:
           java -version
           export SPARK_HOME=/opt/shims/spark34/spark_home/
           ls -l $SPARK_HOME
-          $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \
+          $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \
           -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \
           -DargLine="-Dspark.test.home=$SPARK_HOME"
       - name: Upload test report

backends-clickhouse/pom.xml

@@ -539,8 +539,8 @@
         <activeByDefault>false</activeByDefault>
       </activation>
       <properties>
-        <iceberg.flinkConnector.version>1.18.1</iceberg.flinkConnector.version>
-        <iceberg.flinkConnector.subversion>1.18</iceberg.flinkConnector.subversion>
+        <iceberg.flinkConnector.version>1.19.3</iceberg.flinkConnector.version>
+        <iceberg.flinkConnector.subversion>1.19</iceberg.flinkConnector.subversion>
       </properties>
       <dependencies>
         <dependency>

backends-velox/pom.xml

@@ -408,6 +408,13 @@
             </exclusion>
           </exclusions>
         </dependency>
+        <dependency>
+          <groupId>org.apache.iceberg</groupId>
+          <artifactId>iceberg-core</artifactId>
+          <version>${iceberg.version}</version>
+          <type>test-jar</type>
+          <scope>test</scope>
+        </dependency>
         <dependency>
           <groupId>org.apache.iceberg</groupId>
           <artifactId>iceberg-hive-metastore</artifactId>
@@ -448,16 +455,9 @@
           <version>3.26.3</version>
           <scope>test</scope>
         </dependency>
-        <dependency>
-          <groupId>junit</groupId>
-          <artifactId>junit</artifactId>
-          <version>4.13.2</version>
-          <scope>test</scope>
-        </dependency>
         <dependency>
           <groupId>org.junit.jupiter</groupId>
-          <artifactId>junit-jupiter-api</artifactId>
-          <version>5.11.4</version>
+          <artifactId>junit-jupiter</artifactId>
           <scope>test</scope>
         </dependency>
         <dependency>
@@ -466,6 +466,50 @@
           <version>4.2.2</version>
           <scope>test</scope>
         </dependency>
+        <dependency>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>jetty-server</artifactId>
+          <version>11.0.26</version>
+          <scope>test</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.eclipse.jetty</groupId>
+          <artifactId>jetty-servlet</artifactId>
+          <version>11.0.26</version>
+          <scope>test</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.xerial</groupId>
+          <artifactId>sqlite-jdbc</artifactId>
+          <version>3.50.3.0</version>
+          <scope>test</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+    <profile>
+      <id>iceberg-test</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <properties>
+        <hadoop.version>3.4.1</hadoop.version>
+      </properties>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.iceberg</groupId>
+          <artifactId>iceberg-open-api</artifactId>
+          <version>${iceberg.version}</version>
+          <type>test-jar</type>
+          <scope>test</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.iceberg</groupId>
+          <artifactId>iceberg-open-api</artifactId>
+          <version>${iceberg.version}</version>
+          <classifier>test-fixtures</classifier>
+          <type>test-jar</type>
+          <scope>test</scope>
+        </dependency>
       </dependencies>
     </profile>
     <profile>

backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenCopyOnWriteDelete.java

@@ -16,36 +16,12 @@
  */
 package org.apache.gluten.extensions;
 
-import org.apache.iceberg.PlanningMode;
 import org.apache.iceberg.spark.extensions.TestCopyOnWriteDelete;
 import org.junit.Test;
 
-import java.util.Map;
 import java.util.concurrent.ExecutionException;
 
 public class TestGlutenCopyOnWriteDelete extends TestCopyOnWriteDelete {
-  public TestGlutenCopyOnWriteDelete(
-      String catalogName,
-      String implementation,
-      Map<String, String> config,
-      String fileFormat,
-      Boolean vectorized,
-      String distributionMode,
-      boolean fanoutEnabled,
-      String branch,
-      PlanningMode planningMode) {
-    super(
-        catalogName,
-        implementation,
-        config,
-        fileFormat,
-        vectorized,
-        distributionMode,
-        fanoutEnabled,
-        branch,
-        planningMode);
-  }
-
   @Test
   public synchronized void testDeleteWithConcurrentTableRefresh() {
     System.out.println("Run timeout");

backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadDelete.java

@@ -16,35 +16,12 @@
  */
 package org.apache.gluten.extensions;
 
-import org.apache.iceberg.PlanningMode;
 import org.apache.iceberg.spark.extensions.TestMergeOnReadDelete;
 import org.junit.Test;
 
-import java.util.Map;
 import java.util.concurrent.ExecutionException;
 
 public class TestGlutenMergeOnReadDelete extends TestMergeOnReadDelete {
-  public TestGlutenMergeOnReadDelete(
-      String catalogName,
-      String implementation,
-      Map<String, String> config,
-      String fileFormat,
-      Boolean vectorized,
-      String distributionMode,
-      boolean fanoutEnabled,
-      String branch,
-      PlanningMode planningMode) {
-    super(
-        catalogName,
-        implementation,
-        config,
-        fileFormat,
-        vectorized,
-        distributionMode,
-        fanoutEnabled,
-        branch,
-        planningMode);
-  }
 
   @Test
   public synchronized void testDeleteWithConcurrentTableRefresh() {

backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadMerge.java

@@ -16,7 +16,6 @@
  */
 package org.apache.gluten.extensions;
 
-import org.apache.iceberg.PlanningMode;
 import org.apache.iceberg.RowLevelOperationMode;
 import org.apache.iceberg.Table;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
@@ -25,36 +24,14 @@
 import org.apache.spark.sql.execution.SparkPlan;
 import org.apache.spark.sql.internal.SQLConf;
 import org.junit.Test;
-
-import java.util.Map;
+import org.junit.jupiter.api.TestTemplate;
 
 import static org.apache.iceberg.RowLevelOperationMode.COPY_ON_WRITE;
 import static org.apache.iceberg.TableProperties.MERGE_MODE;
 import static org.apache.iceberg.TableProperties.MERGE_MODE_DEFAULT;
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class TestGlutenMergeOnReadMerge extends TestMergeOnReadMerge {
-  public TestGlutenMergeOnReadMerge(
-      String catalogName,
-      String implementation,
-      Map<String, String> config,
-      String fileFormat,
-      boolean vectorized,
-      String distributionMode,
-      boolean fanoutEnabled,
-      String branch,
-      PlanningMode planningMode) {
-    super(
-        catalogName,
-        implementation,
-        config,
-        fileFormat,
-        vectorized,
-        distributionMode,
-        fanoutEnabled,
-        branch,
-        planningMode);
-  }
 
   @Test
   public synchronized void testMergeWithConcurrentTableRefresh() {
@@ -72,7 +49,7 @@ public synchronized void testMergeWithSnapshotIsolation() {
   }
 
   // The matched join string is changed from Join to ShuffledHashJoinExecTransformer
-  @Test
+  @TestTemplate
   public void testMergeConditionSplitIntoTargetPredicateAndJoinCondition() {
     createAndInitTable(
         "id INT, salary INT, dep STRING, sub_dep STRING",

backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenMergeOnReadUpdate.java

@@ -16,35 +16,12 @@
  */
 package org.apache.gluten.extensions;
 
-import org.apache.iceberg.PlanningMode;
 import org.apache.iceberg.spark.extensions.TestMergeOnReadUpdate;
 import org.junit.Test;
 
-import java.util.Map;
 import java.util.concurrent.ExecutionException;
 
 public class TestGlutenMergeOnReadUpdate extends TestMergeOnReadUpdate {
-  public TestGlutenMergeOnReadUpdate(
-      String catalogName,
-      String implementation,
-      Map<String, String> config,
-      String fileFormat,
-      boolean vectorized,
-      String distributionMode,
-      boolean fanoutEnabled,
-      String branch,
-      PlanningMode planningMode) {
-    super(
-        catalogName,
-        implementation,
-        config,
-        fileFormat,
-        vectorized,
-        distributionMode,
-        fanoutEnabled,
-        branch,
-        planningMode);
-  }
 
   @Test
   public synchronized void testUpdateWithConcurrentTableRefresh() {

backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenStoragePartitionedJoinsInRowLevelOperations.java

@@ -18,12 +18,5 @@
 
 import org.apache.iceberg.spark.extensions.TestStoragePartitionedJoinsInRowLevelOperations;
 
-import java.util.Map;
-
 public class TestGlutenStoragePartitionedJoinsInRowLevelOperations
-    extends TestStoragePartitionedJoinsInRowLevelOperations {
-  public TestGlutenStoragePartitionedJoinsInRowLevelOperations(
-      String catalogName, String implementation, Map<String, String> config) {
-    super(catalogName, implementation, config);
-  }
-}
+    extends TestStoragePartitionedJoinsInRowLevelOperations {}

backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownDQL.java

@@ -18,11 +18,4 @@
 
 import org.apache.iceberg.spark.extensions.TestSystemFunctionPushDownDQL;
 
-import java.util.Map;
-
-public class TestGlutenSystemFunctionPushDownDQL extends TestSystemFunctionPushDownDQL {
-  public TestGlutenSystemFunctionPushDownDQL(
-      String catalogName, String implementation, Map<String, String> config) {
-    super(catalogName, implementation, config);
-  }
-}
+public class TestGlutenSystemFunctionPushDownDQL extends TestSystemFunctionPushDownDQL {}

backends-velox/src-iceberg-spark34/test/java/org/apache/gluten/extensions/TestGlutenSystemFunctionPushDownInRowLevelOperations.java

@@ -16,12 +16,5 @@
  */
 package org.apache.gluten.extensions;
 
-import java.util.Map;
-
 public class TestGlutenSystemFunctionPushDownInRowLevelOperations
-    extends TestGlutenSystemFunctionPushDownDQL {
-  public TestGlutenSystemFunctionPushDownInRowLevelOperations(
-      String catalogName, String implementation, Map<String, String> config) {
-    super(catalogName, implementation, config);
-  }
-}
+    extends TestGlutenSystemFunctionPushDownDQL {}