Core: Upgrade Jetty to 12.1.5

aws/src/integration/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java

@@ -37,10 +37,10 @@
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
 import org.apache.iceberg.rest.auth.OAuth2Properties;
 import org.apache.iceberg.util.ThreadPools;
+import org.eclipse.jetty.ee10.servlet.ServletContextHandler;
+import org.eclipse.jetty.ee10.servlet.ServletHolder;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.handler.gzip.GzipHandler;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;

build.gradle

@@ -379,11 +379,14 @@ project(':iceberg-core') {
     compileOnly(libs.hadoop3.client) {
       exclude group: 'org.apache.avro', module: 'avro'
       exclude group: 'org.slf4j', module: 'slf4j-log4j12'
+      exclude group: "org.eclipse.jetty"
     }
 
     testImplementation libs.jetty.servlet
     testImplementation libs.jakarta.servlet
     testImplementation libs.jetty.server
+    testImplementation libs.jetty.compression.server
+    testImplementation libs.jetty.compression.gzip
     testImplementation libs.mockserver.netty
     testImplementation libs.mockserver.client.java
     testImplementation libs.sqlite.jdbc
@@ -1089,6 +1092,8 @@ project(':iceberg-open-api') {
 
     testFixturesImplementation libs.jetty.servlet
     testFixturesImplementation libs.jetty.server
+    testFixturesImplementation libs.jetty.compression.server
+    testFixturesImplementation libs.jetty.compression.gzip
     testFixturesImplementation libs.sqlite.jdbc
 
     testFixturesCompileOnly libs.apiguardian

core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java

@@ -325,7 +325,8 @@ public <T extends RESTResponse> T handleRequest(
 
           String eTag = ETagProvider.of(response.metadataLocation());
 
-          if (ifNoneMatchHeader.isPresent() && eTag.equals(ifNoneMatchHeader.get().value())) {
+          if (ifNoneMatchHeader.isPresent()
+              && eTag.equals(stripETagSuffix(ifNoneMatchHeader.get().value()))) {
             return null;
           }
 
@@ -738,4 +739,18 @@ private static SnapshotMode snapshotModeFromQueryParams(Map<String, String> quer
             .getOrDefault("snapshots", RESTCatalogProperties.SNAPSHOT_LOADING_MODE_DEFAULT)
             .toUpperCase(Locale.US));
   }
+
+  /**
+   * Strips the compression suffix from an ETag value.
+   *
+   * <p>Jetty 12 appends "--gzip" to ETags when compression is enabled. This method removes such
+   * suffixes to allow proper ETag comparison.
+   */
+  static String stripETagSuffix(String eTag) {
+    if (eTag == null) {
+      return null;
+    }
+    int suffixIndex = eTag.indexOf("--");
+    return suffixIndex > 0 ? eTag.substring(0, suffixIndex) : eTag;
+  }
 }

core/src/test/java/org/apache/iceberg/rest/TestBaseWithRESTServer.java

@@ -34,10 +34,10 @@
 import org.apache.iceberg.inmemory.InMemoryCatalog;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
 import org.apache.iceberg.rest.responses.ErrorResponse;
+import org.eclipse.jetty.compression.server.CompressionHandler;
+import org.eclipse.jetty.ee10.servlet.ServletContextHandler;
+import org.eclipse.jetty.ee10.servlet.ServletHolder;
 import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.handler.gzip.GzipHandler;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.io.TempDir;
@@ -75,7 +75,7 @@ public void before() throws Exception {
         new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
     servletContext.addServlet(
         new ServletHolder(new RESTCatalogServlet(adapterForRESTServer)), "/*");
-    servletContext.setHandler(new GzipHandler());
+    servletContext.setHandler(new CompressionHandler());
 
     this.httpServer = new Server(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0));
     httpServer.setHandler(servletContext);

core/src/test/java/org/apache/iceberg/rest/TestFreshnessAwareLoading.java

@@ -20,6 +20,7 @@
 
 import static org.apache.iceberg.TestBase.FILE_A;
 import static org.apache.iceberg.TestBase.SCHEMA;
+import static org.apache.iceberg.rest.RESTCatalogAdapter.stripETagSuffix;
 import static org.apache.iceberg.rest.RESTTableCache.SessionIdTableId;
 import static org.apache.iceberg.rest.RESTTableCache.TableWithETag;
 import static org.apache.iceberg.rest.RequestMatcher.matches;
@@ -184,7 +185,12 @@ public void notModifiedResponse() {
 
               assertThat(originalRequest.headers().contains(HttpHeaders.IF_NONE_MATCH));
               assertThat(
-                      originalRequest.headers().firstEntry(HttpHeaders.IF_NONE_MATCH).get().value())
+                      stripETagSuffix(
+                          originalRequest
+                              .headers()
+                              .firstEntry(HttpHeaders.IF_NONE_MATCH)
+                              .get()
+                              .value()))
                   .isEqualTo(eTag);
 
               assertThat(

core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java

@@ -113,10 +113,10 @@
 import org.apache.iceberg.util.Pair;
 import org.assertj.core.api.InstanceOfAssertFactories;
 import org.awaitility.Awaitility;
+import org.eclipse.jetty.ee10.servlet.ServletContextHandler;
+import org.eclipse.jetty.ee10.servlet.ServletHolder;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.handler.gzip.GzipHandler;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -409,6 +409,15 @@ protected boolean requiresNamespaceCreate() {
     return true;
   }
 
+  @Override
+  protected boolean supportsNamesWithSlashes() {
+    // names with slashes are rejected and considered as suspicious characters after upgrading Jetty
+    // and the Servlet API. See also
+    // https://jakarta.ee/specifications/servlet/6.0/jakarta-servlet-spec-6.0.html#uri-path-canonicalization
+    // for additional details
+    return false;
+  }
+
   /* RESTCatalog specific tests */
 
   @Test

core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java

@@ -57,10 +57,10 @@
 import org.apache.iceberg.rest.responses.LoadViewResponse;
 import org.apache.iceberg.view.ViewCatalogTests;
 import org.apache.iceberg.view.ViewMetadata;
+import org.eclipse.jetty.ee10.servlet.ServletContextHandler;
+import org.eclipse.jetty.ee10.servlet.ServletHolder;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.handler.gzip.GzipHandler;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;

core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalogWithAssumedViewSupport.java

@@ -31,10 +31,10 @@
 import org.apache.iceberg.inmemory.InMemoryCatalog;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
 import org.apache.iceberg.rest.responses.ConfigResponse;
+import org.eclipse.jetty.ee10.servlet.ServletContextHandler;
+import org.eclipse.jetty.ee10.servlet.ServletHolder;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.handler.gzip.GzipHandler;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
 import org.junit.jupiter.api.BeforeEach;
 
 public class TestRESTViewCatalogWithAssumedViewSupport extends TestRESTViewCatalog {

gradle/libs.versions.toml

@@ -69,7 +69,7 @@ jakarta-el-api = "3.0.3"
 jakarta-servlet-api = "6.1.0"
 jaxb-api = "2.3.1"
 jaxb-runtime = "2.3.9"
-jetty = "11.0.26"
+jetty = "12.1.5"
 junit = "5.14.2"
 junit-platform = "1.14.2"
 junit-pioneer = "2.3.0"
@@ -208,7 +208,9 @@ guava-testlib = { module = "com.google.guava:guava-testlib", version.ref = "guav
 jakarta-el-api = { module = "jakarta.el:jakarta.el-api", version.ref = "jakarta-el-api" }
 jakarta-servlet = {module = "jakarta.servlet:jakarta.servlet-api", version.ref = "jakarta-servlet-api"}
 jetty-server = { module = "org.eclipse.jetty:jetty-server", version.ref = "jetty" }
-jetty-servlet = { module = "org.eclipse.jetty:jetty-servlet", version.ref = "jetty" }
+jetty-servlet = { module = "org.eclipse.jetty.ee10:jetty-ee10-servlet", version.ref = "jetty" }
+jetty-compression-server = { module = "org.eclipse.jetty.compression:jetty-compression-server", version.ref = "jetty" }
+jetty-compression-gzip = { module = "org.eclipse.jetty.compression:jetty-compression-gzip", version.ref = "jetty" }
 junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit" }
 junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit" }
 junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version.ref = "junit-pioneer" }

open-api/src/test/java/org/apache/iceberg/rest/RESTCompatibilityKitCatalogTests.java

@@ -97,4 +97,13 @@ protected boolean supportsNamesWithDot() {
     return PropertyUtil.propertyAsBoolean(
         restCatalog.properties(), RESTCompatibilityKitSuite.RCK_SUPPORTS_NAMES_WITH_DOT, false);
   }
+
+  @Override
+  protected boolean supportsNamesWithSlashes() {
+    // names with slashes are rejected and considered as suspicious characters after upgrading Jetty
+    // and the Servlet API. See also
+    // https://jakarta.ee/specifications/servlet/6.0/jakarta-servlet-spec-6.0.html#uri-path-canonicalization
+    // for additional details
+    return false;
+  }
 }

open-api/src/testFixtures/java/org/apache/iceberg/rest/RESTCatalogServer.java

@@ -28,12 +28,12 @@
 import org.apache.iceberg.jdbc.JdbcCatalog;
 import org.apache.iceberg.relocated.com.google.common.collect.Maps;
 import org.apache.iceberg.util.PropertyUtil;
+import org.eclipse.jetty.compression.server.CompressionHandler;
+import org.eclipse.jetty.ee10.servlet.ServletContextHandler;
+import org.eclipse.jetty.ee10.servlet.ServletHolder;
 import org.eclipse.jetty.server.Connector;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
-import org.eclipse.jetty.server.handler.gzip.GzipHandler;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -115,7 +115,7 @@ public void start(boolean join) throws Exception {
     ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
     ServletHolder servletHolder = new ServletHolder(servlet);
     context.addServlet(servletHolder, "/*");
-    context.insertHandler(new GzipHandler());
+    context.insertHandler(new CompressionHandler());
 
     this.httpServer =
         new Server(

spark/v3.4/build.gradle

@@ -107,9 +107,7 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
     testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
     testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
     testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
-    testImplementation (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
-      transitive = false
-    }
+    testImplementation (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements'))
     testImplementation libs.sqlite.jdbc
     testImplementation libs.awaitility
     // runtime dependencies for running REST Catalog based integration test
@@ -174,11 +172,7 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer
     testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
     testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
     testImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
-    testImplementation (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
-      transitive = false
-    }
-    // runtime dependencies for running REST Catalog based integration test
-    testRuntimeOnly libs.jetty.servlet
+    testImplementation project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')
     testRuntimeOnly libs.sqlite.jdbc
 
     testImplementation libs.avro.avro
@@ -272,10 +266,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio
     integrationRuntimeOnly project(':iceberg-hive-metastore')
     // runtime dependencies for running REST Catalog based integration test
     integrationRuntimeOnly project(path: ':iceberg-core', configuration: 'testArtifacts')
-    integrationRuntimeOnly (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
-      transitive = false
-    }
-    integrationRuntimeOnly libs.jetty.servlet
+    integrationRuntimeOnly project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')
     integrationRuntimeOnly libs.sqlite.jdbc
 
     // Not allowed on our classpath, only the runtime jar is allowed

spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java

@@ -18,9 +18,6 @@
  */
 package org.apache.iceberg.spark.extensions;
 
-import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS;
-
-import java.net.InetAddress;
 import java.util.Random;
 import java.util.concurrent.ThreadLocalRandom;
 import org.apache.iceberg.CatalogUtil;
@@ -49,14 +46,11 @@ public static void startMetastoreAndSpark() {
     TestBase.spark =
         SparkSession.builder()
             .master("local[2]")
-            .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress())
+            .config(baseConfigs(hiveConf))
             .config("spark.testing", "true")
-            .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic")
             .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName())
-            .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname))
             .config("spark.sql.shuffle.partitions", "4")
             .config("spark.sql.hive.metastorePartitionPruningFallbackOnException", "true")
-            .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true")
             .config(
                 SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean()))
             .enableHiveSupport()

spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java

@@ -89,6 +89,7 @@ protected static SparkSession initSpark(String serializer) {
     return SparkSession.builder()
         .master("local[2]")
         .config("spark.serializer", serializer)
+        .config(org.apache.iceberg.spark.TestBase.DISABLE_UI_CONFIGS)
         .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4")
         .getOrCreate();
   }

spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java

@@ -67,6 +67,7 @@ public static void startSpark() {
     TestSparkDistributedDataScanDeletes.spark =
         SparkSession.builder()
             .master("local[2]")
+            .config(org.apache.iceberg.spark.TestBase.DISABLE_UI_CONFIGS)
             .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
             .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4")
             .getOrCreate();

spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java

@@ -60,6 +60,7 @@ public static void startSpark() {
     TestSparkDistributedDataScanFilterFiles.spark =
         SparkSession.builder()
             .master("local[2]")
+            .config(org.apache.iceberg.spark.TestBase.DISABLE_UI_CONFIGS)
             .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
             .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4")
             .getOrCreate();

spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java

@@ -57,6 +57,7 @@ public static void startSpark() {
     TestSparkDistributedDataScanReporting.spark =
         SparkSession.builder()
             .master("local[2]")
+            .config(org.apache.iceberg.spark.TestBase.DISABLE_UI_CONFIGS)
             .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
             .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4")
             .getOrCreate();

spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/DummyMetricsServlet.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark;
+
+import com.codahale.metrics.MetricRegistry;
+import java.util.Properties;
+import org.apache.spark.SparkConf;
+import org.apache.spark.metrics.sink.MetricsServlet;
+import org.sparkproject.jetty.servlet.ServletContextHandler;
+
+/**
+ * A dummy implementation of {@link MetricsServlet} that does not start a server or report metrics.
+ * This is used in tests to avoid conflicts with Spark's jetty dependencies.
+ */
+public class DummyMetricsServlet extends MetricsServlet {
+
+  /**
+   * Constructor required by Spark's reflection-based instantiation.
+   *
+   * @param properties Metrics properties
+   * @param registry Metric registry
+   */
+  public DummyMetricsServlet(Properties properties, MetricRegistry registry) {
+    super(properties, registry);
+  }
+
+  @Override
+  public ServletContextHandler[] getHandlers(SparkConf conf) {
+    return new ServletContextHandler[] {};
+  }
+
+  @Override
+  public void start() {
+    // No-op for tests
+  }
+
+  @Override
+  public void stop() {
+    // No-op for tests
+  }
+
+  @Override
+  public void report() {
+    // No-op for tests
+  }
+}