diff --git a/.github/workflows/static-checks.yml b/.github/workflows/static-checks.yml index 35f01b60a152..d92baff39767 100644 --- a/.github/workflows/static-checks.yml +++ b/.github/workflows/static-checks.yml @@ -42,7 +42,7 @@ jobs: fail-fast: false matrix: # Use JDK 21.0.4 to work around https://github.com/apache/druid/issues/17429 - java: [ '11', '17', '21.0.4' ] + java: [ '17', '21.0.4' ] runs-on: ubuntu-latest steps: - name: checkout branch diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index 9f0736466352..fa5defa5be84 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -57,7 +57,7 @@ jobs: fail-fast: false matrix: # Use JDK 21.0.4 to work around https://github.com/apache/druid/issues/17429 - jdk: [ '11', '17', '21.0.4' ] + jdk: [ '17', '21.0.4' ] runs-on: ubuntu-latest steps: - name: Checkout branch diff --git a/docs/configuration/index.md b/docs/configuration/index.md index c925fe964c14..c8fc8c561637 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -196,9 +196,9 @@ and `druid.tlsPort` properties on each service. Please see `Configuration` secti Druid uses Jetty as an embedded web server. To learn more about TLS/SSL, certificates, and related concepts in Jetty, including explanations of the configuration settings below, see "Configuring SSL/TLS KeyStores" in the [Jetty Operations Guide](https://www.eclipse.org/jetty/documentation.php). -For information about TLS/SSL support in Java in general, see the [Java Secure Socket Extension (JSSE) Reference Guide](https://docs.oracle.com/en/java/javase/11/security/java-secure-socket-extension-jsse-reference-guide.html). +For information about TLS/SSL support in Java in general, see the [Java Secure Socket Extension (JSSE) Reference Guide](https://docs.oracle.com/en/java/javase/17/security/java-secure-socket-extension-jsse-reference-guide.html). The [Java Cryptography Architecture -Standard Algorithm Name Documentation for JDK 11](https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html) lists all possible +Standard Algorithm Name Documentation for JDK 17](https://docs.oracle.com/en/java/javase/17/docs/specs/security/standard-names.html) lists all possible values for the following properties, among others provided by the Java implementation. |Property|Description|Default|Required| @@ -231,7 +231,7 @@ These properties apply to the SSLContext that will be provided to the internal H |`druid.client.https.trustStoreAlgorithm`|Algorithm to be used by TrustManager to validate certificate chains|`javax.net.ssl.TrustManagerFactory.getDefaultAlgorithm()`|no| |`druid.client.https.trustStorePassword`|The [Password Provider](../operations/password-provider.md) or String password for the Trust Store.|none|yes| -This [document](https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html) lists all the possible +This [document](https://docs.oracle.com/en/java/javase/17/docs/specs/security/standard-names.html) lists all the possible values for the above mentioned configs among others provided by Java implementation. ### Authentication and authorization @@ -1502,6 +1502,7 @@ Druid uses Jetty to serve HTTP requests. |`druid.server.http.enableForwardedRequestCustomizer`|If enabled, adds Jetty ForwardedRequestCustomizer which reads X-Forwarded-* request headers to manipulate servlet request object when Druid is used behind a proxy.|false| |`druid.server.http.allowedHttpMethods`|List of HTTP methods that should be allowed in addition to the ones required by Druid APIs. Druid APIs require GET, PUT, POST, and DELETE, which are always allowed. This option is not useful unless you have installed an extension that needs these additional HTTP methods or that adds functionality related to CORS. None of Druid's bundled extensions require these methods.|`[]`| |`druid.server.http.contentSecurityPolicy`|Content-Security-Policy header value to set on each non-POST response. Setting this property to an empty string, or omitting it, both result in the default `frame-ancestors: none` being set.|`frame-ancestors 'none'`| +|`druid.server.http.uriCompliance`|Jetty `UriCompliance` mode for Druid's embedded Jetty servers. To modify, override this config with the string representation of any `UriCompliance` mode that [Jetty supports](https://javadoc.jetty.org/jetty-12/org/eclipse/jetty/http/UriCompliance.html).|LEGACY| #### Indexer processing resources diff --git a/docs/development/extensions-core/simple-client-sslcontext.md b/docs/development/extensions-core/simple-client-sslcontext.md index db1a37afe414..981e00107a45 100644 --- a/docs/development/extensions-core/simple-client-sslcontext.md +++ b/docs/development/extensions-core/simple-client-sslcontext.md @@ -23,9 +23,9 @@ title: "Simple SSLContext Provider Module" --> -This Apache Druid module contains a simple implementation of [SSLContext](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/javax/net/ssl/SSLContext.html) +This Apache Druid module contains a simple implementation of [SSLContext](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/javax/net/ssl/SSLContext.html) that will be injected to be used with HttpClient that Druid processes use internally to communicate with each other. To learn more about -Java's SSL support, please refer to [this](https://docs.oracle.com/en/java/javase/11/security/java-secure-socket-extension-jsse-reference-guide.html) guide. +Java's SSL support, please refer to [this](https://docs.oracle.com/en/java/javase/17/security/java-secure-socket-extension-jsse-reference-guide.html) guide. |Property|Description|Default|Required| @@ -48,5 +48,5 @@ The following table contains optional parameters for supporting client certifica |`druid.client.https.keyManagerPassword`|The [Password Provider](../../operations/password-provider.md) or String password for the Key Manager.|none|no| |`druid.client.https.validateHostnames`|Validate the hostname of the server. This should not be disabled unless you are using [custom TLS certificate checks](../../operations/tls-support.md) and know that standard hostname validation is not needed.|true|no| -This [document](https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html) lists all the possible +This [document](https://docs.oracle.com/en/java/javase/17/docs/specs/security/standard-names.html) lists all the possible values for the above mentioned configs among others provided by Java implementation. diff --git a/docs/ingestion/input-sources.md b/docs/ingestion/input-sources.md index be9a7456052d..b469889494f1 100644 --- a/docs/ingestion/input-sources.md +++ b/docs/ingestion/input-sources.md @@ -184,7 +184,7 @@ Sample specs: |uris|JSON array of URIs where S3 objects to be ingested are located.|None|`uris` or `prefixes` or `objects` must be set| |prefixes|JSON array of URI prefixes for the locations of S3 objects to be ingested. Empty objects starting with one of the given prefixes will be skipped.|None|`uris` or `prefixes` or `objects` must be set| |objects|JSON array of S3 Objects to be ingested.|None|`uris` or `prefixes` or `objects` must be set| -|objectGlob|A glob for the object part of the S3 URI. In the URI `s3://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `s3://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)).|None|no| +|objectGlob|A glob for the object part of the S3 URI. In the URI `s3://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `s3://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)).|None|no| |systemFields|JSON array of system fields to return as part of input rows. Possible values: `__file_uri` (S3 URI starting with `s3://`), `__file_bucket` (S3 bucket), and `__file_path` (S3 object key).|None|no| | endpointConfig |Config for overriding the default S3 endpoint and signing region. This would allow ingesting data from a different S3 store. Please see [s3 config](../development/extensions-core/s3.md#connecting-to-s3-configuration) for more information.|None|No (defaults will be used if not given) | clientConfig |S3 client properties for the overridden s3 endpoint. This is used in conjunction with `endPointConfig`. Please see [s3 config](../development/extensions-core/s3.md#connecting-to-s3-configuration) for more information.|None|No (defaults will be used if not given) @@ -289,7 +289,7 @@ Sample specs: |uris|JSON array of URIs where Google Cloud Storage objects to be ingested are located.|None|`uris` or `prefixes` or `objects` must be set| |prefixes|JSON array of URI prefixes for the locations of Google Cloud Storage objects to be ingested. Empty objects starting with one of the given prefixes will be skipped.|None|`uris` or `prefixes` or `objects` must be set| |objects|JSON array of Google Cloud Storage objects to be ingested.|None|`uris` or `prefixes` or `objects` must be set| -|objectGlob|A glob for the object part of the S3 URI. In the URI `s3://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `s3://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)).|None|no| +|objectGlob|A glob for the object part of the S3 URI. In the URI `s3://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `s3://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)).|None|no| Note that the Google Cloud Storage input source will skip all empty objects only when `prefixes` is specified. @@ -377,7 +377,7 @@ Sample specs: |uris|JSON array of URIs where the Azure objects to be ingested are located. Use this format: `azureStorage://STORAGE_ACCOUNT/CONTAINER/PATH_TO_FILE`|None|One of the following must be set:`uris`, `prefixes`, or `objects`.| |prefixes|JSON array of URI prefixes for the locations of Azure objects to ingest. Use this format`azureStorage://STORAGE_ACCOUNT/CONTAINER/PREFIX`. Empty objects starting with any of the given prefixes are skipped.|None|One of the following must be set:`uris`, `prefixes`, or `objects`.| |objects|JSON array of Azure objects to ingest.|None|One of the following must be set:`uris`, `prefixes`, or `objects`.| -|objectGlob|A glob for the object part of the Azure URI. In the URI `azureStorage://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `azureStorage://foo/bar/file.json` because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)).|None|no| +|objectGlob|A glob for the object part of the Azure URI. In the URI `azureStorage://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `azureStorage://foo/bar/file.json` because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)).|None|no| |systemFields|JSON array of system fields to return as part of input rows. Possible values: `__file_uri` (Azure blob URI starting with `azureStorage://`), `__file_bucket` (Azure bucket), and `__file_path` (Azure object path).|None|no| |properties|Properties object for overriding the default Azure configuration. See below for more information.|None|No (defaults will be used if not given)| @@ -471,7 +471,7 @@ Sample specs: |uris|JSON array of URIs where the Azure objects to be ingested are located, in the form `azure:///`|None|`uris` or `prefixes` or `objects` must be set| |prefixes|JSON array of URI prefixes for the locations of Azure objects to ingest, in the form `azure:///`. Empty objects starting with one of the given prefixes are skipped.|None|`uris` or `prefixes` or `objects` must be set| |objects|JSON array of Azure objects to ingest.|None|`uris` or `prefixes` or `objects` must be set| -|objectGlob|A glob for the object part of the Azure URI. In the URI `azure://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `azure://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)).|None|no| +|objectGlob|A glob for the object part of the Azure URI. In the URI `azure://foo/bar/file.json`, the glob is applied to `bar/file.json`.

The glob must match the entire object part, not just the filename. For example, the glob `*.json` does not match `azure://foo/bar/file.json`, because the object part is `bar/file.json`, and the`*` does not match the slash. To match all objects ending in `.json`, use `**.json` instead.

For more information, refer to the documentation for [`FileSystem#getPathMatcher`](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)).|None|no| |systemFields|JSON array of system fields to return as part of input rows. Possible values: `__file_uri` (Azure blob URI starting with `azure://`), `__file_bucket` (Azure bucket), and `__file_path` (Azure object path).|None|no| Note that the Azure input source skips all empty objects only when `prefixes` is specified. diff --git a/docs/operations/dump-segment.md b/docs/operations/dump-segment.md index f8c99366ad2c..f80b6275192d 100644 --- a/docs/operations/dump-segment.md +++ b/docs/operations/dump-segment.md @@ -36,7 +36,7 @@ java -classpath "/my/druid/lib/*" -Ddruid.extensions.loadList="[]" org.apache.dr --out /home/druid/output.txt ``` -If you use JDK 11 and above, you need to add the following additional parameters +If you use JDK 17 and above, you need to add the following additional parameters ``` --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED diff --git a/docs/operations/java.md b/docs/operations/java.md index 0b8a474ac950..765122e107f7 100644 --- a/docs/operations/java.md +++ b/docs/operations/java.md @@ -27,7 +27,7 @@ a Java runtime for Druid. ## Selecting a Java runtime - The project team recommends Java 17. Although you can use Java 11, support for it is deprecated. +Druid officially supports Java 17. The project team recommends using an OpenJDK-based Java distribution. There are many free and actively-supported distributions available, including @@ -43,7 +43,7 @@ Druid relies on the environment variables `JAVA_HOME` or `DRUID_JAVA_HOME` to fi ## Garbage collection In general, the project team recommends using the G1 collector with default settings. This is the default collector in -Java 11 and 17. +Java 17. Garbage collector selection and tuning is a form of sport in the Java community. There may be situations where adjusting garbage collection configuration improves or worsens performance. The project team's guidance is that most people do @@ -51,18 +51,10 @@ not need to stray away from G1 with default settings. ## Strong encapsulation -Java 9 and beyond (including Java 11 and 17) include the capability for +Java 9 and beyond (including Java 17) include the capability for [strong encapsulation](https://dev.java/learn/strong-encapsulation-\(of-jdk-internals\)/) of internal JDK APIs. Druid uses certain internal JDK APIs, which must be added to `--add-exports` and `--add-opens` on the Java command line. -On Java 11, if these parameters are not included, you will see warnings like the following: - -``` -WARNING: An illegal reflective access operation has occurred -WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations -WARNING: All illegal access operations will be denied in a future release -``` - On Java 17, if these parameters are not included, you will see errors on startup like the following: ``` diff --git a/docs/operations/tls-support.md b/docs/operations/tls-support.md index 543f177bfedd..994f5f27441b 100644 --- a/docs/operations/tls-support.md +++ b/docs/operations/tls-support.md @@ -37,10 +37,10 @@ Apache Druid uses Jetty as its embedded web server. To get familiar with TLS/SSL, along with related concepts like keys and certificates, read [Configuring Secure Protocols](https://www.eclipse.org/jetty/documentation/jetty-12/operations-guide/index.html#og-protocols-ssl) in the Jetty documentation. -To get more in-depth knowledge of TLS/SSL support in Java in general, refer to the [Java Secure Socket Extension (JSSE) Reference Guide](https://docs.oracle.com/en/java/javase/11/security/java-secure-socket-extension-jsse-reference-guide.html). -The [Class SslContextFactory](https://www.eclipse.org/jetty/javadoc/jetty-11/org/eclipse/jetty/util/ssl/SslContextFactory.html) +To get more in-depth knowledge of TLS/SSL support in Java in general, refer to the [Java Secure Socket Extension (JSSE) Reference Guide](https://docs.oracle.com/en/java/javase/17/security/java-secure-socket-extension-jsse-reference-guide.html). +The [Class SslContextFactory](https://javadoc.jetty.org/jetty-12/org/eclipse/jetty/util/ssl/SslContextFactory.html) reference doc can help in understanding TLS/SSL configurations listed below. Finally, [Java Cryptography Architecture -Standard Algorithm Name Documentation for JDK 11](https://docs.oracle.com/en/java/javase/11/docs/specs/security/standard-names.html) lists all possible +Standard Algorithm Name Documentation for JDK 17](https://docs.oracle.com/en/java/javase/17/docs/specs/security/standard-names.html) lists all possible values for the configs below, among others provided by Java implementation. |Property|Description|Default|Required| @@ -79,7 +79,7 @@ The following table contains non-mandatory advanced configuration options, use c ## Internal communication over TLS Whenever possible Druid processes will use HTTPS to talk to each other. To enable this communication Druid's HttpClient needs to -be configured with a proper [SSLContext](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/javax/net/ssl/SSLContext.html) that is able +be configured with a proper [SSLContext](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/javax/net/ssl/SSLContext.html) that is able to validate the Server Certificates, otherwise communication will fail. Since, there are various ways to configure SSLContext, by default, Druid looks for an instance of SSLContext Guice binding diff --git a/docs/querying/dimensionspecs.md b/docs/querying/dimensionspecs.md index 68773abdae10..dba52abcc689 100644 --- a/docs/querying/dimensionspecs.md +++ b/docs/querying/dimensionspecs.md @@ -255,7 +255,7 @@ For a regular dimension, it assumes the string is formatted in [ISO-8601 date and time format](https://en.wikipedia.org/wiki/ISO_8601). * `format` : date time format for the resulting dimension value, in [Joda Time DateTimeFormat](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html), or null to use the default ISO8601 format. -* `locale` : locale (language and country) to use, given as a [IETF BCP 47 language tag](https://www.oracle.com/java/technologies/javase/jdk11-suported-locales.html#util-text), e.g. `en-US`, `en-GB`, `fr-FR`, `fr-CA`, etc. +* `locale` : locale (language and country) to use, given as a [IETF BCP 47 language tag](https://www.oracle.com/java/technologies/javase/jdk17-suported-locales.html#util-text), e.g. `en-US`, `en-GB`, `fr-FR`, `fr-CA`, etc. * `timeZone` : time zone to use in [IANA tz database format](http://en.wikipedia.org/wiki/List_of_tz_database_time_zones), e.g. `Europe/Berlin` (this can possibly be different than the aggregation time-zone) * `granularity` : [granularity](granularities.md) to apply before formatting, or omit to not apply any granularity. * `asMillis` : boolean value, set to true to treat input strings as millis rather than ISO8601 strings. Additionally, if `format` is null or not specified, output will be in millis rather than ISO8601. diff --git a/docs/querying/filters.md b/docs/querying/filters.md index a9a862a49985..d088d92641ea 100644 --- a/docs/querying/filters.md +++ b/docs/querying/filters.md @@ -431,7 +431,7 @@ The regular expression filter is similar to the selector filter, but using regul | -------- | ----------- | -------- | | `type` | Must be `regex`.| Yes | | `dimension` | Input column or virtual column name to filter on. | Yes | -| `pattern` | String pattern to match - any standard [Java regular expression](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/util/regex/Pattern.html). | Yes | +| `pattern` | String pattern to match - any standard [Java regular expression](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/regex/Pattern.html). | Yes | | `extractionFn` | [Extraction function](./dimensionspecs.md#extraction-functions) to apply to `dimension` prior to value matching. See [filtering with extraction functions](#filtering-with-extraction-functions) for details. | No | Note that it is often more optimal to use a like filter instead of a regex for simple matching of prefixes. diff --git a/docs/querying/math-expr.md b/docs/querying/math-expr.md index 926446200f15..06ac395c7ad4 100644 --- a/docs/querying/math-expr.md +++ b/docs/querying/math-expr.md @@ -81,7 +81,7 @@ The following built-in functions are available. |name|description| |----|-----------| |concat|concat(expr, expr...) concatenate a list of strings| -|format|format(pattern[, args...]) returns a string formatted in the manner of Java's [String.format](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/String.html#format(java.lang.String,java.lang.Object...)).| +|format|format(pattern[, args...]) returns a string formatted in the manner of Java's [String.format](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/String.html#format(java.lang.String,java.lang.Object...)).| |like|like(expr, pattern[, escape]) is equivalent to SQL `expr LIKE pattern`| |lookup|lookup(expr, lookup-name[,replaceMissingValueWith]) looks up expr in a registered,`replaceMissingValueWith` is an optional constant string [query-time lookup](../querying/lookups.md)| |parse_long|parse_long(string[, radix]) parses a string as a long with the given radix, or 10 (decimal) if a radix is not provided.| diff --git a/docs/querying/sql-scalar.md b/docs/querying/sql-scalar.md index 05f29436da0f..59c3e2893a86 100644 --- a/docs/querying/sql-scalar.md +++ b/docs/querying/sql-scalar.md @@ -115,7 +115,7 @@ String functions accept strings and return a type appropriate to the function. |`REPLACE(expr, substring, replacement)`|Replaces instances of `substring` in `expr` with `replacement` and returns the result.| |`REPEAT(expr, N)`|Repeats `expr` `N` times.| |`REVERSE(expr)`|Reverses `expr`.| -|`STRING_FORMAT(pattern[, args...])`|Returns a string formatted in the manner of Java's [String.format](https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/String.html#format(java.lang.String,java.lang.Object...)).| +|`STRING_FORMAT(pattern[, args...])`|Returns a string formatted in the manner of Java's [String.format](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/String.html#format(java.lang.String,java.lang.Object...)).| |`STRPOS(expr, substring)`|Returns the index of `substring` within `expr`, with indexes starting from 1. If `substring` is not found, returns 0.| |`SUBSTRING(expr, index[, length])`|Returns a substring of `expr` starting at a given one-based index. If `length` is omitted, extracts characters to the end of the string, otherwise returns a substring of `length` UTF-16 characters.| |`SUBSTR(expr, index[, length])`|Alias for `SUBSTRING`.| diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index 06528d2881ca..cf33bbdb1bf6 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -239,6 +239,10 @@ org.codehaus.jackson jackson-mapper-asl + + org.eclipse.jetty + jetty-servlet + @@ -302,8 +306,8 @@ provided - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee8 + jetty-ee8-servlet provided javax.xml.bind:jaxb-api + org.eclipse.jetty.toolchain:jetty-servlet-api diff --git a/processing/pom.xml b/processing/pom.xml index 175059576671..0ac4aaaa3085 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -421,8 +421,8 @@ test - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee8 + jetty-ee8-servlet test diff --git a/quidem-ut/pom.xml b/quidem-ut/pom.xml index fbb1795f5ce7..d5a6b51931f4 100644 --- a/quidem-ut/pom.xml +++ b/quidem-ut/pom.xml @@ -352,8 +352,8 @@ runtime - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee8 + jetty-ee8-servlet ${jetty.version} diff --git a/server/pom.xml b/server/pom.xml index c42b73017354..98dc2ce9a8eb 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -125,8 +125,8 @@ jetty-util - org.eclipse.jetty - jetty-proxy + org.eclipse.jetty.ee8 + jetty-ee8-proxy org.eclipse.jetty @@ -160,12 +160,12 @@ lz4-java - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee8 + jetty-ee8-servlet - org.eclipse.jetty - jetty-servlets + org.eclipse.jetty.ee8 + jetty-ee8-servlets org.apache.derby diff --git a/server/src/main/java/org/apache/druid/guice/http/JettyHttpClientModule.java b/server/src/main/java/org/apache/druid/guice/http/JettyHttpClientModule.java index 4ef5192e28e4..da543448af8b 100644 --- a/server/src/main/java/org/apache/druid/guice/http/JettyHttpClientModule.java +++ b/server/src/main/java/org/apache/druid/guice/http/JettyHttpClientModule.java @@ -32,7 +32,9 @@ import org.apache.druid.java.util.common.lifecycle.Lifecycle; import org.apache.druid.server.DruidNode; import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.transport.HttpClientTransportDynamic; import org.eclipse.jetty.http.HttpField; +import org.eclipse.jetty.io.ClientConnector; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.jboss.netty.handler.codec.http.HttpHeaders; @@ -91,7 +93,9 @@ public HttpClient get() if (sslContextBinding != null) { final SslContextFactory.Client sslContextFactory = new SslContextFactory.Client(); sslContextFactory.setSslContext(sslContextBinding.getProvider().get()); - httpClient = new HttpClient(sslContextFactory); + ClientConnector clientConnector = new ClientConnector(); + clientConnector.setSslContextFactory(sslContextFactory); + httpClient = new HttpClient(new HttpClientTransportDynamic(clientConnector)); } else { httpClient = new HttpClient(); } diff --git a/server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java b/server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java index 602fe4ec3d2b..92db1824a8ab 100644 --- a/server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java +++ b/server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java @@ -94,6 +94,16 @@ public CoreInjectorBuilder withLifecycle() } public CoreInjectorBuilder forServer() + { + forServerWithoutJetty(); + add(JettyServerModule.class); + return this; + } + + /** + * Needed for Hadoop indexing that needs server-like Injector but can't run jetty 12 + */ + public CoreInjectorBuilder forServerWithoutJetty() { withLogging(); withLifecycle(); @@ -113,7 +123,6 @@ public CoreInjectorBuilder forServer() new SegmentWriteOutMediumModule(), new ServerModule(), new StorageNodeModule(), - new JettyServerModule(), new ExpressionModule(), new BuiltInTypesModule(), new DiscoveryModule(), diff --git a/server/src/main/java/org/apache/druid/initialization/ServerInjectorBuilder.java b/server/src/main/java/org/apache/druid/initialization/ServerInjectorBuilder.java index fd441b958bf7..ff63c9cf34ac 100644 --- a/server/src/main/java/org/apache/druid/initialization/ServerInjectorBuilder.java +++ b/server/src/main/java/org/apache/druid/initialization/ServerInjectorBuilder.java @@ -73,6 +73,21 @@ public static Injector makeServerInjector( .build(); } + /** + * Needed for Hadoop indexing that needs server-like Injector but can't run jetty 12 + */ + public static Injector makeServerInjectorWithoutJettyModules( + final Injector baseInjector, + final Set nodeRoles, + final Iterable modules + ) + { + return new ServerInjectorBuilder(baseInjector) + .nodeRoles(nodeRoles) + .serviceModules(modules) + .buildWithoutJettyModules(); + } + public ServerInjectorBuilder(Injector baseInjector) { this.baseInjector = baseInjector; @@ -91,6 +106,19 @@ public ServerInjectorBuilder serviceModules(final Iterable mod } public Injector build() + { + return this.build(true); + } + + /** + * Needed for Hadoop indexing that needs server-like Injector but can't run jetty 12 + */ + public Injector buildWithoutJettyModules() + { + return this.build(false); + } + + private Injector build(boolean withJettyModules) { Preconditions.checkNotNull(baseInjector); Preconditions.checkNotNull(nodeRoles); @@ -103,7 +131,12 @@ public Injector build() // Create the core set of modules shared by all services. // Here and below, the modules are filtered by the load modules list and // the set of roles which this server provides. - CoreInjectorBuilder coreBuilder = new CoreInjectorBuilder(childInjector, nodeRoles).forServer(); + CoreInjectorBuilder coreBuilder; + if (withJettyModules) { + coreBuilder = new CoreInjectorBuilder(childInjector, nodeRoles).forServer(); + } else { + coreBuilder = new CoreInjectorBuilder(childInjector, nodeRoles).forServerWithoutJetty(); + } // Override with the per-service modules. ServiceInjectorBuilder serviceBuilder = (ServiceInjectorBuilder) new ServiceInjectorBuilder(coreBuilder).addAll( diff --git a/server/src/main/java/org/apache/druid/server/AsyncManagementForwardingServlet.java b/server/src/main/java/org/apache/druid/server/AsyncManagementForwardingServlet.java index 4c8db118d37b..97b1a6721180 100644 --- a/server/src/main/java/org/apache/druid/server/AsyncManagementForwardingServlet.java +++ b/server/src/main/java/org/apache/druid/server/AsyncManagementForwardingServlet.java @@ -35,9 +35,9 @@ import org.apache.druid.server.security.AuthorizationUtils; import org.apache.druid.server.security.AuthorizerMapper; import org.eclipse.jetty.client.HttpClient; -import org.eclipse.jetty.client.api.Request; -import org.eclipse.jetty.client.api.Response; -import org.eclipse.jetty.proxy.AsyncProxyServlet; +import org.eclipse.jetty.client.Request; +import org.eclipse.jetty.client.Response; +import org.eclipse.jetty.ee8.proxy.AsyncProxyServlet; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; diff --git a/server/src/main/java/org/apache/druid/server/QueryResultPusher.java b/server/src/main/java/org/apache/druid/server/QueryResultPusher.java index 4208f984cee1..2ee9c0d5d96f 100644 --- a/server/src/main/java/org/apache/druid/server/QueryResultPusher.java +++ b/server/src/main/java/org/apache/druid/server/QueryResultPusher.java @@ -39,7 +39,6 @@ import org.apache.druid.query.context.ResponseContext; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.ForbiddenException; -import org.eclipse.jetty.http.HttpFields; import org.eclipse.jetty.http.HttpHeader; import javax.annotation.Nullable; @@ -52,6 +51,7 @@ import java.io.Closeable; import java.io.IOException; import java.io.OutputStream; +import java.util.HashMap; import java.util.Map; public abstract class QueryResultPusher @@ -67,7 +67,7 @@ public abstract class QueryResultPusher private final QueryResource.QueryMetricCounter counter; private final MediaType contentType; private final Map extraHeaders; - private final HttpFields trailerFields; + private final Map trailerFields; private StreamingHttpResponseAccumulator accumulator; private AsyncContext asyncContext; @@ -92,7 +92,7 @@ public QueryResultPusher( this.counter = counter; this.contentType = contentType; this.extraHeaders = extraHeaders; - this.trailerFields = new HttpFields(); + this.trailerFields = new HashMap<>(); } /** @@ -151,16 +151,11 @@ public Response push() response.setHeader(entry.getKey(), entry.getValue()); } - if (response instanceof org.eclipse.jetty.server.Response) { - org.eclipse.jetty.server.Response jettyResponse = (org.eclipse.jetty.server.Response) response; + response.setHeader(HttpHeader.TRAILER.toString(), RESULT_TRAILER_HEADERS); + response.setTrailerFields(() -> trailerFields); - jettyResponse.setHeader(HttpHeader.TRAILER.toString(), RESULT_TRAILER_HEADERS); - jettyResponse.setTrailers(() -> trailerFields); - - // Start with complete status - - trailerFields.put(QueryResource.RESPONSE_COMPLETE_TRAILER_HEADER, "true"); - } + // Start with complete status + trailerFields.put(QueryResource.RESPONSE_COMPLETE_TRAILER_HEADER, "true"); accumulator = new StreamingHttpResponseAccumulator(queryResponse.getResponseContext(), resultsWriter); @@ -427,10 +422,7 @@ public void initialize() response.setContentType(contentType.toString()); - if (response instanceof org.eclipse.jetty.server.Response) { - org.eclipse.jetty.server.Response jettyResponse = (org.eclipse.jetty.server.Response) response; - jettyResponse.setTrailers(() -> trailerFields); - } + response.setTrailerFields(() -> trailerFields); try { out = new CountingOutputStream(response.getOutputStream()); diff --git a/server/src/main/java/org/apache/druid/server/http/OverlordProxyServlet.java b/server/src/main/java/org/apache/druid/server/http/OverlordProxyServlet.java index a43a5ce379d6..484d7858a0da 100644 --- a/server/src/main/java/org/apache/druid/server/http/OverlordProxyServlet.java +++ b/server/src/main/java/org/apache/druid/server/http/OverlordProxyServlet.java @@ -28,8 +28,8 @@ import org.apache.druid.server.JettyUtils; import org.apache.druid.server.security.AuthConfig; import org.eclipse.jetty.client.HttpClient; -import org.eclipse.jetty.client.api.Request; -import org.eclipse.jetty.proxy.ProxyServlet; +import org.eclipse.jetty.client.Request; +import org.eclipse.jetty.ee8.proxy.ProxyServlet; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; diff --git a/server/src/main/java/org/apache/druid/server/initialization/ServerConfig.java b/server/src/main/java/org/apache/druid/server/initialization/ServerConfig.java index 8fdef26a4105..45a2df48dfa0 100644 --- a/server/src/main/java/org/apache/druid/server/initialization/ServerConfig.java +++ b/server/src/main/java/org/apache/druid/server/initialization/ServerConfig.java @@ -20,6 +20,14 @@ package org.apache.druid.server.initialization; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import org.apache.druid.common.exception.ErrorResponseTransformStrategy; @@ -29,12 +37,14 @@ import org.apache.druid.query.QueryContexts; import org.apache.druid.server.SubqueryGuardrailHelper; import org.apache.druid.utils.JvmUtils; +import org.eclipse.jetty.http.UriCompliance; import org.joda.time.Period; import javax.annotation.Nullable; import javax.validation.constraints.Max; import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; +import java.io.IOException; import java.util.List; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -73,7 +83,8 @@ public ServerConfig( boolean showDetailedJettyErrors, @NotNull ErrorResponseTransformStrategy errorResponseTransformStrategy, @Nullable String contentSecurityPolicy, - boolean enableHSTS + boolean enableHSTS, + @Nullable UriCompliance uriCompliance ) { this.numThreads = numThreads; @@ -97,6 +108,7 @@ public ServerConfig( this.errorResponseTransformStrategy = errorResponseTransformStrategy; this.contentSecurityPolicy = contentSecurityPolicy; this.enableHSTS = enableHSTS; + this.uriCompliance = uriCompliance != null ? uriCompliance : UriCompliance.LEGACY; } public ServerConfig() @@ -185,6 +197,11 @@ public ServerConfig(boolean enableQueryRequestsQueuing) @JsonProperty private boolean enableHSTS = false; + @JsonProperty + @JsonDeserialize(using = UriComplianceDeserializer.class) + @JsonSerialize(using = UriComplianceSerializer.class) + private UriCompliance uriCompliance = UriCompliance.LEGACY; + /** * This feature flag enables query requests queuing when admins want to reserve some threads for * non-query requests. This feature flag is not documented and will be removed in the future. @@ -306,6 +323,11 @@ public boolean isEnableQueryRequestsQueuing() return enableQueryRequestsQueuing; } + public UriCompliance getUriCompliance() + { + return uriCompliance; + } + @Override public boolean equals(Object o) { @@ -337,7 +359,8 @@ public boolean equals(Object o) errorResponseTransformStrategy.equals(that.errorResponseTransformStrategy) && Objects.equals(contentSecurityPolicy, that.getContentSecurityPolicy()) && enableHSTS == that.enableHSTS && - enableQueryRequestsQueuing == that.enableQueryRequestsQueuing; + enableQueryRequestsQueuing == that.enableQueryRequestsQueuing && + Objects.equals(uriCompliance, that.uriCompliance); } @Override @@ -365,7 +388,8 @@ public int hashCode() showDetailedJettyErrors, contentSecurityPolicy, enableHSTS, - enableQueryRequestsQueuing + enableQueryRequestsQueuing, + uriCompliance ); } @@ -395,6 +419,7 @@ public String toString() ", contentSecurityPolicy=" + contentSecurityPolicy + ", enableHSTS=" + enableHSTS + ", enableQueryRequestsQueuing=" + enableQueryRequestsQueuing + + ", uriCompliance=" + uriCompliance + '}'; } @@ -402,4 +427,23 @@ public static int getDefaultNumThreads() { return Math.max(10, (JvmUtils.getRuntimeInfo().getAvailableProcessors() * 17) / 16 + 2) + 30; } + + public static class UriComplianceDeserializer extends JsonDeserializer + { + @Override + public UriCompliance deserialize(JsonParser p, DeserializationContext ctxt) throws IOException + { + String value = p.getValueAsString(); + return UriCompliance.valueOf(value); + } + } + + public static class UriComplianceSerializer extends JsonSerializer + { + @Override + public void serialize(UriCompliance value, JsonGenerator gen, SerializerProvider serializers) throws IOException + { + gen.writeString(value.getName()); + } + } } diff --git a/server/src/main/java/org/apache/druid/server/initialization/jetty/CliIndexerServerModule.java b/server/src/main/java/org/apache/druid/server/initialization/jetty/CliIndexerServerModule.java index feb61965daa3..4782211da492 100644 --- a/server/src/main/java/org/apache/druid/server/initialization/jetty/CliIndexerServerModule.java +++ b/server/src/main/java/org/apache/druid/server/initialization/jetty/CliIndexerServerModule.java @@ -166,7 +166,8 @@ public ServerConfig makeAdjustedServerConfig(ServerConfig oldConfig) oldConfig.isShowDetailedJettyErrors(), oldConfig.getErrorResponseTransformStrategy(), oldConfig.getContentSecurityPolicy(), - oldConfig.isEnableHSTS() + oldConfig.isEnableHSTS(), + oldConfig.getUriCompliance() ); } } diff --git a/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyBindings.java b/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyBindings.java index d955ce984849..3934e33277b0 100644 --- a/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyBindings.java +++ b/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyBindings.java @@ -22,8 +22,8 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Binder; import com.google.inject.multibindings.Multibinder; +import org.eclipse.jetty.ee8.servlets.QoSFilter; import org.eclipse.jetty.server.Handler; -import org.eclipse.jetty.servlets.QoSFilter; import javax.servlet.DispatcherType; import javax.servlet.Filter; diff --git a/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyMonitoringConnectionFactory.java b/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyMonitoringConnectionFactory.java index 07955bb0339a..85ded3250628 100644 --- a/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyMonitoringConnectionFactory.java +++ b/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyMonitoringConnectionFactory.java @@ -56,7 +56,7 @@ public String getProtocol() public Connection newConnection(Connector connector, EndPoint endPoint) { final Connection connection = connectionFactory.newConnection(connector, endPoint); - connection.addListener( + connection.addEventListener( new Connection.Listener() { @Override diff --git a/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyRequestLog.java b/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyRequestLog.java index c11ab8e3b1fe..1ad600f43fb0 100644 --- a/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyRequestLog.java +++ b/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyRequestLog.java @@ -36,10 +36,10 @@ public void log(Request request, Response response) if (logger.isDebugEnabled()) { logger.debug( "%s %s %s %s %d", - request.getRemoteAddr(), + Request.getRemoteAddr(request), request.getMethod(), request.getHttpURI().toString(), - request.getProtocol(), + request.getConnectionMetaData().getProtocol(), response.getStatus() ); } diff --git a/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerInitUtils.java b/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerInitUtils.java index 9a5efe97af3b..1b94ea2f7ce4 100644 --- a/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerInitUtils.java +++ b/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerInitUtils.java @@ -26,15 +26,16 @@ import org.apache.druid.java.util.common.ISE; import org.apache.druid.server.initialization.ServerConfig; import org.apache.druid.server.security.AllowHttpMethodsResourceFilter; +import org.eclipse.jetty.ee8.servlet.FilterHolder; +import org.eclipse.jetty.ee8.servlet.FilterMapping; +import org.eclipse.jetty.ee8.servlet.ServletContextHandler; import org.eclipse.jetty.rewrite.handler.HeaderPatternRule; import org.eclipse.jetty.rewrite.handler.RewriteHandler; import org.eclipse.jetty.server.Handler; -import org.eclipse.jetty.server.handler.HandlerList; -import org.eclipse.jetty.server.handler.RequestLogHandler; import org.eclipse.jetty.server.handler.gzip.GzipHandler; -import org.eclipse.jetty.servlet.FilterHolder; -import org.eclipse.jetty.servlet.FilterMapping; -import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.util.compression.CompressionPool; +import org.eclipse.jetty.util.compression.DeflaterPool; +import org.eclipse.jetty.util.compression.InflaterPool; import javax.ws.rs.HttpMethod; import java.util.Arrays; @@ -45,16 +46,15 @@ public class JettyServerInitUtils { private static final String[] GZIP_METHODS = new String[]{HttpMethod.GET, HttpMethod.POST}; - public static GzipHandler wrapWithDefaultGzipHandler(final Handler handler, int inflateBufferSize, int compressionLevel) + public static GzipHandler wrapWithDefaultGzipHandler(final ServletContextHandler handler, int inflateBufferSize, int compressionLevel) { GzipHandler gzipHandler = new GzipHandler(); gzipHandler.setMinGzipSize(0); gzipHandler.setIncludedMethods(GZIP_METHODS); gzipHandler.setInflateBufferSize(inflateBufferSize); - gzipHandler.setCompressionLevel(compressionLevel); + gzipHandler.setDeflaterPool(new DeflaterPool(CompressionPool.DEFAULT_CAPACITY, compressionLevel, true)); + gzipHandler.setInflaterPool(new InflaterPool(CompressionPool.DEFAULT_CAPACITY, true)); - // We don't actually have any precomputed .gz resources, and checking for them inside jars is expensive. - gzipHandler.setCheckGzExists(false); gzipHandler.setHandler(handler); return gzipHandler; } @@ -109,15 +109,6 @@ public static void addFilters(ServletContextHandler handler, Set allowedHttpMethods) { FilterHolder holder = new FilterHolder(new AllowHttpMethodsResourceFilter(allowedHttpMethods)); @@ -135,7 +126,7 @@ public static void maybeAddHSTSPatternRule(ServerConfig serverConfig, RewriteHan } } - public static void maybeAddHSTSRewriteHandler(ServerConfig serverConfig, HandlerList handlerList) + public static void maybeAddHSTSRewriteHandler(ServerConfig serverConfig, Handler.Sequence handlerList) { if (serverConfig.isEnableHSTS()) { RewriteHandler rewriteHandler = new RewriteHandler(); diff --git a/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java b/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java index 28381cbcafa4..b2e767a8b974 100644 --- a/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java +++ b/server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java @@ -64,11 +64,13 @@ import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.server.handler.ErrorHandler; +import org.eclipse.jetty.util.Callback; import org.eclipse.jetty.util.component.LifeCycle; import org.eclipse.jetty.util.ssl.KeyStoreScanner; import org.eclipse.jetty.util.ssl.SslContextFactory; @@ -81,11 +83,7 @@ import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509ExtendedTrustManager; import javax.servlet.RequestDispatcher; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import java.io.BufferedReader; -import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; import java.security.KeyStore; @@ -225,6 +223,7 @@ static Server makeAndInitializeServer( if (node.isEnablePlaintextPort()) { log.info("Creating http connector with port [%d]", node.getPlaintextPort()); HttpConfiguration httpConfiguration = new HttpConfiguration(); + httpConfiguration.setUriCompliance(config.getUriCompliance()); if (config.isEnableForwardedRequestCustomizer()) { httpConfiguration.addCustomizer(new ForwardedRequestCustomizer()); } @@ -309,6 +308,7 @@ static Server makeAndInitializeServer( } final HttpConfiguration httpsConfiguration = new HttpConfiguration(); + httpsConfiguration.setUriCompliance(config.getUriCompliance()); if (config.isEnableForwardedRequestCustomizer()) { httpsConfiguration.addCustomizer(new ForwardedRequestCustomizer()); } @@ -364,7 +364,7 @@ static Server makeAndInitializeServer( if (gracefulStop > 0) { server.setStopTimeout(gracefulStop); } - server.addLifeCycleListener(new LifeCycle.Listener() + server.addEventListener(new LifeCycle.Listener() { @Override public void lifeCycleStarting(LifeCycle event) @@ -467,25 +467,20 @@ public void stop() server.setErrorHandler(new ErrorHandler() { @Override - public boolean isShowServlet() - { - return false; - } - - @Override - public void handle( - String target, + public boolean handle( Request baseRequest, - HttpServletRequest request, - HttpServletResponse response - ) throws IOException, ServletException + Response response, + Callback callback + ) throws Exception { - request.setAttribute(RequestDispatcher.ERROR_EXCEPTION, null); - super.handle(target, baseRequest, request, response); + baseRequest.setAttribute(RequestDispatcher.ERROR_EXCEPTION, null); + return super.handle(baseRequest, response, callback); } }); } + server.setRequestLog(new JettyRequestLog()); + return server; } diff --git a/server/src/main/java/org/apache/druid/server/initialization/jetty/StandardResponseHeaderFilterHolder.java b/server/src/main/java/org/apache/druid/server/initialization/jetty/StandardResponseHeaderFilterHolder.java index 2f12ff2402c8..30054abca688 100644 --- a/server/src/main/java/org/apache/druid/server/initialization/jetty/StandardResponseHeaderFilterHolder.java +++ b/server/src/main/java/org/apache/druid/server/initialization/jetty/StandardResponseHeaderFilterHolder.java @@ -24,7 +24,7 @@ import org.apache.commons.lang3.CharUtils; import org.apache.druid.java.util.common.IAE; import org.apache.druid.server.initialization.ServerConfig; -import org.eclipse.jetty.client.api.Response; +import org.eclipse.jetty.client.Response; import javax.annotation.Nullable; import javax.servlet.DispatcherType; @@ -66,7 +66,7 @@ public StandardResponseHeaderFilterHolder(final ServerConfig serverConfig) * Remove any standard headers in proxyResponse if they were also set in the origin response, serverResponse. * This prevents duplicates headers from appearing in proxy responses. * - * Used by implementations of {@link org.eclipse.jetty.proxy.AsyncProxyServlet}. + * Used by implementations of {@link org.eclipse.jetty.ee8.proxy.AsyncProxyServlet}. */ public static void deduplicateHeadersInProxyServlet( final HttpServletResponse proxyResponse, @@ -74,8 +74,9 @@ public static void deduplicateHeadersInProxyServlet( ) { for (final String headerName : StandardResponseHeaderFilterHolder.STANDARD_HEADERS) { - if (serverResponse.getHeaders().containsKey(headerName) && proxyResponse.containsHeader(headerName)) { - ((org.eclipse.jetty.server.Response) proxyResponse).getHttpFields().remove(headerName); + if (serverResponse.getHeaders().contains(headerName) && proxyResponse.containsHeader(headerName)) { + // In EE8 compatible Jetty 12 using servlet API 4.x, setting a header to null is the accepted way to remove it. + proxyResponse.setHeader(headerName, null); } } } diff --git a/server/src/main/java/org/apache/druid/server/security/AuthenticationUtils.java b/server/src/main/java/org/apache/druid/server/security/AuthenticationUtils.java index d4fba7388abf..9acde0d16175 100644 --- a/server/src/main/java/org/apache/druid/server/security/AuthenticationUtils.java +++ b/server/src/main/java/org/apache/druid/server/security/AuthenticationUtils.java @@ -20,8 +20,8 @@ package org.apache.druid.server.security; import com.fasterxml.jackson.databind.ObjectMapper; -import org.eclipse.jetty.servlet.FilterHolder; -import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.ee8.servlet.FilterHolder; +import org.eclipse.jetty.ee8.servlet.ServletContextHandler; import java.util.List; diff --git a/server/src/main/java/org/apache/druid/server/security/Authenticator.java b/server/src/main/java/org/apache/druid/server/security/Authenticator.java index 0afb9109f0f9..ef73cb186ed5 100644 --- a/server/src/main/java/org/apache/druid/server/security/Authenticator.java +++ b/server/src/main/java/org/apache/druid/server/security/Authenticator.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.apache.druid.server.initialization.jetty.ServletFilterHolder; -import org.eclipse.jetty.client.api.Request; +import org.eclipse.jetty.client.Request; import javax.annotation.Nullable; import javax.servlet.Filter; diff --git a/server/src/test/java/org/apache/druid/initialization/ServerConfigTest.java b/server/src/test/java/org/apache/druid/initialization/ServerConfigTest.java index 96745e163a00..913c7f6109af 100644 --- a/server/src/test/java/org/apache/druid/initialization/ServerConfigTest.java +++ b/server/src/test/java/org/apache/druid/initialization/ServerConfigTest.java @@ -25,6 +25,7 @@ import org.apache.druid.common.exception.AllowedRegexErrorResponseTransformStrategy; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.server.initialization.ServerConfig; +import org.eclipse.jetty.http.UriCompliance; import org.junit.Assert; import org.junit.Test; @@ -43,6 +44,7 @@ public void testSerde() throws Exception Assert.assertEquals(defaultConfig, defaultConfig2); Assert.assertFalse(defaultConfig2.isEnableForwardedRequestCustomizer()); Assert.assertFalse(defaultConfig2.isEnableHSTS()); + Assert.assertEquals(UriCompliance.LEGACY, defaultConfig.getUriCompliance()); ServerConfig modifiedConfig = new ServerConfig( 999, @@ -65,7 +67,8 @@ public void testSerde() throws Exception true, new AllowedRegexErrorResponseTransformStrategy(ImmutableList.of(".*")), "my-cool-policy", - true + true, + UriCompliance.RFC3986 ); String modifiedConfigJson = OBJECT_MAPPER.writeValueAsString(modifiedConfig); ServerConfig modifiedConfig2 = OBJECT_MAPPER.readValue(modifiedConfigJson, ServerConfig.class); @@ -79,6 +82,7 @@ public void testSerde() throws Exception Assert.assertEquals("my-cool-policy", modifiedConfig.getContentSecurityPolicy()); Assert.assertEquals("my-cool-policy", modifiedConfig2.getContentSecurityPolicy()); Assert.assertTrue(modifiedConfig2.isEnableHSTS()); + Assert.assertEquals(UriCompliance.RFC3986, modifiedConfig2.getUriCompliance()); } @Test diff --git a/server/src/test/java/org/apache/druid/server/AsyncManagementForwardingServletTest.java b/server/src/test/java/org/apache/druid/server/AsyncManagementForwardingServletTest.java index b49520021441..c576f966cf65 100644 --- a/server/src/test/java/org/apache/druid/server/AsyncManagementForwardingServletTest.java +++ b/server/src/test/java/org/apache/druid/server/AsyncManagementForwardingServletTest.java @@ -44,13 +44,11 @@ import org.apache.druid.server.security.AuthenticationUtils; import org.apache.druid.server.security.AuthorizerMapper; import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.ee8.servlet.DefaultServlet; +import org.eclipse.jetty.ee8.servlet.ServletContextHandler; +import org.eclipse.jetty.ee8.servlet.ServletHolder; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.handler.HandlerList; -import org.eclipse.jetty.servlet.DefaultServlet; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHandler; -import org.eclipse.jetty.servlet.ServletHolder; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -429,8 +427,8 @@ public void testLocalRequest() throws Exception private static Server makeTestServer(int port, ExpectedRequest expectedRequest) { Server server = new Server(port); - ServletHandler handler = new ServletHandler(); - handler.addServletWithMapping(new ServletHolder(new HttpServlet() + ServletContextHandler servletContextHandler = new ServletContextHandler(); + servletContextHandler.addServlet(new ServletHolder(new HttpServlet() { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException @@ -475,7 +473,7 @@ private void handle(HttpServletRequest req, HttpServletResponse resp) throws IOE } }), "/*"); - server.setHandler(handler); + server.setHandler(servletContextHandler); return server; } @@ -534,12 +532,12 @@ public String getCurrentLeader() AuthenticationUtils.addAuthenticationFilterChain(root, ImmutableList.of(new AllowAllAuthenticator())); JettyServerInitUtils.addExtensionFilters(root, injector); - final HandlerList handlerList = new HandlerList(); - handlerList.setHandlers( - new Handler[]{JettyServerInitUtils.wrapWithDefaultGzipHandler( + final Handler.Sequence handlerList = new Handler.Sequence( + JettyServerInitUtils.wrapWithDefaultGzipHandler( root, ServerConfig.DEFAULT_GZIP_INFLATE_BUFFER_SIZE, - Deflater.DEFAULT_COMPRESSION)} + Deflater.DEFAULT_COMPRESSION + ) ); server.setHandler(handlerList); } diff --git a/server/src/test/java/org/apache/druid/server/QueryResourceTest.java b/server/src/test/java/org/apache/druid/server/QueryResourceTest.java index ed3c7b0b4a08..684c2c157342 100644 --- a/server/src/test/java/org/apache/druid/server/QueryResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/QueryResourceTest.java @@ -74,7 +74,6 @@ import org.apache.druid.server.log.TestRequestLogger; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.mocks.ExceptionalInputStream; -import org.apache.druid.server.mocks.MockAsyncContext; import org.apache.druid.server.mocks.MockHttpServletRequest; import org.apache.druid.server.mocks.MockHttpServletResponse; import org.apache.druid.server.scheduling.HiLoQueryLaningStrategy; @@ -91,11 +90,7 @@ import org.apache.druid.server.security.ForbiddenException; import org.apache.druid.server.security.Resource; import org.apache.http.HttpStatus; -import org.easymock.EasyMock; -import org.eclipse.jetty.http.HttpFields; import org.eclipse.jetty.http.HttpHeader; -import org.eclipse.jetty.server.HttpChannel; -import org.eclipse.jetty.server.HttpOutput; import org.hamcrest.CoreMatchers; import org.hamcrest.MatcherAssert; import org.joda.time.Interval; @@ -121,6 +116,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; @@ -275,7 +271,15 @@ public void testGoodQuery() throws IOException { expectPermissiveHappyPathAuth(); - Assert.assertEquals(200, expectAsyncRequestFlow(SIMPLE_TIMESERIES_QUERY).getStatus()); + HttpServletResponse servletResponse = expectAsyncRequestFlow(SIMPLE_TIMESERIES_QUERY); + Assert.assertEquals(200, servletResponse.getStatus()); + Assert.assertTrue(servletResponse.containsHeader(HttpHeader.TRAILER.toString())); + + final Map fields = servletResponse.getTrailerFields().get(); + Assert.assertFalse(fields.containsKey(QueryResource.ERROR_MESSAGE_TRAILER_HEADER)); + + Assert.assertTrue(fields.containsKey(QueryResource.RESPONSE_COMPLETE_TRAILER_HEADER)); + Assert.assertEquals(fields.get(QueryResource.RESPONSE_COMPLETE_TRAILER_HEADER), "true"); } @Test @@ -487,16 +491,12 @@ public QueryRunner getQueryRunnerForSegments( expectPermissiveHappyPathAuth(); - org.eclipse.jetty.server.Response response = this.jettyResponseforRequest(testServletRequest); - Assert.assertNull(queryResource.doPost(new ByteArrayInputStream( - SIMPLE_TIMESERIES_QUERY.getBytes(StandardCharsets.UTF_8)), - null /*pretty*/, - testServletRequest - )); + HttpServletResponse response = expectAsyncRequestFlow(testServletRequest, SIMPLE_TIMESERIES_QUERY.getBytes(StandardCharsets.UTF_8), queryResource); + Assert.assertTrue(response.containsHeader(HttpHeader.TRAILER.toString())); Assert.assertEquals(response.getHeader(HttpHeader.TRAILER.toString()), QueryResultPusher.RESULT_TRAILER_HEADERS); - final HttpFields fields = response.getTrailers().get(); + final Map fields = response.getTrailerFields().get(); Assert.assertTrue(fields.containsKey(QueryResource.ERROR_MESSAGE_TRAILER_HEADER)); Assert.assertEquals( fields.get(QueryResource.ERROR_MESSAGE_TRAILER_HEADER), @@ -507,49 +507,6 @@ public QueryRunner getQueryRunnerForSegments( Assert.assertEquals(fields.get(QueryResource.RESPONSE_COMPLETE_TRAILER_HEADER), "false"); } - @Test - public void testSuccessResponseWithTrailerHeader() throws IOException - { - queryResource = new QueryResource( - new QueryLifecycleFactory( - CONGLOMERATE, - TEST_SEGMENT_WALKER, - new DefaultGenericQueryMetricsFactory(), - new NoopServiceEmitter(), - testRequestLogger, - new AuthConfig(), - NoopPolicyEnforcer.instance(), - AuthTestUtils.TEST_AUTHORIZER_MAPPER, - Suppliers.ofInstance(new DefaultQueryConfig(ImmutableMap.of())) - ), - jsonMapper, - queryScheduler, - null, - new QueryResourceQueryResultPusherFactory( - jsonMapper, - ResponseContextConfig.newConfig(true), - DRUID_NODE - ), - new ResourceIOReaderWriterFactory(jsonMapper, smileMapper) - ); - - expectPermissiveHappyPathAuth(); - - org.eclipse.jetty.server.Response response = this.jettyResponseforRequest(testServletRequest); - Assert.assertNull(queryResource.doPost(new ByteArrayInputStream( - SIMPLE_TIMESERIES_QUERY.getBytes(StandardCharsets.UTF_8)), - null /*pretty*/, - testServletRequest - )); - Assert.assertTrue(response.containsHeader(HttpHeader.TRAILER.toString())); - - final HttpFields fields = response.getTrailers().get(); - Assert.assertFalse(fields.containsKey(QueryResource.ERROR_MESSAGE_TRAILER_HEADER)); - - Assert.assertTrue(fields.containsKey(QueryResource.RESPONSE_COMPLETE_TRAILER_HEADER)); - Assert.assertEquals(fields.get(QueryResource.RESPONSE_COMPLETE_TRAILER_HEADER), "true"); - } - @Test public void testResponseContextContainsMissingSegments_whenLastSegmentIsMissing() throws IOException { @@ -635,29 +592,20 @@ public QueryRunner getQueryRunnerForSegments(Query query, Iterable fields = response.getTrailerFields().get(); Assert.assertTrue(response.containsHeader(QueryResource.HEADER_RESPONSE_CONTEXT)); Assert.assertEquals( jsonMapper.writeValueAsString(ImmutableMap.of("missingSegments", ImmutableList.of(missingSegDesc))), response.getHeader(QueryResource.HEADER_RESPONSE_CONTEXT) ); - Assert.assertTrue(observedFields.containsKey(QueryResource.RESPONSE_COMPLETE_TRAILER_HEADER)); - Assert.assertEquals("true", observedFields.get(QueryResource.RESPONSE_COMPLETE_TRAILER_HEADER)); + Assert.assertTrue(fields.containsKey(QueryResource.RESPONSE_COMPLETE_TRAILER_HEADER)); + Assert.assertEquals("true", fields.get(QueryResource.RESPONSE_COMPLETE_TRAILER_HEADER)); } @@ -1782,29 +1730,4 @@ private Response expectSynchronousRequestFlow( { return queryResource.doPost(new ByteArrayInputStream(bytes), null, req); } - - private org.eclipse.jetty.server.Response jettyResponseforRequest(MockHttpServletRequest req) throws IOException - { - HttpChannel channelMock = EasyMock.mock(HttpChannel.class); - HttpOutput outputMock = EasyMock.mock(HttpOutput.class); - org.eclipse.jetty.server.Response response = new org.eclipse.jetty.server.Response(channelMock, outputMock); - - EasyMock.expect(channelMock.isSendError()).andReturn(false); - EasyMock.expect(channelMock.isCommitted()).andReturn(true); - - outputMock.close(); - EasyMock.expectLastCall().andVoid(); - - outputMock.write(EasyMock.anyObject(byte[].class), EasyMock.anyInt(), EasyMock.anyInt()); - EasyMock.expectLastCall().andVoid(); - - EasyMock.replay(outputMock, channelMock); - - req.newAsyncContext(() -> { - final MockAsyncContext retVal = new MockAsyncContext(); - retVal.response = response; - return retVal; - }); - return response; - } } diff --git a/server/src/test/java/org/apache/druid/server/initialization/BaseJettyTest.java b/server/src/test/java/org/apache/druid/server/initialization/BaseJettyTest.java index 422d336e3bf1..ec0a9ffbcef2 100644 --- a/server/src/test/java/org/apache/druid/server/initialization/BaseJettyTest.java +++ b/server/src/test/java/org/apache/druid/server/initialization/BaseJettyTest.java @@ -31,12 +31,11 @@ import org.apache.druid.server.DruidNode; import org.apache.druid.server.initialization.jetty.JettyServerInitUtils; import org.apache.druid.server.initialization.jetty.JettyServerInitializer; +import org.eclipse.jetty.ee8.servlet.DefaultServlet; +import org.eclipse.jetty.ee8.servlet.ServletContextHandler; +import org.eclipse.jetty.ee8.servlet.ServletHolder; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.handler.HandlerList; -import org.eclipse.jetty.servlet.DefaultServlet; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; import org.joda.time.Duration; import org.junit.After; import org.junit.Before; @@ -154,13 +153,12 @@ public void initialize(Server server, Injector injector) JettyServerInitUtils.addExtensionFilters(root, injector); root.addFilter(GuiceFilter.class, "/*", null); - final HandlerList handlerList = new HandlerList(); - handlerList.setHandlers( - new Handler[]{JettyServerInitUtils.wrapWithDefaultGzipHandler( + final Handler.Sequence handlerList = new Handler.Sequence( + JettyServerInitUtils.wrapWithDefaultGzipHandler( root, ServerConfig.DEFAULT_GZIP_INFLATE_BUFFER_SIZE, Deflater.DEFAULT_COMPRESSION - )} + ) ); server.setHandler(handlerList); } diff --git a/server/src/test/java/org/apache/druid/server/initialization/JettyQosTest.java b/server/src/test/java/org/apache/druid/server/initialization/JettyQosTest.java index b0fea90bc44f..a12b2db6da09 100644 --- a/server/src/test/java/org/apache/druid/server/initialization/JettyQosTest.java +++ b/server/src/test/java/org/apache/druid/server/initialization/JettyQosTest.java @@ -45,8 +45,8 @@ import org.apache.druid.server.initialization.jetty.JettyServerInitializer; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.server.security.AuthorizerMapper; +import org.eclipse.jetty.ee8.servlets.QoSFilter; import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.servlets.QoSFilter; import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.jboss.netty.handler.codec.http.HttpMethod; import org.junit.Assert; diff --git a/server/src/test/java/org/apache/druid/server/initialization/JettyTest.java b/server/src/test/java/org/apache/druid/server/initialization/JettyTest.java index a2fcea834347..2185685f54ac 100644 --- a/server/src/test/java/org/apache/druid/server/initialization/JettyTest.java +++ b/server/src/test/java/org/apache/druid/server/initialization/JettyTest.java @@ -563,13 +563,6 @@ public void testCustomCheckX509TrustManagerSetEndpointIdentificationAlgorithmToN Assert.assertTrue(endpointIdentificationAlgorithm == null || endpointIdentificationAlgorithm.isEmpty()); } - @Test - public void testJettyErrorHandlerWithFilter() - { - // Response filter is disabled by default hence we show servlet information - Assert.assertTrue(server.getErrorHandler().isShowServlet()); - } - private void waitForJettyServerModuleActiveConnectionsZero(JettyServerModule jsm) throws InterruptedException { // it can take a bit to close the connection, so maybe sleep for a while and hope it closes diff --git a/server/src/test/java/org/apache/druid/server/initialization/JettyWithResponseFilterEnabledTest.java b/server/src/test/java/org/apache/druid/server/initialization/JettyWithResponseFilterEnabledTest.java index 62e6189d7ded..ea2446fc07c2 100644 --- a/server/src/test/java/org/apache/druid/server/initialization/JettyWithResponseFilterEnabledTest.java +++ b/server/src/test/java/org/apache/druid/server/initialization/JettyWithResponseFilterEnabledTest.java @@ -19,9 +19,6 @@ package org.apache.druid.server.initialization; -import org.junit.Assert; -import org.junit.Test; - public class JettyWithResponseFilterEnabledTest extends JettyTest { @Override @@ -31,12 +28,4 @@ public void setProperties() super.setProperties(); System.setProperty("druid.server.http.showDetailedJettyErrors", "false"); } - - @Test - @Override - public void testJettyErrorHandlerWithFilter() - { - // Response filter is enabled by config hence we do not show servlet information - Assert.assertFalse(server.getErrorHandler().isShowServlet()); - } } diff --git a/server/src/test/java/org/apache/druid/server/initialization/jetty/StandardResponseHeaderFilterHolderTest.java b/server/src/test/java/org/apache/druid/server/initialization/jetty/StandardResponseHeaderFilterHolderTest.java index e85dc6a862fa..6eb2be2eb398 100644 --- a/server/src/test/java/org/apache/druid/server/initialization/jetty/StandardResponseHeaderFilterHolderTest.java +++ b/server/src/test/java/org/apache/druid/server/initialization/jetty/StandardResponseHeaderFilterHolderTest.java @@ -23,6 +23,9 @@ import org.apache.druid.server.initialization.ServerConfig; import org.easymock.Capture; import org.easymock.EasyMock; +import org.eclipse.jetty.client.Response; +import org.eclipse.jetty.http.HttpField; +import org.eclipse.jetty.http.HttpFields; import org.hamcrest.CoreMatchers; import org.hamcrest.MatcherAssert; import org.junit.After; @@ -46,6 +49,9 @@ public class StandardResponseHeaderFilterHolderTest public HttpServletResponse httpResponse; public FilterChain filterChain; + public HttpServletResponse proxyResponse; + public Response clientResponse; + @Before public void setUp() { @@ -53,12 +59,15 @@ public void setUp() httpRequest = EasyMock.strictMock(HttpServletRequest.class); httpResponse = EasyMock.strictMock(HttpServletResponse.class); filterChain = EasyMock.strictMock(FilterChain.class); + + proxyResponse = EasyMock.strictMock(HttpServletResponse.class); + clientResponse = EasyMock.strictMock(Response.class); } @After public void tearDown() { - EasyMock.verify(serverConfig, httpRequest, httpResponse, filterChain); + EasyMock.verify(serverConfig, httpRequest, httpResponse, filterChain, proxyResponse, clientResponse); } @Test @@ -130,6 +139,42 @@ public void test_get_invalidContentSecurityPolicy() ); } + @Test + public void test_deduplicateHeadersInProxyServlet_withDuplicates() + { + EasyMock.expect(proxyResponse.containsHeader("Cache-Control")).andReturn(true).once(); + proxyResponse.setHeader("Cache-Control", null); + EasyMock.expectLastCall().once(); + EasyMock.expect(proxyResponse.containsHeader("Strict-Transport-Security")).andReturn(false).once(); + + EasyMock.expect(clientResponse.getHeaders()) + .andReturn( + HttpFields.from(new HttpField("Cache-Control", "true"), new HttpField("Strict-Transport-Security", "true")) + ).times(3); + + replayAllMocks(); + + StandardResponseHeaderFilterHolder.deduplicateHeadersInProxyServlet(proxyResponse, clientResponse); + } + + @Test + public void test_duplicateHeadersInProxyServlet_withNoDuplicates() + { + EasyMock.expect(proxyResponse.containsHeader("Cache-Control")).andReturn(false).once(); + EasyMock.expect(proxyResponse.containsHeader("Strict-Transport-Security")).andReturn(false).once(); + + EasyMock.expect(clientResponse.getHeaders()) + .andReturn(HttpFields.from( + new HttpField("Cache-Control", "true"), + new HttpField("Strict-Transport-Security", "true") + )) + .times(3); + + replayAllMocks(); + + StandardResponseHeaderFilterHolder.deduplicateHeadersInProxyServlet(proxyResponse, clientResponse); + } + private StandardResponseHeaderFilterHolder.StandardResponseHeaderFilter makeFilter() { return (StandardResponseHeaderFilterHolder.StandardResponseHeaderFilter) @@ -163,6 +208,6 @@ private void runFilterAndVerifyHeaders(final Map expectedHeaders private void replayAllMocks() { - EasyMock.replay(serverConfig, httpRequest, httpResponse, filterChain); + EasyMock.replay(serverConfig, httpRequest, httpResponse, filterChain, proxyResponse, clientResponse); } } diff --git a/server/src/test/java/org/apache/druid/server/mocks/MockHttpServletResponse.java b/server/src/test/java/org/apache/druid/server/mocks/MockHttpServletResponse.java index 62e6ca4a2668..9b21c856f582 100644 --- a/server/src/test/java/org/apache/druid/server/mocks/MockHttpServletResponse.java +++ b/server/src/test/java/org/apache/druid/server/mocks/MockHttpServletResponse.java @@ -34,6 +34,8 @@ import java.util.Collection; import java.util.LinkedHashMap; import java.util.Locale; +import java.util.Map; +import java.util.function.Supplier; /** * A fake HttpServletResponse used in tests. A lot of methods are implemented as @@ -59,6 +61,8 @@ public static MockHttpServletResponse forRequest(MockHttpServletRequest req) new LinkedHashMap<>(), ArrayList::new ); + private Supplier> trailerSupplier; + public final ByteArrayOutputStream baos = new ByteArrayOutputStream(); private int statusCode; @@ -148,6 +152,8 @@ public void addDateHeader(String name, long date) @Override public void setHeader(String name, String value) { + // HttpServletResponse 4.0.1 spec dictates that setHeader overwrites existing values. + headers.removeAll(name); headers.put(name, value); } @@ -337,4 +343,16 @@ public Locale getLocale() { throw new UnsupportedOperationException(); } + + @Override + public void setTrailerFields(Supplier> supplier) + { + this.trailerSupplier = supplier; + } + + @Override + public Supplier> getTrailerFields() + { + return trailerSupplier; + } } diff --git a/services/pom.xml b/services/pom.xml index 5dba67fbb57c..3e63e26bac67 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -99,8 +99,8 @@ jetty-http - org.eclipse.jetty - jetty-proxy + org.eclipse.jetty.ee8 + jetty-ee8-proxy org.eclipse.jetty @@ -155,8 +155,8 @@ jsr311-api - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee8 + jetty-ee8-servlet org.eclipse.jetty diff --git a/services/src/main/java/org/apache/druid/cli/CliOverlord.java b/services/src/main/java/org/apache/druid/cli/CliOverlord.java index c88233244c3c..81a0c1367165 100644 --- a/services/src/main/java/org/apache/druid/cli/CliOverlord.java +++ b/services/src/main/java/org/apache/druid/cli/CliOverlord.java @@ -140,14 +140,13 @@ import org.apache.druid.storage.local.LocalTmpStorageConfig; import org.apache.druid.tasklogs.TaskLogStreamer; import org.apache.druid.tasklogs.TaskLogs; +import org.eclipse.jetty.ee8.servlet.DefaultServlet; +import org.eclipse.jetty.ee8.servlet.FilterHolder; +import org.eclipse.jetty.ee8.servlet.ServletContextHandler; +import org.eclipse.jetty.ee8.servlet.ServletHolder; import org.eclipse.jetty.rewrite.handler.RewriteHandler; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.handler.HandlerList; -import org.eclipse.jetty.servlet.DefaultServlet; -import org.eclipse.jetty.servlet.FilterHolder; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; import java.util.HashMap; import java.util.List; @@ -539,17 +538,13 @@ public void initialize(Server server, Injector injector) RewriteHandler rewriteHandler = WebConsoleJettyServerInitializer.createWebConsoleRewriteHandler(); JettyServerInitUtils.maybeAddHSTSPatternRule(serverConfig, rewriteHandler); - HandlerList handlerList = new HandlerList(); - handlerList.setHandlers( - new Handler[]{ - rewriteHandler, - JettyServerInitUtils.getJettyRequestLogHandler(), - JettyServerInitUtils.wrapWithDefaultGzipHandler( - root, - serverConfig.getInflateBufferSize(), - serverConfig.getCompressionLevel() - ) - } + Handler.Sequence handlerList = new Handler.Sequence( + rewriteHandler, + JettyServerInitUtils.wrapWithDefaultGzipHandler( + root, + serverConfig.getInflateBufferSize(), + serverConfig.getCompressionLevel() + ) ); server.setHandler(handlerList); diff --git a/services/src/main/java/org/apache/druid/cli/CoordinatorJettyServerInitializer.java b/services/src/main/java/org/apache/druid/cli/CoordinatorJettyServerInitializer.java index afb261e09be0..ece5fa2be4ac 100644 --- a/services/src/main/java/org/apache/druid/cli/CoordinatorJettyServerInitializer.java +++ b/services/src/main/java/org/apache/druid/cli/CoordinatorJettyServerInitializer.java @@ -35,14 +35,13 @@ import org.apache.druid.server.security.AuthenticationUtils; import org.apache.druid.server.security.Authenticator; import org.apache.druid.server.security.AuthenticatorMapper; +import org.eclipse.jetty.ee8.servlet.DefaultServlet; +import org.eclipse.jetty.ee8.servlet.FilterHolder; +import org.eclipse.jetty.ee8.servlet.ServletContextHandler; +import org.eclipse.jetty.ee8.servlet.ServletHolder; import org.eclipse.jetty.rewrite.handler.RewriteHandler; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.handler.HandlerList; -import org.eclipse.jetty.servlet.DefaultServlet; -import org.eclipse.jetty.servlet.FilterHolder; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; import java.util.List; import java.util.Properties; @@ -136,17 +135,13 @@ public void initialize(Server server, Injector injector) RewriteHandler rewriteHandler = WebConsoleJettyServerInitializer.createWebConsoleRewriteHandler(); JettyServerInitUtils.maybeAddHSTSPatternRule(serverConfig, rewriteHandler); - HandlerList handlerList = new HandlerList(); - handlerList.setHandlers( - new Handler[]{ - rewriteHandler, - JettyServerInitUtils.getJettyRequestLogHandler(), - JettyServerInitUtils.wrapWithDefaultGzipHandler( - root, - serverConfig.getInflateBufferSize(), - serverConfig.getCompressionLevel() - ) - } + Handler.Sequence handlerList = new Handler.Sequence( + rewriteHandler, + JettyServerInitUtils.wrapWithDefaultGzipHandler( + root, + serverConfig.getInflateBufferSize(), + serverConfig.getCompressionLevel() + ) ); server.setHandler(handlerList); diff --git a/services/src/main/java/org/apache/druid/cli/MiddleManagerJettyServerInitializer.java b/services/src/main/java/org/apache/druid/cli/MiddleManagerJettyServerInitializer.java index 660b946e9aa5..b338aa805fd7 100644 --- a/services/src/main/java/org/apache/druid/cli/MiddleManagerJettyServerInitializer.java +++ b/services/src/main/java/org/apache/druid/cli/MiddleManagerJettyServerInitializer.java @@ -32,13 +32,13 @@ import org.apache.druid.server.security.AuthenticationUtils; import org.apache.druid.server.security.Authenticator; import org.apache.druid.server.security.AuthenticatorMapper; +import org.eclipse.jetty.ee8.servlet.DefaultServlet; +import org.eclipse.jetty.ee8.servlet.FilterHolder; +import org.eclipse.jetty.ee8.servlet.ServletContextHandler; +import org.eclipse.jetty.ee8.servlet.ServletHolder; +import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.DefaultHandler; -import org.eclipse.jetty.server.handler.HandlerList; -import org.eclipse.jetty.servlet.DefaultServlet; -import org.eclipse.jetty.servlet.FilterHolder; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; import java.util.Collections; import java.util.List; @@ -96,11 +96,9 @@ public void initialize(Server server, Injector injector) final FilterHolder guiceFilterHolder = JettyServerInitUtils.getGuiceFilterHolder(injector); root.addFilter(guiceFilterHolder, "/*", null); - final HandlerList handlerList = new HandlerList(); + final Handler.Sequence handlerList = new Handler.Sequence(); JettyServerInitUtils.maybeAddHSTSRewriteHandler(serverConfig, handlerList); - handlerList.addHandler(JettyServerInitUtils.getJettyRequestLogHandler()); - handlerList.addHandler(JettyServerInitUtils.wrapWithDefaultGzipHandler( root, serverConfig.getInflateBufferSize(), diff --git a/services/src/main/java/org/apache/druid/cli/QueryJettyServerInitializer.java b/services/src/main/java/org/apache/druid/cli/QueryJettyServerInitializer.java index 2df6d01b7b6d..a236e887e574 100644 --- a/services/src/main/java/org/apache/druid/cli/QueryJettyServerInitializer.java +++ b/services/src/main/java/org/apache/druid/cli/QueryJettyServerInitializer.java @@ -39,14 +39,13 @@ import org.apache.druid.server.security.AuthenticationUtils; import org.apache.druid.server.security.Authenticator; import org.apache.druid.server.security.AuthenticatorMapper; +import org.eclipse.jetty.ee8.servlet.DefaultServlet; +import org.eclipse.jetty.ee8.servlet.FilterHolder; +import org.eclipse.jetty.ee8.servlet.ServletContextHandler; +import org.eclipse.jetty.ee8.servlet.ServletHolder; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.handler.HandlerList; import org.eclipse.jetty.server.handler.StatisticsHandler; -import org.eclipse.jetty.servlet.DefaultServlet; -import org.eclipse.jetty.servlet.FilterHolder; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; import java.util.Collections; import java.util.List; @@ -149,13 +148,9 @@ public void initialize(Server server, Injector injector) final FilterHolder guiceFilterHolder = JettyServerInitUtils.getGuiceFilterHolder(injector); root.addFilter(guiceFilterHolder, "/*", null); - final HandlerList handlerList = new HandlerList(); - // Do not change the order of the handlers that have already been added - for (Handler handler : server.getHandlers()) { - handlerList.addHandler(handler); - } - - handlerList.addHandler(JettyServerInitUtils.getJettyRequestLogHandler()); + final Handler.Sequence handlerList = new Handler.Sequence( + server.getHandlers() + ); // Add all extension handlers for (Handler handler : extensionHandlers) { diff --git a/services/src/main/java/org/apache/druid/cli/RouterJettyServerInitializer.java b/services/src/main/java/org/apache/druid/cli/RouterJettyServerInitializer.java index 0ca3ef173205..6ec4a38548c1 100644 --- a/services/src/main/java/org/apache/druid/cli/RouterJettyServerInitializer.java +++ b/services/src/main/java/org/apache/druid/cli/RouterJettyServerInitializer.java @@ -40,14 +40,13 @@ import org.apache.druid.server.security.AuthenticatorMapper; import org.apache.druid.sql.avatica.DruidAvaticaJsonHandler; import org.apache.druid.sql.avatica.DruidAvaticaProtobufHandler; +import org.eclipse.jetty.ee8.servlet.DefaultServlet; +import org.eclipse.jetty.ee8.servlet.FilterHolder; +import org.eclipse.jetty.ee8.servlet.ServletContextHandler; +import org.eclipse.jetty.ee8.servlet.ServletHolder; import org.eclipse.jetty.rewrite.handler.RewriteHandler; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.handler.HandlerList; -import org.eclipse.jetty.servlet.DefaultServlet; -import org.eclipse.jetty.servlet.FilterHolder; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; import javax.servlet.Servlet; import java.util.List; @@ -151,17 +150,13 @@ public void initialize(Server server, Injector injector) RewriteHandler rewriteHandler = WebConsoleJettyServerInitializer.createWebConsoleRewriteHandler(); JettyServerInitUtils.maybeAddHSTSPatternRule(serverConfig, rewriteHandler); - final HandlerList handlerList = new HandlerList(); - handlerList.setHandlers( - new Handler[]{ - rewriteHandler, - JettyServerInitUtils.getJettyRequestLogHandler(), - JettyServerInitUtils.wrapWithDefaultGzipHandler( - root, - serverConfig.getInflateBufferSize(), - serverConfig.getCompressionLevel() - ) - } + final Handler.Sequence handlerList = new Handler.Sequence( + rewriteHandler, + JettyServerInitUtils.wrapWithDefaultGzipHandler( + root, + serverConfig.getInflateBufferSize(), + serverConfig.getCompressionLevel() + ) ); server.setHandler(handlerList); } diff --git a/services/src/main/java/org/apache/druid/cli/WebConsoleJettyServerInitializer.java b/services/src/main/java/org/apache/druid/cli/WebConsoleJettyServerInitializer.java index bccc47a15da8..435e59d9166e 100644 --- a/services/src/main/java/org/apache/druid/cli/WebConsoleJettyServerInitializer.java +++ b/services/src/main/java/org/apache/druid/cli/WebConsoleJettyServerInitializer.java @@ -22,10 +22,10 @@ import com.google.common.collect.ImmutableList; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.server.security.AuthenticationUtils; +import org.eclipse.jetty.ee8.servlet.ServletContextHandler; import org.eclipse.jetty.rewrite.handler.RedirectPatternRule; import org.eclipse.jetty.rewrite.handler.RewriteHandler; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.util.resource.Resource; +import org.eclipse.jetty.util.resource.ResourceFactory; import java.util.List; @@ -50,7 +50,7 @@ static void intializeServerForWebConsoleRoot(ServletContextHandler root) root.setInitParameter("org.eclipse.jetty.servlet.Default.redirectWelcome", "true"); root.setWelcomeFiles(new String[]{WEB_CONSOLE_ROOT_DOCUMENT}); - root.setBaseResource(Resource.newClassPathResource("org/apache/druid/console")); + root.setBaseResource(ResourceFactory.root().newClassLoaderResource("org/apache/druid/console", false)); AuthenticationUtils.addNoopAuthenticationAndAuthorizationFilters(root, UNSECURED_PATHS_FOR_UI); AuthenticationUtils.addNoopAuthorizationFilters(root, UNAUTHORIZED_PATHS_FOR_UI); diff --git a/services/src/main/java/org/apache/druid/server/AsyncQueryForwardingServlet.java b/services/src/main/java/org/apache/druid/server/AsyncQueryForwardingServlet.java index dba842ec6896..0fa20e242275 100644 --- a/services/src/main/java/org/apache/druid/server/AsyncQueryForwardingServlet.java +++ b/services/src/main/java/org/apache/druid/server/AsyncQueryForwardingServlet.java @@ -61,14 +61,14 @@ import org.apache.druid.server.security.AuthorizationUtils; import org.apache.druid.sql.http.SqlQuery; import org.apache.druid.sql.http.SqlResource; +import org.eclipse.jetty.client.BytesRequestContent; import org.eclipse.jetty.client.HttpClient; -import org.eclipse.jetty.client.api.Request; -import org.eclipse.jetty.client.api.Response; -import org.eclipse.jetty.client.api.Result; -import org.eclipse.jetty.client.util.BytesContentProvider; +import org.eclipse.jetty.client.Request; +import org.eclipse.jetty.client.Response; +import org.eclipse.jetty.client.Result; +import org.eclipse.jetty.ee8.proxy.AsyncProxyServlet; import org.eclipse.jetty.http.HttpHeader; import org.eclipse.jetty.http.HttpMethod; -import org.eclipse.jetty.proxy.AsyncProxyServlet; import javax.annotation.Nullable; import javax.servlet.ServletException; @@ -438,7 +438,7 @@ protected void sendProxyRequest( byte[] avaticaQuery = (byte[]) clientRequest.getAttribute(AVATICA_QUERY_ATTRIBUTE); if (avaticaQuery != null) { - proxyRequest.content(new BytesContentProvider(avaticaQuery)); + proxyRequest.body(new BytesRequestContent(avaticaQuery)); } final Query query = (Query) clientRequest.getAttribute(QUERY_ATTRIBUTE); @@ -483,9 +483,12 @@ private void setProxyRequestContent(Request proxyRequest, HttpServletRequest cli final ObjectMapper objectMapper = (ObjectMapper) clientRequest.getAttribute(OBJECTMAPPER_ATTRIBUTE); try { byte[] bytes = objectMapper.writeValueAsBytes(content); - proxyRequest.content(new BytesContentProvider(bytes)); - proxyRequest.getHeaders().put(HttpHeader.CONTENT_LENGTH, String.valueOf(bytes.length)); - proxyRequest.getHeaders().put(HttpHeader.CONTENT_TYPE, objectMapper.getFactory() instanceof SmileFactory ? SmileMediaTypes.APPLICATION_JACKSON_SMILE : MediaType.APPLICATION_JSON); + Request.Content requestContent = new BytesRequestContent(bytes); + proxyRequest.body(requestContent); + proxyRequest.headers(headers -> { + headers.put(HttpHeader.CONTENT_LENGTH, String.valueOf(requestContent.getLength())); + headers.put(HttpHeader.CONTENT_TYPE, objectMapper.getFactory() instanceof SmileFactory ? SmileMediaTypes.APPLICATION_JACKSON_SMILE : MediaType.APPLICATION_JSON); + }); } catch (JsonProcessingException e) { throw new RuntimeException(e); diff --git a/services/src/test/java/org/apache/druid/server/AsyncQueryForwardingServletTest.java b/services/src/test/java/org/apache/druid/server/AsyncQueryForwardingServletTest.java index f974d361b1f8..2e6e2b3f5935 100644 --- a/services/src/test/java/org/apache/druid/server/AsyncQueryForwardingServletTest.java +++ b/services/src/test/java/org/apache/druid/server/AsyncQueryForwardingServletTest.java @@ -82,18 +82,17 @@ import org.apache.druid.sql.http.SqlQuery; import org.easymock.EasyMock; import org.eclipse.jetty.client.HttpClient; -import org.eclipse.jetty.client.HttpResponse; -import org.eclipse.jetty.client.api.Request; -import org.eclipse.jetty.client.api.Result; +import org.eclipse.jetty.client.Request; +import org.eclipse.jetty.client.Response; +import org.eclipse.jetty.client.Result; +import org.eclipse.jetty.ee8.servlet.DefaultServlet; +import org.eclipse.jetty.ee8.servlet.ServletContextHandler; +import org.eclipse.jetty.ee8.servlet.ServletHolder; import org.eclipse.jetty.http.HttpField; import org.eclipse.jetty.http.HttpFields; +import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.handler.HandlerList; -import org.eclipse.jetty.servlet.DefaultServlet; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHandler; -import org.eclipse.jetty.servlet.ServletHolder; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -119,6 +118,7 @@ import java.util.Optional; import java.util.Properties; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicLong; import java.util.zip.Deflater; @@ -628,12 +628,36 @@ public int read() final AtomicLong didService = new AtomicLong(); final Request proxyRequestMock = Mockito.spy(Request.class); - HttpResponse response = new HttpResponse(proxyRequestMock, ImmutableList.of()) + Response response = new Response() { + @Override + public Request getRequest() + { + return null; + } + + @Override + public HttpVersion getVersion() + { + return null; + } + + @Override + public int getStatus() + { + return 0; + } + + @Override + public String getReason() + { + return ""; + } + @Override public HttpFields getHeaders() { - HttpFields httpFields = new HttpFields(); + HttpFields.Mutable httpFields = HttpFields.build(); if (isJDBCSql) { httpFields.add(new HttpField("X-Druid-SQL-Query-Id", "jdbcDummy")); } else if (isNativeSql) { @@ -641,6 +665,18 @@ public HttpFields getHeaders() } return httpFields; } + + @Override + public HttpFields getTrailers() + { + return null; + } + + @Override + public CompletableFuture abort(Throwable throwable) + { + return null; + } }; final Result result = new Result(proxyRequestMock, response); final StubServiceEmitter stubServiceEmitter = new StubServiceEmitter("", ""); @@ -697,8 +733,8 @@ protected void doService( private static Server makeTestDeleteServer(int port, final CountDownLatch latch) { Server server = new Server(port); - ServletHandler handler = new ServletHandler(); - handler.addServletWithMapping(new ServletHolder(new HttpServlet() + ServletContextHandler servletContextHandler = new ServletContextHandler(); + servletContextHandler.addServlet(new ServletHolder(new HttpServlet() { @Override protected void doDelete(HttpServletRequest req, HttpServletResponse resp) @@ -708,7 +744,7 @@ protected void doDelete(HttpServletRequest req, HttpServletResponse resp) } }), "/default/*"); - server.setHandler(handler); + server.setHandler(servletContextHandler); return server; } @@ -790,15 +826,13 @@ protected String rewriteURI(HttpServletRequest request, String scheme, String ho root.addFilter(GuiceFilter.class, "/default/*", null); root.addFilter(GuiceFilter.class, "/exception/*", null); - final HandlerList handlerList = new HandlerList(); + final Handler.Sequence handlerList = new Handler.Sequence(); handlerList.setHandlers( - new Handler[]{ - JettyServerInitUtils.wrapWithDefaultGzipHandler( - root, - ServerConfig.DEFAULT_GZIP_INFLATE_BUFFER_SIZE, - Deflater.DEFAULT_COMPRESSION - ) - } + JettyServerInitUtils.wrapWithDefaultGzipHandler( + root, + ServerConfig.DEFAULT_GZIP_INFLATE_BUFFER_SIZE, + Deflater.DEFAULT_COMPRESSION + ) ); server.setHandler(handlerList); } diff --git a/sql/pom.xml b/sql/pom.xml index 7f5cceab1dc2..fc088ed456b1 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -86,6 +86,11 @@ org.eclipse.jetty jetty-http + + + javax.servlet + javax.servlet-api + @@ -100,6 +105,19 @@ org.eclipse.jetty jetty-server + + org.eclipse.jetty + jetty-http + + + org.eclipse.jetty + jetty-io + + + org.eclipse.jetty + jetty-util + + joda-time joda-time diff --git a/sql/src/main/java/org/apache/druid/sql/avatica/DruidAvaticaHandler.java b/sql/src/main/java/org/apache/druid/sql/avatica/DruidAvaticaHandler.java new file mode 100644 index 000000000000..867f7734988a --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/avatica/DruidAvaticaHandler.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.avatica; + +import org.apache.calcite.avatica.metrics.MetricsSystem; +import org.apache.calcite.avatica.metrics.Timer; +import org.apache.calcite.avatica.remote.LocalService; +import org.apache.calcite.avatica.remote.MetricsHelper; +import org.apache.calcite.avatica.remote.Service; +import org.apache.calcite.avatica.server.MetricsAwareAvaticaHandler; +import org.apache.calcite.avatica.util.UnsynchronizedBuffer; +import org.eclipse.jetty.server.Handler; + +/** + * Base class for Druid's custom Avatica server handlers that are compatible with Jetty 12. + * + *

This class provides a Jetty 12-compatible implementation of {@link MetricsAwareAvaticaHandler} + * to enable Druid's JDBC support via Apache Calcite's Avatica server. Since Calcite's Avatica server + * does not natively support Jetty 12, this custom implementation allows Druid to continue using + * Avatica for JDBC connectivity during the Jetty 12 migration. + * + *

Concrete implementations handle different wire protocols: + *

    + *
  • {@link DruidAvaticaJsonHandler} - JSON-based protocol
  • + *
  • {@link DruidAvaticaProtobufHandler} - Protocol Buffers-based protocol
  • + *
+ */ +public abstract class DruidAvaticaHandler extends Handler.Abstract implements MetricsAwareAvaticaHandler +{ + protected final Service service; + protected final MetricsSystem metrics; + protected final Timer requestTimer; + protected final ThreadLocal threadLocalBuffer; + + protected DruidAvaticaHandler( + final DruidMeta druidMeta, + final AvaticaMonitor avaticaMonitor, + final Class timerClass + ) + { + this.service = new LocalService(druidMeta); + this.metrics = avaticaMonitor; + this.threadLocalBuffer = ThreadLocal.withInitial(UnsynchronizedBuffer::new); + this.requestTimer = this.metrics.getTimer( + MetricsHelper.concat(timerClass, MetricsAwareAvaticaHandler.REQUEST_TIMER_NAME) + ); + } + + @Override + public MetricsSystem getMetrics() + { + return metrics; + } + + @Override + public void setServerRpcMetadata(Service.RpcMetadataResponse metadata) + { + service.setRpcMetadata(metadata); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/avatica/DruidAvaticaJsonHandler.java b/sql/src/main/java/org/apache/druid/sql/avatica/DruidAvaticaJsonHandler.java index 4f1a5818bf45..f6d0738e2321 100644 --- a/sql/src/main/java/org/apache/druid/sql/avatica/DruidAvaticaJsonHandler.java +++ b/sql/src/main/java/org/apache/druid/sql/avatica/DruidAvaticaJsonHandler.java @@ -19,25 +19,37 @@ package org.apache.druid.sql.avatica; -import com.google.inject.Inject; -import org.apache.calcite.avatica.remote.LocalService; +import org.apache.calcite.avatica.AvaticaUtils; +import org.apache.calcite.avatica.metrics.Timer; +import org.apache.calcite.avatica.remote.JsonHandler; import org.apache.calcite.avatica.remote.Service; import org.apache.calcite.avatica.server.AvaticaJsonHandler; +import org.apache.calcite.avatica.util.UnsynchronizedBuffer; import org.apache.druid.guice.annotations.Self; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.server.DruidNode; +import org.eclipse.jetty.io.Content; import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; +import org.eclipse.jetty.util.Callback; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; +import javax.inject.Inject; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Objects; -public class DruidAvaticaJsonHandler extends AvaticaJsonHandler +public class DruidAvaticaJsonHandler extends DruidAvaticaHandler { + private static final Logger LOG = new Logger(DruidAvaticaJsonHandler.class); + public static final String AVATICA_PATH_NO_TRAILING_SLASH = "/druid/v2/sql/avatica"; public static final String AVATICA_PATH = AVATICA_PATH_NO_TRAILING_SLASH + "/"; + + private final JsonHandler jsonHandler; + @Inject public DruidAvaticaJsonHandler( final DruidMeta druidMeta, @@ -45,20 +57,72 @@ public DruidAvaticaJsonHandler( final AvaticaMonitor avaticaMonitor ) { - super(new LocalService(druidMeta), avaticaMonitor); + super(druidMeta, Objects.requireNonNull(avaticaMonitor), AvaticaJsonHandler.class); + this.jsonHandler = new JsonHandler(service, this.metrics); setServerRpcMetadata(new Service.RpcMetadataResponse(druidNode.getHostAndPortToUse())); } @Override - public void handle( - final String target, - final Request baseRequest, - final HttpServletRequest request, - final HttpServletResponse response - ) throws IOException, ServletException + public boolean handle(Request request, Response response, Callback callback) throws Exception + { + String requestURI = request.getHttpURI().getPath(); + try (Timer.Context ctx = this.requestTimer.start()) { + if (AVATICA_PATH_NO_TRAILING_SLASH.equals(StringUtils.maybeRemoveTrailingSlash(requestURI))) { + response.getHeaders().put("Content-Type", "application/json;charset=utf-8"); + + if (!"POST".equals(request.getMethod())) { + response.setStatus(405); + response.write( + true, + ByteBuffer.wrap("This server expects only POST calls.".getBytes(StandardCharsets.UTF_8)), callback + ); + return true; + } + + String rawRequest = request.getHeaders().get("request"); + if (rawRequest == null) { + // Avoid a new buffer creation for every HTTP request + final UnsynchronizedBuffer buffer = threadLocalBuffer.get(); + try (InputStream inputStream = Content.Source.asInputStream(request)) { + byte[] bytes = AvaticaUtils.readFullyToBytes(inputStream, buffer); + String encoding = request.getHeaders().get("Content-Encoding"); + if (encoding == null) { + encoding = "UTF-8"; + } + rawRequest = AvaticaUtils.newString(bytes, encoding); + } + finally { + // Reset the offset into the buffer after we're done + buffer.reset(); + } + } + final String jsonRequest = rawRequest; + LOG.trace("request: %s", jsonRequest); + + org.apache.calcite.avatica.remote.Handler.HandlerResponse jsonResponse; + try { + jsonResponse = jsonHandler.apply(jsonRequest); + } + catch (Exception e) { + LOG.debug(e, "Error invoking request"); + jsonResponse = jsonHandler.convertToErrorResponse(e); + } + + LOG.trace("response: %s", jsonResponse); + response.setStatus(jsonResponse.getStatusCode()); + response.write(true, ByteBuffer.wrap(jsonResponse.getResponse().getBytes(StandardCharsets.UTF_8)), callback); + return true; + } + } + return false; + } + + @Override + public void setServerRpcMetadata(Service.RpcMetadataResponse metadata) { - if (AVATICA_PATH_NO_TRAILING_SLASH.equals(StringUtils.maybeRemoveTrailingSlash(request.getRequestURI()))) { - super.handle(target, baseRequest, request, response); + super.setServerRpcMetadata(metadata); + if (jsonHandler != null) { + jsonHandler.setRpcMetadata(metadata); } } } diff --git a/sql/src/main/java/org/apache/druid/sql/avatica/DruidAvaticaProtobufHandler.java b/sql/src/main/java/org/apache/druid/sql/avatica/DruidAvaticaProtobufHandler.java index a15efadda6f6..7d5c6183025a 100644 --- a/sql/src/main/java/org/apache/druid/sql/avatica/DruidAvaticaProtobufHandler.java +++ b/sql/src/main/java/org/apache/druid/sql/avatica/DruidAvaticaProtobufHandler.java @@ -19,46 +19,100 @@ package org.apache.druid.sql.avatica; -import com.google.inject.Inject; -import org.apache.calcite.avatica.remote.LocalService; +import org.apache.calcite.avatica.AvaticaUtils; +import org.apache.calcite.avatica.metrics.Timer; +import org.apache.calcite.avatica.remote.ProtobufHandler; +import org.apache.calcite.avatica.remote.ProtobufTranslation; +import org.apache.calcite.avatica.remote.ProtobufTranslationImpl; import org.apache.calcite.avatica.remote.Service; import org.apache.calcite.avatica.server.AvaticaProtobufHandler; +import org.apache.calcite.avatica.util.UnsynchronizedBuffer; import org.apache.druid.guice.annotations.Self; import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.server.DruidNode; +import org.eclipse.jetty.io.Content; import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; +import org.eclipse.jetty.util.Callback; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; +import javax.inject.Inject; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; -public class DruidAvaticaProtobufHandler extends AvaticaProtobufHandler +public class DruidAvaticaProtobufHandler extends DruidAvaticaHandler { + + private static final Logger LOG = new Logger(DruidAvaticaProtobufHandler.class); + public static final String AVATICA_PATH_NO_TRAILING_SLASH = "/druid/v2/sql/avatica-protobuf"; public static final String AVATICA_PATH = AVATICA_PATH_NO_TRAILING_SLASH + "/"; + private final ProtobufHandler protobufHandler; + @Inject public DruidAvaticaProtobufHandler( final DruidMeta druidMeta, @Self final DruidNode druidNode, - final AvaticaMonitor avaticaMonitor + final AvaticaMonitor metrics ) { - super(new LocalService(druidMeta), avaticaMonitor); + super(druidMeta, metrics, AvaticaProtobufHandler.class); + ProtobufTranslation protobufTranslation = new ProtobufTranslationImpl(); + this.protobufHandler = new ProtobufHandler(service, protobufTranslation, this.metrics); setServerRpcMetadata(new Service.RpcMetadataResponse(druidNode.getHostAndPortToUse())); } @Override - public void handle( - final String target, - final Request baseRequest, - final HttpServletRequest request, - final HttpServletResponse response - ) throws IOException, ServletException + public boolean handle(Request request, Response response, Callback callback) throws Exception + { + String requestURI = request.getHttpURI().getPath(); + if (AVATICA_PATH_NO_TRAILING_SLASH.equals(StringUtils.maybeRemoveTrailingSlash(requestURI))) { + try (Timer.Context ctx = this.requestTimer.start()) { + if (!"POST".equals(request.getMethod())) { + response.setStatus(405); + response.write( + true, + ByteBuffer.wrap("This server expects only POST calls.".getBytes(StandardCharsets.UTF_8)), callback + ); + return true; + } + final byte[] requestBytes; + // Avoid a new buffer creation for every HTTP request + final UnsynchronizedBuffer buffer = threadLocalBuffer.get(); + try (InputStream inputStream = Content.Source.asInputStream(request)) { + requestBytes = AvaticaUtils.readFullyToBytes(inputStream, buffer); + } + finally { + buffer.reset(); + } + + response.getHeaders().put("Content-Type", "application/octet-stream;charset=utf-8"); + + org.apache.calcite.avatica.remote.Handler.HandlerResponse handlerResponse; + try { + handlerResponse = protobufHandler.apply(requestBytes); + } + catch (Exception e) { + LOG.debug(e, "Error invoking request"); + handlerResponse = protobufHandler.convertToErrorResponse(e); + } + + response.setStatus(handlerResponse.getStatusCode()); + response.write(true, ByteBuffer.wrap(handlerResponse.getResponse()), callback); + return true; + } + } + return false; + } + + @Override + public void setServerRpcMetadata(Service.RpcMetadataResponse metadata) { - if (AVATICA_PATH_NO_TRAILING_SLASH.equals(StringUtils.maybeRemoveTrailingSlash(request.getRequestURI()))) { - super.handle(target, baseRequest, request, response); + super.setServerRpcMetadata(metadata); + if (protobufHandler != null) { + protobufHandler.setRpcMetadata(metadata); } } } diff --git a/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java b/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java index f832ec80655f..a586358e1262 100644 --- a/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java +++ b/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java @@ -24,7 +24,6 @@ import com.google.inject.Injector; import com.google.inject.Provides; import com.google.inject.name.Named; -import org.apache.calcite.avatica.server.AbstractAvaticaHandler; import org.apache.druid.guice.LazySingleton; import org.apache.druid.initialization.DruidModule; import org.apache.druid.java.util.common.StringUtils; @@ -32,6 +31,7 @@ import org.apache.druid.server.DruidNode; import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.avatica.AvaticaMonitor; +import org.apache.druid.sql.avatica.DruidAvaticaHandler; import org.apache.druid.sql.avatica.DruidAvaticaJsonHandler; import org.apache.druid.sql.avatica.DruidMeta; import org.apache.druid.sql.calcite.SqlTestFrameworkConfig; @@ -185,7 +185,7 @@ public void close() } } - protected AbstractAvaticaHandler getAvaticaHandler(final DruidMeta druidMeta) + protected DruidAvaticaHandler getAvaticaHandler(final DruidMeta druidMeta) { return new DruidAvaticaJsonHandler( druidMeta, diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java index 4ef05973389b..22ea225301ca 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java @@ -39,7 +39,6 @@ import org.apache.calcite.avatica.Meta; import org.apache.calcite.avatica.MissingResultsException; import org.apache.calcite.avatica.NoSuchStatementException; -import org.apache.calcite.avatica.server.AbstractAvaticaHandler; import org.apache.druid.guice.LazySingleton; import org.apache.druid.guice.LifecycleModule; import org.apache.druid.guice.StartupInjectorBuilder; @@ -255,7 +254,7 @@ protected String getJdbcUrlTail() } // Default implementation is for JSON to allow debugging of tests. - protected AbstractAvaticaHandler getAvaticaHandler(final DruidMeta druidMeta) + protected DruidAvaticaHandler getAvaticaHandler(final DruidMeta druidMeta) { return new DruidAvaticaJsonHandler( druidMeta, diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaJsonHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaJsonHandlerTest.java new file mode 100644 index 000000000000..94bb835e0e99 --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaJsonHandlerTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.avatica; + +import org.apache.druid.server.DruidNode; +import org.easymock.EasyMock; +import org.eclipse.jetty.http.HttpURI; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; +import org.eclipse.jetty.util.Callback; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; + +public class DruidAvaticaJsonHandlerTest extends DruidAvaticaHandlerTest +{ + @Override + protected String getJdbcUrlTail() + { + return DruidAvaticaJsonHandler.AVATICA_PATH; + } + + @Override + protected DruidAvaticaHandler getAvaticaHandler(final DruidMeta druidMeta) + { + return new DruidAvaticaJsonHandler( + druidMeta, + new DruidNode("dummy", "dummy", false, 1, null, true, false), + new AvaticaMonitor() + ); + } + + @Test + public void testNonPostRequestReturns405() throws Exception + { + DruidMeta druidMeta = EasyMock.mock(DruidMeta.class); + DruidAvaticaProtobufHandler handler = new DruidAvaticaProtobufHandler( + druidMeta, + new DruidNode("dummy", "dummy", false, 1, null, true, false), + new AvaticaMonitor() + ); + + Request request = EasyMock.mock(Request.class); + Response response = EasyMock.mock(Response.class); + Callback callback = EasyMock.mock(Callback.class); + HttpURI httpURI = EasyMock.mock(HttpURI.class); + + EasyMock.expect(request.getHttpURI()).andReturn(httpURI); + EasyMock.expect(httpURI.getPath()).andReturn(DruidAvaticaProtobufHandler.AVATICA_PATH_NO_TRAILING_SLASH); + EasyMock.expect(request.getMethod()).andReturn("GET"); + + response.setStatus(405); + EasyMock.expectLastCall(); + + response.write( + EasyMock.eq(true), + EasyMock.anyObject(ByteBuffer.class), + EasyMock.eq(callback) + ); + EasyMock.expectLastCall(); + + EasyMock.replay(request, response, callback, httpURI); + + boolean handled = handler.handle(request, response, callback); + + Assertions.assertTrue(handled, "Handler should have handled the request"); + EasyMock.verify(request, response, callback, httpURI); + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaProtobufHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaProtobufHandlerTest.java index bbcb0dba9286..decf79ed2e0a 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaProtobufHandlerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaProtobufHandlerTest.java @@ -19,9 +19,17 @@ package org.apache.druid.sql.avatica; -import org.apache.calcite.avatica.server.AbstractAvaticaHandler; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.server.DruidNode; +import org.easymock.EasyMock; +import org.eclipse.jetty.http.HttpURI; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; +import org.eclipse.jetty.util.Callback; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; public class DruidAvaticaProtobufHandlerTest extends DruidAvaticaHandlerTest { @@ -35,7 +43,7 @@ protected String getJdbcUrlTail() } @Override - protected AbstractAvaticaHandler getAvaticaHandler(final DruidMeta druidMeta) + protected DruidAvaticaProtobufHandler getAvaticaHandler(final DruidMeta druidMeta) { return new DruidAvaticaProtobufHandler( druidMeta, @@ -43,4 +51,41 @@ protected AbstractAvaticaHandler getAvaticaHandler(final DruidMeta druidMeta) new AvaticaMonitor() ); } + + @Test + public void testNonPostRequestReturns405() throws Exception + { + DruidMeta druidMeta = EasyMock.mock(DruidMeta.class); + DruidAvaticaProtobufHandler handler = new DruidAvaticaProtobufHandler( + druidMeta, + new DruidNode("dummy", "dummy", false, 1, null, true, false), + new AvaticaMonitor() + ); + + Request request = EasyMock.mock(Request.class); + Response response = EasyMock.mock(Response.class); + Callback callback = EasyMock.mock(Callback.class); + HttpURI httpURI = EasyMock.mock(HttpURI.class); + + EasyMock.expect(request.getHttpURI()).andReturn(httpURI); + EasyMock.expect(httpURI.getPath()).andReturn(DruidAvaticaProtobufHandler.AVATICA_PATH_NO_TRAILING_SLASH); + EasyMock.expect(request.getMethod()).andReturn("GET"); + + response.setStatus(405); + EasyMock.expectLastCall(); + + response.write( + EasyMock.eq(true), + EasyMock.anyObject(ByteBuffer.class), + EasyMock.eq(callback) + ); + EasyMock.expectLastCall(); + + EasyMock.replay(request, response, callback, httpURI); + + boolean handled = handler.handle(request, response, callback); + + Assertions.assertTrue(handled, "Handler should have handled the request"); + EasyMock.verify(request, response, callback, httpURI); + } } diff --git a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java index 4f0d53dfd027..799cf1cc79f5 100644 --- a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java +++ b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java @@ -437,15 +437,7 @@ public void testCountStarWithMissingIntervalsContext() throws Exception final MockHttpServletResponse response = postForAsyncResponse(sqlQuery, makeRegularUserReq()); - // In tests, MockHttpServletResponse stores headers as a MultiMap. - // This allows the same header key to be set multiple times (e.g., once at the start and once at the end of query processing). - // As a result, we observe duplicate context entries for this test in the expected set. - // This differs from typical behavior for other headers, where a new value would overwrite any previously set value. final Object expectedMissingHeaders = ImmutableList.of( - ImmutableMap.of( - "uncoveredIntervals", "2030-01-01/78149827981274-01-01", - "uncoveredIntervalsOverflowed", "true" - ), ImmutableMap.of( "uncoveredIntervals", "2030-01-01/78149827981274-01-01", "uncoveredIntervalsOverflowed", "true"