diff --git a/distribution/docker/Dockerfile.mysql b/distribution/docker/Dockerfile.mysql index 7a5209674f38..66fb9e90a711 100644 --- a/distribution/docker/Dockerfile.mysql +++ b/distribution/docker/Dockerfile.mysql @@ -22,10 +22,10 @@ FROM $DRUID_RELEASE WORKDIR /opt/druid/extensions/mysql-metadata-storage -ARG MYSQL_URL=https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar -ARG MYSQL_JAR=mysql-connector-java-5.1.49.jar -# https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar.sha1 -ARG MYSQL_SHA=cf76d2e4c9c3782a85c15c87bec5772b34ffd0e5 +ARG MYSQL_URL=https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.2.0/mysql-connector-j-8.2.0.jar +ARG MYSQL_JAR=mysql-connector-j-8.2.0.jar +# https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.2.0/mysql-connector-j-8.2.0.jar.sha1 +ARG MYSQL_SHA=56d34aea30915904b1c883f1cfae731dd2df6029 RUN wget -q ${MYSQL_URL} \ && echo "${MYSQL_SHA} ${MYSQL_JAR}" | sha1sum -c \ diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 03a55b69103a..0f78097ba351 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -615,11 +615,11 @@ You can use the following properties to specify permissible JDBC options for: These properties do not apply to metadata storage connections. -|Property|Possible values|Description|Default| -|--------|---------------|-----------|-------| -|`druid.access.jdbc.enforceAllowedProperties`|Boolean|When true, Druid applies `druid.access.jdbc.allowedProperties` to JDBC connections starting with `jdbc:postgresql:`, `jdbc:mysql:`, or `jdbc:mariadb:`. When false, Druid allows any kind of JDBC connections without JDBC property validation. This config is for backward compatibility especially during upgrades since enforcing allow list can break existing ingestion jobs or lookups based on JDBC. This config is deprecated and will be removed in a future release.|true| -|`druid.access.jdbc.allowedProperties`|List of JDBC properties|Defines a list of allowed JDBC properties. Druid always enforces the list for all JDBC connections starting with `jdbc:postgresql:`, `jdbc:mysql:`, and `jdbc:mariadb:` if `druid.access.jdbc.enforceAllowedProperties` is set to true.

This option is tested against MySQL connector 5.1.49, MariaDB connector 2.7.4, and PostgreSQL connector 42.2.14. Other connector versions might not work.|`["useSSL", "requireSSL", "ssl", "sslmode"]`| -|`druid.access.jdbc.allowUnknownJdbcUrlFormat`|Boolean|When false, Druid only accepts JDBC connections starting with `jdbc:postgresql:` or `jdbc:mysql:`. When true, Druid allows JDBC connections to any kind of database, but only enforces `druid.access.jdbc.allowedProperties` for PostgreSQL and MySQL/MariaDB.|true| +|Property|Possible values| Description |Default| +|--------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------| +|`druid.access.jdbc.enforceAllowedProperties`|Boolean| When true, Druid applies `druid.access.jdbc.allowedProperties` to JDBC connections starting with `jdbc:postgresql:`, `jdbc:mysql:`, or `jdbc:mariadb:`. When false, Druid allows any kind of JDBC connections without JDBC property validation. This config is for backward compatibility especially during upgrades since enforcing allow list can break existing ingestion jobs or lookups based on JDBC. This config is deprecated and will be removed in a future release. |true| +|`druid.access.jdbc.allowedProperties`|List of JDBC properties| Defines a list of allowed JDBC properties. Druid always enforces the list for all JDBC connections starting with `jdbc:postgresql:`, `jdbc:mysql:`, and `jdbc:mariadb:` if `druid.access.jdbc.enforceAllowedProperties` is set to true.

This option is tested against MySQL connector 8.2.0, MariaDB connector 2.7.4, and PostgreSQL connector 42.2.14. Other connector versions might not work. |`["useSSL", "requireSSL", "ssl", "sslmode"]`| +|`druid.access.jdbc.allowUnknownJdbcUrlFormat`|Boolean| When false, Druid only accepts JDBC connections starting with `jdbc:postgresql:` or `jdbc:mysql:`. When true, Druid allows JDBC connections to any kind of database, but only enforces `druid.access.jdbc.allowedProperties` for PostgreSQL and MySQL/MariaDB. |true| ### Task logging diff --git a/docs/development/extensions-core/mysql.md b/docs/development/extensions-core/mysql.md index 1a0d3d04a83f..bc6012dbb5a3 100644 --- a/docs/development/extensions-core/mysql.md +++ b/docs/development/extensions-core/mysql.md @@ -36,10 +36,10 @@ This extension can use Oracle's MySQL JDBC driver which is not included in the D install it separately. There are a few ways to obtain this library: - It can be downloaded from the MySQL site at: https://dev.mysql.com/downloads/connector/j/ -- It can be fetched from Maven Central at: https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar +- It can be fetched from Maven Central at: https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.2.0/mysql-connector-j-8.2.0.jar - It may be available through your package manager, e.g. as `libmysql-java` on APT for a Debian-based OS -This fetches the MySQL connector JAR file with a name like `mysql-connector-java-5.1.49.jar`. +This fetches the MySQL connector JAR file with a name like `mysql-connector-j-8.2.0.jar`. Copy or symlink this file inside the folder `extensions/mysql-metadata-storage` under the distribution root directory. diff --git a/docs/operations/use_sbt_to_build_fat_jar.md b/docs/operations/use_sbt_to_build_fat_jar.md index 055ddd236c40..eeb58c9734eb 100644 --- a/docs/operations/use_sbt_to_build_fat_jar.md +++ b/docs/operations/use_sbt_to_build_fat_jar.md @@ -102,7 +102,7 @@ libraryDependencies ++= Seq( "com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-smile-provider" % "2.3.0", "com.fasterxml.jackson.module" % "jackson-module-jaxb-annotations" % "2.3.0", "com.sun.jersey" % "jersey-servlet" % "1.17.1", - "mysql" % "mysql-connector-java" % "5.1.34", + "mysql" % "mysql-connector-java" % "8.2.0", "org.scalatest" %% "scalatest" % "2.2.3" % "test", "org.mockito" % "mockito-core" % "1.10.19" % "test" ) diff --git a/extensions-core/lookups-cached-global/pom.xml b/extensions-core/lookups-cached-global/pom.xml index 24fb514abf31..4c91106458ed 100644 --- a/extensions-core/lookups-cached-global/pom.xml +++ b/extensions-core/lookups-cached-global/pom.xml @@ -158,8 +158,8 @@ test - mysql - mysql-connector-java + com.mysql + mysql-connector-j ${mysql.version} test diff --git a/extensions-core/lookups-cached-global/src/test/java/org/apache/druid/query/lookup/namespace/JdbcExtractionNamespaceUrlCheckTest.java b/extensions-core/lookups-cached-global/src/test/java/org/apache/druid/query/lookup/namespace/JdbcExtractionNamespaceUrlCheckTest.java index d2ba30b49464..2b49877c9975 100644 --- a/extensions-core/lookups-cached-global/src/test/java/org/apache/druid/query/lookup/namespace/JdbcExtractionNamespaceUrlCheckTest.java +++ b/extensions-core/lookups-cached-global/src/test/java/org/apache/druid/query/lookup/namespace/JdbcExtractionNamespaceUrlCheckTest.java @@ -156,46 +156,6 @@ public boolean isEnforceAllowedProperties() } ); } - - @Test - public void testWhenInvalidUrlFormat() - { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Invalid URL format for MySQL: [jdbc:mysql:/invalid-url::3006]"); - new JdbcExtractionNamespace( - new MetadataStorageConnectorConfig() - { - @Override - public String getConnectURI() - { - return "jdbc:mysql:/invalid-url::3006"; - } - }, - TABLE_NAME, - KEY_NAME, - VAL_NAME, - TS_COLUMN, - "some filter", - new Period(10), - null, - 0, - null, - new JdbcAccessSecurityConfig() - { - @Override - public Set getAllowedProperties() - { - return ImmutableSet.of("valid_key1", "valid_key2"); - } - - @Override - public boolean isEnforceAllowedProperties() - { - return true; - } - } - ); - } } public static class PostgreSqlTest diff --git a/extensions-core/lookups-cached-single/pom.xml b/extensions-core/lookups-cached-single/pom.xml index ecec17198d6c..f99cb49237d1 100644 --- a/extensions-core/lookups-cached-single/pom.xml +++ b/extensions-core/lookups-cached-single/pom.xml @@ -117,8 +117,8 @@ test - mysql - mysql-connector-java + com.mysql + mysql-connector-j ${mysql.version} test diff --git a/extensions-core/lookups-cached-single/src/test/java/org/apache/druid/server/lookup/jdbc/JdbcDataFetcherUrlCheckTest.java b/extensions-core/lookups-cached-single/src/test/java/org/apache/druid/server/lookup/jdbc/JdbcDataFetcherUrlCheckTest.java index b38875b51892..c2279438626e 100644 --- a/extensions-core/lookups-cached-single/src/test/java/org/apache/druid/server/lookup/jdbc/JdbcDataFetcherUrlCheckTest.java +++ b/extensions-core/lookups-cached-single/src/test/java/org/apache/druid/server/lookup/jdbc/JdbcDataFetcherUrlCheckTest.java @@ -139,41 +139,6 @@ public boolean isEnforceAllowedProperties() } ); } - - @Test - public void testWhenInvalidUrlFormat() - { - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Invalid URL format for MySQL: [jdbc:mysql:/invalid-url::3006]"); - new JdbcDataFetcher( - new MetadataStorageConnectorConfig() - { - @Override - public String getConnectURI() - { - return "jdbc:mysql:/invalid-url::3006"; - } - }, - TABLE_NAME, - KEY_COLUMN, - VALUE_COLUMN, - 100, - new JdbcAccessSecurityConfig() - { - @Override - public Set getAllowedProperties() - { - return ImmutableSet.of("valid_key1", "valid_key2"); - } - - @Override - public boolean isEnforceAllowedProperties() - { - return true; - } - } - ); - } } public static class PostgreSqlTest diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index 84f0483ad3bc..46d2880bf219 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -48,8 +48,8 @@ provided - mysql - mysql-connector-java + com.mysql + mysql-connector-j ${mysql.version} provided diff --git a/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java index 91abb44380fa..2ce656a87914 100644 --- a/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java +++ b/extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java @@ -47,7 +47,7 @@ public class MySQLConnector extends SQLMetadataConnector private static final String QUOTE_STRING = "`"; private static final String COLLATION = "CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"; private static final String MYSQL_TRANSIENT_EXCEPTION_CLASS_NAME - = "com.mysql.jdbc.exceptions.MySQLTransientException"; + = "java.sql.SQLTransientException"; private static final String MARIA_DB_PACKET_EXCEPTION_CLASS_NAME = "org.mariadb.jdbc.internal.util.exceptions.MaxAllowedPacketException"; private static final String MYSQL_PACKET_EXCEPTION_CLASS_NAME @@ -279,7 +279,7 @@ private Class tryLoadDriverClass(String className, boolean failIfNotFound) if (failIfNotFound) { throw new ISE(e, "Could not find %s on the classpath. The MySQL Connector library is not included in the Druid " + "distribution but is required to use MySQL. Please download a compatible library (for example " - + "'mysql-connector-java-5.1.49.jar') and place it under 'extensions/mysql-metadata-storage/'. See " + + "'mysql-connector-j-8.2.0.jar') and place it under 'extensions/mysql-metadata-storage/'. See " + "https://druid.apache.org/downloads for more details.", className ); diff --git a/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/firehose/sql/MySQLFirehoseDatabaseConnectorTest.java b/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/firehose/sql/MySQLFirehoseDatabaseConnectorTest.java index 035aec7d6aa5..43d96961c1ba 100644 --- a/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/firehose/sql/MySQLFirehoseDatabaseConnectorTest.java +++ b/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/firehose/sql/MySQLFirehoseDatabaseConnectorTest.java @@ -25,7 +25,6 @@ import com.google.common.collect.ImmutableSet; import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.metadata.MetadataStorageConnectorConfig; import org.apache.druid.metadata.storage.mysql.MySQLConnectorDriverConfig; import org.apache.druid.metadata.storage.mysql.MySQLMetadataStorageModule; @@ -338,29 +337,6 @@ public boolean isEnforceAllowedProperties() ); } - @Test - public void testFindPropertyKeysFromInvalidConnectUrl() - { - final String url = "jdbc:mysql:/invalid-url::3006"; - MetadataStorageConnectorConfig connectorConfig = new MetadataStorageConnectorConfig() - { - @Override - public String getConnectURI() - { - return url; - } - }; - - expectedException.expect(RuntimeException.class); - expectedException.expectMessage(StringUtils.format("Invalid URL format for MySQL: [%s]", url)); - new MySQLFirehoseDatabaseConnector( - connectorConfig, - null, - new JdbcAccessSecurityConfig(), - mySQLConnectorDriverConfig - ); - } - private static JdbcAccessSecurityConfig newSecurityConfigEnforcingAllowList(Set allowedProperties) { return new JdbcAccessSecurityConfig() diff --git a/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLConnectorTest.java b/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLConnectorTest.java index b60168d5f429..4c0db1574436 100644 --- a/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLConnectorTest.java +++ b/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLConnectorTest.java @@ -20,8 +20,6 @@ package org.apache.druid.metadata.storage.mysql; import com.google.common.base.Supplier; -import com.mysql.jdbc.exceptions.MySQLTransactionRollbackException; -import com.mysql.jdbc.exceptions.MySQLTransientException; import org.apache.druid.metadata.MetadataStorageConnectorConfig; import org.apache.druid.metadata.MetadataStorageTablesConfig; import org.junit.Assert; @@ -57,16 +55,13 @@ public void testIsExceptionTransientMySql() new MySQLConnectorSslConfig(), MYSQL_DRIVER_CONFIG ); - Assert.assertTrue(connector.connectorIsTransientException(new MySQLTransientException())); - Assert.assertTrue(connector.connectorIsTransientException(new MySQLTransactionRollbackException())); Assert.assertTrue( connector.connectorIsTransientException(new SQLException("some transient failure", "s0", 1317)) ); Assert.assertFalse( connector.connectorIsTransientException(new SQLException("totally realistic test data", "s0", 1337)) ); - // this method does not specially handle normal transient exceptions either, since it is not vendor specific - Assert.assertFalse( + Assert.assertTrue( connector.connectorIsTransientException(new SQLTransientConnectionException("transient")) ); } @@ -81,7 +76,7 @@ public void testIsExceptionTransientNoMySqlClazz() MARIADB_DRIVER_CONFIG ); // no vendor specific for MariaDb, so should always be false - Assert.assertFalse(connector.connectorIsTransientException(new MySQLTransientException())); + Assert.assertFalse(connector.connectorIsTransientException(new SQLTransientException())); Assert.assertFalse( connector.connectorIsTransientException(new SQLException("some transient failure", "s0", 1317)) ); @@ -116,7 +111,7 @@ public void testIsRootCausePacketTooBigException() connector.isRootCausePacketTooBigException(new SQLTransientException()) ); Assert.assertFalse( - connector.isRootCausePacketTooBigException(new MySQLTransientException()) + connector.isRootCausePacketTooBigException(new SQLTransientException()) ); } diff --git a/integration-tests-ex/cases/pom.xml b/integration-tests-ex/cases/pom.xml index eb4822c425df..e3c1fce920b4 100644 --- a/integration-tests-ex/cases/pom.xml +++ b/integration-tests-ex/cases/pom.xml @@ -182,8 +182,8 @@ - mysql - mysql-connector-java + com.mysql + mysql-connector-j ${mysql.version} runtime @@ -340,7 +340,7 @@ - mysql:mysql-connector-java:jar + com.mysql:mysql-connector-j:jar diff --git a/integration-tests-ex/image/docker/Dockerfile b/integration-tests-ex/image/docker/Dockerfile index 962562d6676e..a77a5c2d023e 100644 --- a/integration-tests-ex/image/docker/Dockerfile +++ b/integration-tests-ex/image/docker/Dockerfile @@ -54,7 +54,7 @@ ENV DRUID_HOME=/usr/local/druid COPY apache-druid-${DRUID_VERSION}-bin.tar.gz /usr/local/ COPY druid-it-tools-${DRUID_VERSION}.jar /tmp/druid/extensions/druid-it-tools/ COPY kafka-protobuf-provider-${CONFLUENT_VERSION}.jar /tmp/druid/lib/ -COPY mysql-connector-java-${MYSQL_VERSION}.jar /tmp/druid/lib/ +COPY mysql-connector-j-${MYSQL_VERSION}.jar /tmp/druid/lib/ COPY mariadb-java-client-${MARIADB_VERSION}.jar /tmp/druid/lib/ COPY test-setup.sh / COPY druid.sh / diff --git a/integration-tests-ex/image/pom.xml b/integration-tests-ex/image/pom.xml index 07e25393e673..62eb6ca2ee5a 100644 --- a/integration-tests-ex/image/pom.xml +++ b/integration-tests-ex/image/pom.xml @@ -102,8 +102,8 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven - mysql - mysql-connector-java + com.mysql + mysql-connector-j ${mysql.version} @@ -131,8 +131,8 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven - mysql - mysql-connector-java + com.mysql + mysql-connector-j ${mysql.version} jar true diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile index 85eb624cbb5c..40b1cee4f9bf 100644 --- a/integration-tests/docker/Dockerfile +++ b/integration-tests/docker/Dockerfile @@ -53,11 +53,11 @@ COPY extensions/ /usr/local/druid/extensions/ # target path must match the exact path referenced in environment-configs/common # alternatively: Download the MariaDB Java connector, and pretend it is the mysql connector RUN if [ "$MYSQL_DRIVER_CLASSNAME" = "com.mysql.jdbc.Driver" ] ; \ - then wget -q "https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_VERSION/mysql-connector-java-$MYSQL_VERSION.jar" \ - -O /usr/local/druid/lib/mysql-connector-java.jar; \ + then wget -q "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/$MYSQL_VERSION/mysql-connector-j-$MYSQL_VERSION.jar" \ + -O /usr/local/druid/lib/mysql-connector-j.jar; \ elif [ "$MYSQL_DRIVER_CLASSNAME" = "org.mariadb.jdbc.Driver" ] ; \ then wget -q "https://repo1.maven.org/maven2/org/mariadb/jdbc/mariadb-java-client/$MARIA_VERSION/mariadb-java-client-$MARIA_VERSION.jar" \ - -O /usr/local/druid/lib/mysql-connector-java.jar; \ + -O /usr/local/druid/lib/mysql-connector-j.jar; \ fi # download kafka protobuf provider diff --git a/integration-tests/docker/environment-configs/common b/integration-tests/docker/environment-configs/common index e4bc11b7ce44..3ce06d90ea26 100644 --- a/integration-tests/docker/environment-configs/common +++ b/integration-tests/docker/environment-configs/common @@ -24,7 +24,7 @@ LC_ALL=C.UTF-8 # JAVA OPTS COMMON_DRUID_JAVA_OPTS=-Duser.timezone=UTC -Dfile.encoding=UTF-8 -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:OnOutOfMemoryError='chmod 644 /shared/logs/*.hprof' -XX:HeapDumpPath=/shared/logs DRUID_DEP_BIN_DIR=/shared/docker/bin -DRUID_DEP_LIB_DIR=/shared/hadoop_xml:/shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar +DRUID_DEP_LIB_DIR=/shared/hadoop_xml:/shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-j.jar # Druid configs # If you are making a change in load list below, make the necessary changes in github actions too diff --git a/integration-tests/docker/environment-configs/common-ldap b/integration-tests/docker/environment-configs/common-ldap index 021c9192927d..fae9b18c341a 100644 --- a/integration-tests/docker/environment-configs/common-ldap +++ b/integration-tests/docker/environment-configs/common-ldap @@ -25,7 +25,7 @@ AWS_REGION=us-east-1 # JAVA OPTS COMMON_DRUID_JAVA_OPTS=-Duser.timezone=UTC -Dfile.encoding=UTF-8 -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:OnOutOfMemoryError='chmod 644 /shared/logs/*.hprof' -XX:HeapDumpPath=/shared/logs DRUID_DEP_BIN_DIR=/shared/docker/bin -DRUID_DEP_LIB_DIR=/shared/hadoop_xml:/shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar +DRUID_DEP_LIB_DIR=/shared/hadoop_xml:/shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-j.jar # Druid configs # If you are making a change in load list below, make the necessary changes in github actions too diff --git a/pom.xml b/pom.xml index 6cf5534b9989..1645f2925a92 100644 --- a/pom.xml +++ b/pom.xml @@ -104,7 +104,7 @@ 2.12.7.20221012 1.9.13 2.22.1 - 5.1.49 + 8.2.0 2.7.3 3.10.6.Final 4.1.107.Final diff --git a/processing/pom.xml b/processing/pom.xml index affd900fe6ca..d1bd2b080c23 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -449,8 +449,8 @@ test - mysql - mysql-connector-java + com.mysql + mysql-connector-j ${mysql.version} test diff --git a/processing/src/main/java/org/apache/druid/utils/ConnectionUriUtils.java b/processing/src/main/java/org/apache/druid/utils/ConnectionUriUtils.java index bc3700ead0bc..80c296ef6539 100644 --- a/processing/src/main/java/org/apache/druid/utils/ConnectionUriUtils.java +++ b/processing/src/main/java/org/apache/druid/utils/ConnectionUriUtils.java @@ -28,6 +28,8 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.Set; @@ -43,7 +45,8 @@ public final class ConnectionUriUtils public static final String MARIADB_PREFIX = "jdbc:mariadb:"; public static final String POSTGRES_DRIVER = "org.postgresql.Driver"; - public static final String MYSQL_NON_REGISTERING_DRIVER = "com.mysql.jdbc.NonRegisteringDriver"; + public static final String MYSQL_CONNECTION_URL = "com.mysql.cj.conf.ConnectionUrl"; + public static final String MYSQL_HOST_INFO = "com.mysql.cj.conf.HostInfo"; /** * This method checks {@param actualProperties} against {@param allowedProperties} if they are not system properties. @@ -99,7 +102,7 @@ public static Set tryParseJdbcUriParameters(String connectionUri, boolea } catch (ClassNotFoundException notFoundMaria2x) { throw new RuntimeException( - "Failed to find MySQL driver class. Please check the MySQL connector version 5.1.49 is in the classpath", + "Failed to find MySQL driver class. Please check the MySQL connector version 8.2.0 is in the classpath", notFoundMysql ); } @@ -177,21 +180,66 @@ public static Set tryParsePostgresConnectionUri(String connectionUri) } public static Set tryParseMySqlConnectionUri(String connectionUri) - throws ClassNotFoundException, NoSuchMethodException, InstantiationException, IllegalAccessException, + throws ClassNotFoundException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { - Class driverClass = Class.forName(MYSQL_NON_REGISTERING_DRIVER); - Method parseUrl = driverClass.getMethod("parseURL", String.class, Properties.class); - // almost the same as postgres, but is an instance level method - Properties properties = (Properties) parseUrl.invoke( - driverClass.getConstructor().newInstance(), - connectionUri, - null - ); - - if (properties == null) { + Class connectionUrlClass = Class.forName(MYSQL_CONNECTION_URL); + Method isConnectionStringSupported = connectionUrlClass.getMethod("acceptsUrl", String.class); + if (!(boolean) isConnectionStringSupported.invoke(connectionUrlClass, connectionUri)) { throw new IAE("Invalid URL format for MySQL: [%s]", connectionUri); } + Method getConnectionUrlInstanceMethod = connectionUrlClass.getMethod("getConnectionUrlInstance", String.class, Properties.class); + Object conUrl = getConnectionUrlInstanceMethod.invoke(connectionUrlClass, connectionUri, null); + Method getHostsList = connectionUrlClass.getMethod("getHostsList"); + return getKeysFromOptions(getPropertiesFromHosts((List) getHostsList.invoke(conUrl))); + } + + private static Properties getPropertiesFromHosts(List hostsList) + throws NoSuchMethodException, InvocationTargetException, IllegalAccessException, ClassNotFoundException + { + Properties properties = new Properties(); + Class hostInfoClass = Class.forName(MYSQL_HOST_INFO); + for (Object host : hostsList) { + Method getHostMethod = hostInfoClass.getMethod("getHost"); + String hostName = (String) getHostMethod.invoke(host); + if (hostName != null) { + properties.setProperty("HOST", hostName); + } + + Method getPortMethod = hostInfoClass.getMethod("getPort"); + Integer port = (Integer) getPortMethod.invoke(host); + if (port != null) { + properties.setProperty("PORT", String.valueOf(port)); + } + + Method getUserMethod = hostInfoClass.getMethod("getUser"); + String user = (String) getUserMethod.invoke(host); + if (user != null) { + properties.setProperty("user", user); + } + + Method getPasswordMethod = hostInfoClass.getMethod("getPassword"); + String password = (String) getPasswordMethod.invoke(host); + if (password != null) { + properties.setProperty("password", password); + } + + Method getHostPropertiesMethod = hostInfoClass.getMethod("getHostProperties"); + Map hostProperties = + (Map) getHostPropertiesMethod.invoke(host); + if (hostProperties != null) { + for (Map.Entry entry : hostProperties.entrySet()) { + if (entry.getKey() != null && entry.getValue() != null) { + properties.setProperty(entry.getKey(), entry.getValue()); + } + } + } + } + return properties; + } + + private static Set getKeysFromOptions(Properties properties) + { Set keys = Sets.newHashSetWithExpectedSize(properties.size()); properties.forEach((k, v) -> keys.add((String) k)); return keys; diff --git a/processing/src/test/java/org/apache/druid/utils/ConnectionUriUtilsTest.java b/processing/src/test/java/org/apache/druid/utils/ConnectionUriUtilsTest.java index 754eb05b3055..bb7831573098 100644 --- a/processing/src/test/java/org/apache/druid/utils/ConnectionUriUtilsTest.java +++ b/processing/src/test/java/org/apache/druid/utils/ConnectionUriUtilsTest.java @@ -103,7 +103,7 @@ public void testTryParses() props = ConnectionUriUtils.tryParseJdbcUriParameters(MYSQL_URI, false); // though this would be 4 if mysql wasn't loaded in classpath because it would fall back to mariadb - Assert.assertEquals(9, props.size()); + Assert.assertEquals(6, props.size()); props = ConnectionUriUtils.tryParseJdbcUriParameters(MARIA_URI, false); Assert.assertEquals(4, props.size()); @@ -126,13 +126,6 @@ public void tryParseInvalidPostgres() ConnectionUriUtils.tryParseJdbcUriParameters("jdbc:postgresql://bad:1234¶m", true); } - @Test - public void tryParseInvalidMySql() - { - expectedException.expect(IAE.class); - ConnectionUriUtils.tryParseJdbcUriParameters("jdbc:mysql:/bad", true); - } - @Test public void testMySqlFallbackMySqlMaria2x() { @@ -202,7 +195,7 @@ public void testMySQLDriver() throws Exception Set props = ConnectionUriUtils.tryParseMySqlConnectionUri(MYSQL_URI); // mysql actually misses 'keyonly', but spits out several keys that are not actually uri parameters // DBNAME, HOST, PORT, HOST.1, PORT.1, NUM_HOSTS - Assert.assertEquals(9, props.size()); + Assert.assertEquals(6, props.size()); Assert.assertTrue(props.contains("user")); Assert.assertTrue(props.contains("password")); Assert.assertTrue(props.contains("otherOptions")); diff --git a/server/src/main/java/org/apache/druid/server/initialization/JdbcAccessSecurityConfig.java b/server/src/main/java/org/apache/druid/server/initialization/JdbcAccessSecurityConfig.java index f2eda2cf34a4..42b0044b1d01 100644 --- a/server/src/main/java/org/apache/druid/server/initialization/JdbcAccessSecurityConfig.java +++ b/server/src/main/java/org/apache/druid/server/initialization/JdbcAccessSecurityConfig.java @@ -55,7 +55,7 @@ public class JdbcAccessSecurityConfig "HOST", "PORT", "NUM_HOSTS", - "DBNAME", + "dbname", // PostgreSQL "PGHOST",