upgrade mysql:mysql-connector-java to 8.2.0

distribution/docker/Dockerfile.mysql

@@ -22,10 +22,10 @@ FROM $DRUID_RELEASE
 
 WORKDIR /opt/druid/extensions/mysql-metadata-storage
 
-ARG MYSQL_URL=https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar
-ARG MYSQL_JAR=mysql-connector-java-5.1.49.jar
-# https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar.sha1
-ARG MYSQL_SHA=cf76d2e4c9c3782a85c15c87bec5772b34ffd0e5
+ARG MYSQL_URL=https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.2.0/mysql-connector-j-8.2.0.jar
+ARG MYSQL_JAR=mysql-connector-j-8.2.0.jar
+# https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.2.0/mysql-connector-j-8.2.0.jar.sha1
+ARG MYSQL_SHA=56d34aea30915904b1c883f1cfae731dd2df6029
 
 RUN wget -q ${MYSQL_URL} \
  && echo "${MYSQL_SHA}  ${MYSQL_JAR}" | sha1sum -c \

docs/configuration/index.md

@@ -615,11 +615,11 @@ You can use the following properties to specify permissible JDBC options for:
 
 These properties do not apply to metadata storage connections.
 
-|Property|Possible values|Description|Default|
-|--------|---------------|-----------|-------|
-|`druid.access.jdbc.enforceAllowedProperties`|Boolean|When true, Druid applies `druid.access.jdbc.allowedProperties` to JDBC connections starting with `jdbc:postgresql:`, `jdbc:mysql:`, or `jdbc:mariadb:`. When false, Druid allows any kind of JDBC connections without JDBC property validation. This config is for backward compatibility especially during upgrades since enforcing allow list can break existing ingestion jobs or lookups based on JDBC. This config is deprecated and will be removed in a future release.|true|
-|`druid.access.jdbc.allowedProperties`|List of JDBC properties|Defines a list of allowed JDBC properties. Druid always enforces the list for all JDBC connections starting with `jdbc:postgresql:`, `jdbc:mysql:`, and `jdbc:mariadb:` if `druid.access.jdbc.enforceAllowedProperties` is set to true.<br/><br/>This option is tested against MySQL connector 5.1.49, MariaDB connector 2.7.4, and PostgreSQL connector 42.2.14. Other connector versions might not work.|`["useSSL", "requireSSL", "ssl", "sslmode"]`|
-|`druid.access.jdbc.allowUnknownJdbcUrlFormat`|Boolean|When false, Druid only accepts JDBC connections starting with `jdbc:postgresql:` or `jdbc:mysql:`. When true, Druid allows JDBC connections to any kind of database, but only enforces `druid.access.jdbc.allowedProperties` for PostgreSQL and MySQL/MariaDB.|true|
+|Property|Possible values| Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |Default|
+|--------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------|
+|`druid.access.jdbc.enforceAllowedProperties`|Boolean| When true, Druid applies `druid.access.jdbc.allowedProperties` to JDBC connections starting with `jdbc:postgresql:`, `jdbc:mysql:`, or `jdbc:mariadb:`. When false, Druid allows any kind of JDBC connections without JDBC property validation. This config is for backward compatibility especially during upgrades since enforcing allow list can break existing ingestion jobs or lookups based on JDBC. This config is deprecated and will be removed in a future release. |true|
+|`druid.access.jdbc.allowedProperties`|List of JDBC properties| Defines a list of allowed JDBC properties. Druid always enforces the list for all JDBC connections starting with `jdbc:postgresql:`, `jdbc:mysql:`, and `jdbc:mariadb:` if `druid.access.jdbc.enforceAllowedProperties` is set to true.<br/><br/>This option is tested against MySQL connector 8.2.0, MariaDB connector 2.7.4, and PostgreSQL connector 42.2.14. Other connector versions might not work.                                                                           |`["useSSL", "requireSSL", "ssl", "sslmode"]`|
+|`druid.access.jdbc.allowUnknownJdbcUrlFormat`|Boolean| When false, Druid only accepts JDBC connections starting with `jdbc:postgresql:` or `jdbc:mysql:`. When true, Druid allows JDBC connections to any kind of database, but only enforces `druid.access.jdbc.allowedProperties` for PostgreSQL and MySQL/MariaDB.                                                                                                                                                                                                                 |true|
 
 ### Task logging
 

docs/development/extensions-core/mysql.md

@@ -36,10 +36,10 @@ This extension can use Oracle's MySQL JDBC driver which is not included in the D
 install it separately. There are a few ways to obtain this library:
 
 - It can be downloaded from the MySQL site at: https://dev.mysql.com/downloads/connector/j/
-- It can be fetched from Maven Central at: https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar
+- It can be fetched from Maven Central at: https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.2.0/mysql-connector-j-8.2.0.jar
 - It may be available through your package manager, e.g. as `libmysql-java` on APT for a Debian-based OS
 
-This fetches the MySQL connector JAR file with a name like `mysql-connector-java-5.1.49.jar`.
+This fetches the MySQL connector JAR file with a name like `mysql-connector-j-8.2.0.jar`.
 
 Copy or symlink this file inside the folder `extensions/mysql-metadata-storage` under the distribution root directory.
 

docs/operations/use_sbt_to_build_fat_jar.md

@@ -102,7 +102,7 @@ libraryDependencies ++= Seq(
   "com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-smile-provider" % "2.3.0",
   "com.fasterxml.jackson.module" % "jackson-module-jaxb-annotations" % "2.3.0",
   "com.sun.jersey" % "jersey-servlet" % "1.17.1",
-  "mysql" % "mysql-connector-java" % "5.1.34",
+  "mysql" % "mysql-connector-java" % "8.2.0",
   "org.scalatest" %% "scalatest" % "2.2.3" % "test",
   "org.mockito" % "mockito-core" % "1.10.19" % "test"
 )

extensions-core/lookups-cached-global/pom.xml

@@ -158,8 +158,8 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>mysql</groupId>
-      <artifactId>mysql-connector-java</artifactId>
+      <groupId>com.mysql</groupId>
+      <artifactId>mysql-connector-j</artifactId>
       <version>${mysql.version}</version>
       <scope>test</scope>
     </dependency>

extensions-core/lookups-cached-global/src/test/java/org/apache/druid/query/lookup/namespace/JdbcExtractionNamespaceUrlCheckTest.java

@@ -156,46 +156,6 @@ public boolean isEnforceAllowedProperties()
           }
       );
     }
-
-    @Test
-    public void testWhenInvalidUrlFormat()
-    {
-      expectedException.expect(IllegalArgumentException.class);
-      expectedException.expectMessage("Invalid URL format for MySQL: [jdbc:mysql:/invalid-url::3006]");
-      new JdbcExtractionNamespace(
-          new MetadataStorageConnectorConfig()
-          {
-            @Override
-            public String getConnectURI()
-            {
-              return "jdbc:mysql:/invalid-url::3006";
-            }
-          },
-          TABLE_NAME,
-          KEY_NAME,
-          VAL_NAME,
-          TS_COLUMN,
-          "some filter",
-          new Period(10),
-          null,
-          0,
-          null,
-          new JdbcAccessSecurityConfig()
-          {
-            @Override
-            public Set<String> getAllowedProperties()
-            {
-              return ImmutableSet.of("valid_key1", "valid_key2");
-            }
-
-            @Override
-            public boolean isEnforceAllowedProperties()
-            {
-              return true;
-            }
-          }
-      );
-    }
   }
 
   public static class PostgreSqlTest

extensions-core/lookups-cached-single/pom.xml

@@ -117,8 +117,8 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>mysql</groupId>
-      <artifactId>mysql-connector-java</artifactId>
+      <groupId>com.mysql</groupId>
+      <artifactId>mysql-connector-j</artifactId>
       <version>${mysql.version}</version>
       <scope>test</scope>
     </dependency>

extensions-core/lookups-cached-single/src/test/java/org/apache/druid/server/lookup/jdbc/JdbcDataFetcherUrlCheckTest.java

@@ -139,41 +139,6 @@ public boolean isEnforceAllowedProperties()
           }
       );
     }
-
-    @Test
-    public void testWhenInvalidUrlFormat()
-    {
-      expectedException.expect(IllegalArgumentException.class);
-      expectedException.expectMessage("Invalid URL format for MySQL: [jdbc:mysql:/invalid-url::3006]");
-      new JdbcDataFetcher(
-          new MetadataStorageConnectorConfig()
-          {
-            @Override
-            public String getConnectURI()
-            {
-              return "jdbc:mysql:/invalid-url::3006";
-            }
-          },
-          TABLE_NAME,
-          KEY_COLUMN,
-          VALUE_COLUMN,
-          100,
-          new JdbcAccessSecurityConfig()
-          {
-            @Override
-            public Set<String> getAllowedProperties()
-            {
-              return ImmutableSet.of("valid_key1", "valid_key2");
-            }
-
-            @Override
-            public boolean isEnforceAllowedProperties()
-            {
-              return true;
-            }
-          }
-      );
-    }
   }
 
   public static class PostgreSqlTest

extensions-core/mysql-metadata-storage/pom.xml

@@ -48,8 +48,8 @@
             <scope>provided</scope>
         </dependency>
         <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
+            <groupId>com.mysql</groupId>
+            <artifactId>mysql-connector-j</artifactId>
             <version>${mysql.version}</version>
             <scope>provided</scope>
         </dependency>

extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLConnector.java

@@ -47,7 +47,7 @@ public class MySQLConnector extends SQLMetadataConnector
   private static final String QUOTE_STRING = "`";
   private static final String COLLATION = "CHARACTER SET utf8mb4 COLLATE utf8mb4_bin";
   private static final String MYSQL_TRANSIENT_EXCEPTION_CLASS_NAME
-      = "com.mysql.jdbc.exceptions.MySQLTransientException";
+      = "java.sql.SQLTransientException";
   private static final String MARIA_DB_PACKET_EXCEPTION_CLASS_NAME
       = "org.mariadb.jdbc.internal.util.exceptions.MaxAllowedPacketException";
   private static final String MYSQL_PACKET_EXCEPTION_CLASS_NAME
@@ -279,7 +279,7 @@ private Class<?> tryLoadDriverClass(String className, boolean failIfNotFound)
       if (failIfNotFound) {
         throw new ISE(e, "Could not find %s on the classpath. The MySQL Connector library is not included in the Druid "
                          + "distribution but is required to use MySQL. Please download a compatible library (for example "
-                         + "'mysql-connector-java-5.1.49.jar') and place it under 'extensions/mysql-metadata-storage/'. See "
+                         + "'mysql-connector-j-8.2.0.jar') and place it under 'extensions/mysql-metadata-storage/'. See "
                          + "https://druid.apache.org/downloads for more details.",
                       className
         );

extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/firehose/sql/MySQLFirehoseDatabaseConnectorTest.java

@@ -25,7 +25,6 @@
 import com.google.common.collect.ImmutableSet;
 import nl.jqno.equalsverifier.EqualsVerifier;
 import org.apache.druid.jackson.DefaultObjectMapper;
-import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.metadata.MetadataStorageConnectorConfig;
 import org.apache.druid.metadata.storage.mysql.MySQLConnectorDriverConfig;
 import org.apache.druid.metadata.storage.mysql.MySQLMetadataStorageModule;
@@ -338,29 +337,6 @@ public boolean isEnforceAllowedProperties()
     );
   }
 
-  @Test
-  public void testFindPropertyKeysFromInvalidConnectUrl()
-  {
-    final String url = "jdbc:mysql:/invalid-url::3006";
-    MetadataStorageConnectorConfig connectorConfig = new MetadataStorageConnectorConfig()
-    {
-      @Override
-      public String getConnectURI()
-      {
-        return url;
-      }
-    };
-
-    expectedException.expect(RuntimeException.class);
-    expectedException.expectMessage(StringUtils.format("Invalid URL format for MySQL: [%s]", url));
-    new MySQLFirehoseDatabaseConnector(
-        connectorConfig,
-        null,
-        new JdbcAccessSecurityConfig(),
-        mySQLConnectorDriverConfig
-    );
-  }
-
   private static JdbcAccessSecurityConfig newSecurityConfigEnforcingAllowList(Set<String> allowedProperties)
   {
     return new JdbcAccessSecurityConfig()

extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLConnectorTest.java

@@ -20,8 +20,6 @@
 package org.apache.druid.metadata.storage.mysql;
 
 import com.google.common.base.Supplier;
-import com.mysql.jdbc.exceptions.MySQLTransactionRollbackException;
-import com.mysql.jdbc.exceptions.MySQLTransientException;
 import org.apache.druid.metadata.MetadataStorageConnectorConfig;
 import org.apache.druid.metadata.MetadataStorageTablesConfig;
 import org.junit.Assert;
@@ -57,16 +55,13 @@ public void testIsExceptionTransientMySql()
         new MySQLConnectorSslConfig(),
         MYSQL_DRIVER_CONFIG
     );
-    Assert.assertTrue(connector.connectorIsTransientException(new MySQLTransientException()));
-    Assert.assertTrue(connector.connectorIsTransientException(new MySQLTransactionRollbackException()));
     Assert.assertTrue(
         connector.connectorIsTransientException(new SQLException("some transient failure", "s0", 1317))
     );
     Assert.assertFalse(
         connector.connectorIsTransientException(new SQLException("totally realistic test data", "s0", 1337))
     );
-    // this method does not specially handle normal transient exceptions either, since it is not vendor specific
-    Assert.assertFalse(
+    Assert.assertTrue(
         connector.connectorIsTransientException(new SQLTransientConnectionException("transient"))
     );
   }
@@ -81,7 +76,7 @@ public void testIsExceptionTransientNoMySqlClazz()
         MARIADB_DRIVER_CONFIG
     );
     // no vendor specific for MariaDb, so should always be false
-    Assert.assertFalse(connector.connectorIsTransientException(new MySQLTransientException()));
+    Assert.assertFalse(connector.connectorIsTransientException(new SQLTransientException()));
     Assert.assertFalse(
         connector.connectorIsTransientException(new SQLException("some transient failure", "s0", 1317))
     );
@@ -116,7 +111,7 @@ public void testIsRootCausePacketTooBigException()
         connector.isRootCausePacketTooBigException(new SQLTransientException())
     );
     Assert.assertFalse(
-        connector.isRootCausePacketTooBigException(new MySQLTransientException())
+        connector.isRootCausePacketTooBigException(new SQLTransientException())
     );
   }
 

integration-tests-ex/cases/pom.xml

@@ -182,8 +182,8 @@
         </dependency>
         <!-- Tests can choose either the MySQL or MariaDB driver. -->
         <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
+            <groupId>com.mysql</groupId>
+            <artifactId>mysql-connector-j</artifactId>
             <version>${mysql.version}</version>
             <scope>runtime</scope>
         </dependency>
@@ -340,7 +340,7 @@
                     </ignoredUsedUndeclaredDependencies>
                     <!-- Dynamically loaded. -->
                     <ignoredUnusedDeclaredDependencies>
-                        <ignoredUnusedDeclaredDependency>mysql:mysql-connector-java:jar</ignoredUnusedDeclaredDependency>
+                        <ignoredUnusedDeclaredDependency>com.mysql:mysql-connector-j:jar</ignoredUnusedDeclaredDependency>
                     </ignoredUnusedDeclaredDependencies>
                 </configuration>
             </plugin>

integration-tests-ex/image/docker/Dockerfile

@@ -54,7 +54,7 @@ ENV DRUID_HOME=/usr/local/druid
 COPY apache-druid-${DRUID_VERSION}-bin.tar.gz /usr/local/
 COPY druid-it-tools-${DRUID_VERSION}.jar /tmp/druid/extensions/druid-it-tools/
 COPY kafka-protobuf-provider-${CONFLUENT_VERSION}.jar /tmp/druid/lib/
-COPY mysql-connector-java-${MYSQL_VERSION}.jar /tmp/druid/lib/
+COPY mysql-connector-j-${MYSQL_VERSION}.jar /tmp/druid/lib/
 COPY mariadb-java-client-${MARIADB_VERSION}.jar /tmp/druid/lib/
 COPY test-setup.sh /
 COPY druid.sh /

integration-tests-ex/image/pom.xml

@@ -102,8 +102,8 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven
                 </dependency>
                 <!-- Tests can choose either the MySQL or MariaDB driver. -->
                 <dependency>
-                    <groupId>mysql</groupId>
-                    <artifactId>mysql-connector-java</artifactId>
+                    <groupId>com.mysql</groupId>
+                    <artifactId>mysql-connector-j</artifactId>
                     <version>${mysql.version}</version>
                 </dependency>
                 <dependency>
@@ -131,8 +131,8 @@ Reference: https://dzone.com/articles/build-docker-image-from-maven
                                 <configuration>
                                     <artifactItems>
                                         <artifactItem>
-                                            <groupId>mysql</groupId>
-                                            <artifactId>mysql-connector-java</artifactId>
+                                            <groupId>com.mysql</groupId>
+                                            <artifactId>mysql-connector-j</artifactId>
                                             <version>${mysql.version}</version>
                                             <type>jar</type>
                                             <overWrite>true</overWrite>

integration-tests/docker/Dockerfile

@@ -53,11 +53,11 @@ COPY extensions/ /usr/local/druid/extensions/
 # target path must match the exact path referenced in environment-configs/common
 # alternatively: Download the MariaDB Java connector, and pretend it is the mysql connector
 RUN if [ "$MYSQL_DRIVER_CLASSNAME" = "com.mysql.jdbc.Driver" ] ; \
-    then wget -q "https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_VERSION/mysql-connector-java-$MYSQL_VERSION.jar" \
-    -O /usr/local/druid/lib/mysql-connector-java.jar; \
+    then wget -q "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/$MYSQL_VERSION/mysql-connector-j-$MYSQL_VERSION.jar" \
+    -O /usr/local/druid/lib/mysql-connector-j.jar; \
     elif [ "$MYSQL_DRIVER_CLASSNAME" = "org.mariadb.jdbc.Driver" ] ; \
     then wget -q "https://repo1.maven.org/maven2/org/mariadb/jdbc/mariadb-java-client/$MARIA_VERSION/mariadb-java-client-$MARIA_VERSION.jar" \
-    -O /usr/local/druid/lib/mysql-connector-java.jar; \
+    -O /usr/local/druid/lib/mysql-connector-j.jar; \
     fi
 
 # download kafka protobuf provider

integration-tests/docker/environment-configs/common

@@ -24,7 +24,7 @@ LC_ALL=C.UTF-8
 # JAVA OPTS
 COMMON_DRUID_JAVA_OPTS=-Duser.timezone=UTC -Dfile.encoding=UTF-8 -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:OnOutOfMemoryError='chmod 644 /shared/logs/*.hprof' -XX:HeapDumpPath=/shared/logs
 DRUID_DEP_BIN_DIR=/shared/docker/bin
-DRUID_DEP_LIB_DIR=/shared/hadoop_xml:/shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
+DRUID_DEP_LIB_DIR=/shared/hadoop_xml:/shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-j.jar
 
 # Druid configs
 # If you are making a change in load list below, make the necessary changes in github actions too

integration-tests/docker/environment-configs/common-ldap

@@ -25,7 +25,7 @@ AWS_REGION=us-east-1
 # JAVA OPTS
 COMMON_DRUID_JAVA_OPTS=-Duser.timezone=UTC -Dfile.encoding=UTF-8 -Dlog4j.configurationFile=/shared/docker/lib/log4j2.xml -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:OnOutOfMemoryError='chmod 644 /shared/logs/*.hprof'  -XX:HeapDumpPath=/shared/logs
 DRUID_DEP_BIN_DIR=/shared/docker/bin
-DRUID_DEP_LIB_DIR=/shared/hadoop_xml:/shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-java.jar
+DRUID_DEP_LIB_DIR=/shared/hadoop_xml:/shared/docker/lib/*:/usr/local/druid/lib/mysql-connector-j.jar
 
 # Druid configs
 # If you are making a change in load list below, make the necessary changes in github actions too