Skip to content

Commit e80374e

Browse files
zy-kkkdataroaring
zy-kkk
authored and
dataroaring
committed
[improvement](jdbc catalog) Compatible with ojdbc6 by adding version check (#39341)
In previous versions, we used a method based on JDBC 4.2 to read data, so it was equivalent to abandoning support for ojdbc6. However, we recently found that a large number of users still use Oracle version 11g, which will have some unexpected compatibility issues when using ojdbc8 to connect. Therefore, I use version verification to make it compatible with both ojdbc6 and ojdbc8, so that good compatibility can be obtained through ojdbc6, and better reading efficiency can be obtained through ojdbc8.
1 parent 805a0d5 commit e80374e

File tree

6 files changed

+137
-38
lines changed

6 files changed

+137
-38
lines changed

‎fe/be-java-extensions/jdbc-scanner/pom.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@ under the License.
4545
<artifactId>HikariCP</artifactId>
4646
<scope>provided</scope>
4747
</dependency>
48+
<dependency>
49+
<groupId>org.semver4j</groupId>
50+
<artifactId>semver4j</artifactId>
51+
</dependency>
4852
</dependencies>
4953

5054
<build>

‎fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,13 @@
3333
import org.apache.thrift.TDeserializer;
3434
import org.apache.thrift.TException;
3535
import org.apache.thrift.protocol.TBinaryProtocol;
36+
import org.semver4j.Semver;
3637

3738
import java.io.FileNotFoundException;
3839
import java.lang.reflect.Array;
3940
import java.net.MalformedURLException;
4041
import java.sql.Connection;
42+
import java.sql.DatabaseMetaData;
4143
import java.sql.Date;
4244
import java.sql.PreparedStatement;
4345
import java.sql.ResultSet;
@@ -66,6 +68,7 @@ public abstract class BaseJdbcExecutor implements JdbcExecutor {
6668
protected VectorTable outputTable = null;
6769
protected int batchSizeNum = 0;
6870
protected int curBlockRows = 0;
71+
protected String jdbcDriverVersion;
6972

7073
public BaseJdbcExecutor(byte[] thriftParams) throws Exception {
7174
TJdbcExecutorCtorParams request = new TJdbcExecutorCtorParams();
@@ -92,11 +95,12 @@ public BaseJdbcExecutor(byte[] thriftParams) throws Exception {
9295
.setConnectionPoolKeepAlive(request.connection_pool_keep_alive);
9396
JdbcDataSource.getDataSource().setCleanupInterval(request.connection_pool_cache_clear_time);
9497
init(config, request.statement);
98+
this.jdbcDriverVersion = getJdbcDriverVersion();
9599
}
96100

97101
public void close() throws Exception {
98102
try {
99-
if (stmt != null) {
103+
if (stmt != null && !stmt.isClosed()) {
100104
try {
101105
stmt.cancel();
102106
} catch (SQLException e) {
@@ -525,6 +529,30 @@ private void insertNullColumn(int parameterIndex, ColumnType.Type dorisType)
525529
}
526530
}
527531

532+
private String getJdbcDriverVersion() {
533+
try {
534+
if (conn != null) {
535+
DatabaseMetaData metaData = conn.getMetaData();
536+
return metaData.getDriverVersion();
537+
} else {
538+
return null;
539+
}
540+
} catch (SQLException e) {
541+
LOG.warn("Failed to retrieve JDBC Driver version", e);
542+
return null;
543+
}
544+
}
545+
546+
protected boolean isJdbcVersionGreaterThanOrEqualTo(String version) {
547+
Semver currentVersion = Semver.coerce(jdbcDriverVersion);
548+
Semver targetVersion = Semver.coerce(version);
549+
if (currentVersion != null && targetVersion != null) {
550+
return currentVersion.isGreaterThanOrEqualTo(targetVersion);
551+
} else {
552+
return false;
553+
}
554+
}
555+
528556
protected String trimSpaces(String str) {
529557
int end = str.length() - 1;
530558
while (end >= 0 && str.charAt(end) == ' ') {

‎fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java‎

Lines changed: 78 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
import java.nio.charset.StandardCharsets;
3434
import java.sql.Clob;
3535
import java.sql.SQLException;
36-
import java.time.LocalDate;
36+
import java.sql.Timestamp;
3737
import java.time.LocalDateTime;
3838

3939
public class OracleJdbcExecutor extends BaseJdbcExecutor {
@@ -65,42 +65,83 @@ protected void initializeBlock(int columnCount, String[] replaceStringList, int
6565

6666
@Override
6767
protected Object getColumnValue(int columnIndex, ColumnType type, String[] replaceStringList) throws SQLException {
68-
try {
69-
switch (type.getType()) {
70-
case TINYINT:
71-
return resultSet.getObject(columnIndex + 1, Byte.class);
72-
case SMALLINT:
73-
return resultSet.getObject(columnIndex + 1, Short.class);
74-
case INT:
75-
return resultSet.getObject(columnIndex + 1, Integer.class);
76-
case BIGINT:
77-
return resultSet.getObject(columnIndex + 1, Long.class);
78-
case FLOAT:
79-
return resultSet.getObject(columnIndex + 1, Float.class);
80-
case DOUBLE:
81-
return resultSet.getObject(columnIndex + 1, Double.class);
82-
case LARGEINT:
83-
case DECIMALV2:
84-
case DECIMAL32:
85-
case DECIMAL64:
86-
case DECIMAL128:
87-
return resultSet.getObject(columnIndex + 1, BigDecimal.class);
88-
case DATE:
89-
case DATEV2:
90-
return resultSet.getObject(columnIndex + 1, LocalDate.class);
91-
case DATETIME:
92-
case DATETIMEV2:
93-
return resultSet.getObject(columnIndex + 1, LocalDateTime.class);
94-
case CHAR:
95-
case VARCHAR:
96-
case STRING:
97-
return resultSet.getObject(columnIndex + 1);
98-
default:
99-
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
100-
}
101-
} catch (AbstractMethodError e) {
102-
LOG.warn("Detected an outdated ojdbc driver. Please use ojdbc8 or above.", e);
103-
throw new SQLException("Detected an outdated ojdbc driver. Please use ojdbc8 or above.");
68+
if (isJdbcVersionGreaterThanOrEqualTo("12.2.0")) {
69+
return newGetColumnValue(columnIndex, type, replaceStringList);
70+
} else {
71+
return oldGetColumnValue(columnIndex, type, replaceStringList);
72+
}
73+
}
74+
75+
private Object newGetColumnValue(int columnIndex, ColumnType type, String[] replaceStringList) throws SQLException {
76+
switch (type.getType()) {
77+
case TINYINT:
78+
return resultSet.getObject(columnIndex + 1, Byte.class);
79+
case SMALLINT:
80+
return resultSet.getObject(columnIndex + 1, Short.class);
81+
case INT:
82+
return resultSet.getObject(columnIndex + 1, Integer.class);
83+
case BIGINT:
84+
return resultSet.getObject(columnIndex + 1, Long.class);
85+
case FLOAT:
86+
return resultSet.getObject(columnIndex + 1, Float.class);
87+
case DOUBLE:
88+
return resultSet.getObject(columnIndex + 1, Double.class);
89+
case LARGEINT:
90+
case DECIMALV2:
91+
case DECIMAL32:
92+
case DECIMAL64:
93+
case DECIMAL128:
94+
return resultSet.getObject(columnIndex + 1, BigDecimal.class);
95+
case DATETIME:
96+
case DATETIMEV2:
97+
return resultSet.getObject(columnIndex + 1, LocalDateTime.class);
98+
case CHAR:
99+
case VARCHAR:
100+
case STRING:
101+
return resultSet.getObject(columnIndex + 1);
102+
default:
103+
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
104+
}
105+
}
106+
107+
private Object oldGetColumnValue(int columnIndex, ColumnType type, String[] replaceStringList) throws SQLException {
108+
switch (type.getType()) {
109+
case TINYINT:
110+
byte tinyIntVal = resultSet.getByte(columnIndex + 1);
111+
return resultSet.wasNull() ? null : tinyIntVal;
112+
case SMALLINT:
113+
short smallIntVal = resultSet.getShort(columnIndex + 1);
114+
return resultSet.wasNull() ? null : smallIntVal;
115+
case INT:
116+
int intVal = resultSet.getInt(columnIndex + 1);
117+
return resultSet.wasNull() ? null : intVal;
118+
case BIGINT:
119+
long bigIntVal = resultSet.getLong(columnIndex + 1);
120+
return resultSet.wasNull() ? null : bigIntVal;
121+
case FLOAT:
122+
float floatVal = resultSet.getFloat(columnIndex + 1);
123+
return resultSet.wasNull() ? null : floatVal;
124+
case DOUBLE:
125+
double doubleVal = resultSet.getDouble(columnIndex + 1);
126+
return resultSet.wasNull() ? null : doubleVal;
127+
case LARGEINT:
128+
case DECIMALV2:
129+
case DECIMAL32:
130+
case DECIMAL64:
131+
case DECIMAL128:
132+
BigDecimal decimalVal = resultSet.getBigDecimal(columnIndex + 1);
133+
return resultSet.wasNull() ? null : decimalVal;
134+
case DATETIME:
135+
case DATETIMEV2:
136+
Timestamp timestampVal = resultSet.getTimestamp(columnIndex + 1);
137+
return resultSet.wasNull() ? null : timestampVal.toLocalDateTime();
138+
case CHAR:
139+
case VARCHAR:
140+
case STRING:
141+
Object stringVal = resultSet.getObject(columnIndex + 1);
142+
return resultSet.wasNull() ? null : stringVal;
143+
default:
144+
throw new IllegalArgumentException("Unsupported column type: " + type.getType());
104145
}
105146
}
106147

‎fe/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -368,6 +368,7 @@ under the License.
368368
<airlift.version>202</airlift.version>
369369
<azure.sdk.version>1.2.24</azure.sdk.version>
370370
<azure.sdk.batch.version>12.22.0</azure.sdk.batch.version>
371+
<semver4j.version>5.3.0</semver4j.version>
371372
</properties>
372373
<profiles>
373374
<profile>
@@ -1752,6 +1753,11 @@ under the License.
17521753
<type>pom</type>
17531754
<scope>import</scope>
17541755
</dependency>
1756+
<dependency>
1757+
<groupId>org.semver4j</groupId>
1758+
<artifactId>semver4j</artifactId>
1759+
<version>${semver4j.version}</version>
1760+
</dependency>
17551761
</dependencies>
17561762
</dependencyManagement>
17571763
<dependencies>

‎regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,3 +265,7 @@ Doris
265265
-- !query_lower_3 --
266266
doris
267267

268+
-- !query_ojdbc6_all_types --
269+
1 111 123 7456123.89 573 34 673.43 34.1264 60.0 23.231 99 9999 999999999 999999999999999999 999 99999 9999999999 9999999999999999999 1 china beijing alice abcdefghrjkmnopq 123.45 12300 0.0012345 2022-01-21T05:23:01 2019-11-12T20:33:57.999 2019-11-12T20:33:57.999998 2019-11-12T20:33:57.999996 2019-11-12T20:33:57.999997 223-9 12 10:23:1.123457000
270+
2 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N
271+

‎regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ suite("test_oracle_jdbc_catalog", "p0,external,oracle,external_docker,external_d
2121
String s3_endpoint = getS3Endpoint()
2222
String bucket = getS3BucketName()
2323
String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/ojdbc8.jar"
24+
String driver6_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/ojdbc6.jar"
2425
if (enabled != null && enabled.equalsIgnoreCase("true")) {
2526
String catalog_name = "oracle_catalog";
2627
String internal_db_name = "regression_test_jdbc_catalog_p0";
@@ -281,5 +282,20 @@ suite("test_oracle_jdbc_catalog", "p0,external,oracle,external_docker,external_d
281282
qt_query_lower_3 """ select doris_3 from doris_test.lower_test; """
282283

283284
sql """drop catalog if exists ${catalog_name} """
285+
286+
// test for ojdbc6
287+
sql """drop catalog if exists oracle_ojdbc6; """
288+
sql """create catalog if not exists oracle_ojdbc6 properties(
289+
"type"="jdbc",
290+
"user"="doris_test",
291+
"password"="123456",
292+
"jdbc_url" = "jdbc:oracle:thin:@${externalEnvIp}:${oracle_port}:${SID}",
293+
"driver_url" = "${driver6_url}",
294+
"driver_class" = "oracle.jdbc.OracleDriver"
295+
);"""
296+
sql """ use oracle_ojdbc6.DORIS_TEST; """
297+
qt_query_ojdbc6_all_types """ select * from oracle_ojdbc6.DORIS_TEST.TEST_ALL_TYPES order by 1; """
298+
299+
sql """drop catalog if exists oracle_ojdbc6; """
284300
}
285301
}

0 commit comments

Comments
 (0)