From 2154d2fdf0d826bd6fe1410aaa8eb621ad05408c Mon Sep 17 00:00:00 2001 From: James Turton Date: Thu, 24 Nov 2022 09:53:02 +0200 Subject: [PATCH 1/5] Fix default value of validateCertificates in SplunkConnection. --- .../org/apache/drill/exec/store/splunk/SplunkConnection.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkConnection.java b/contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkConnection.java index f3a1e458b1d..61331852bdb 100644 --- a/contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkConnection.java +++ b/contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkConnection.java @@ -71,7 +71,7 @@ public SplunkConnection(SplunkPluginConfig config, String queryUserName) { this.owner = config.getOwner(); this.token = config.getToken(); this.cookie = config.getCookie(); - this.validateCertificates = Optional.ofNullable(config.getValidateCertificates()).orElse(false); + this.validateCertificates = Optional.ofNullable(config.getValidateCertificates()).orElse(true); this.connectionAttempts = config.getReconnectRetries(); service = connect(); } From ada7c4dd96d645a824d6f1ed75f2d56f4cabc00a Mon Sep 17 00:00:00 2001 From: James Turton Date: Mon, 12 Dec 2022 15:09:12 +0200 Subject: [PATCH 2/5] Upgrade splunk-sdk-java to 1.9.3. --- contrib/storage-splunk/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/storage-splunk/pom.xml b/contrib/storage-splunk/pom.xml index 140bcb33f40..81abc6e6943 100644 --- a/contrib/storage-splunk/pom.xml +++ b/contrib/storage-splunk/pom.xml @@ -42,7 +42,7 @@ com.splunk splunk - 1.9.1 + 1.9.3 org.apache.maven.plugins From 2e88b4bd38adb2302212d6387c98f91754c609aa Mon Sep 17 00:00:00 2001 From: James Turton Date: Mon, 12 Dec 2022 15:09:26 +0200 Subject: [PATCH 3/5] Remove the dependency of Splunk unit tests on the _audit index. --- .../exec/store/splunk/SplunkPluginTest.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/contrib/storage-splunk/src/test/java/org/apache/drill/exec/store/splunk/SplunkPluginTest.java b/contrib/storage-splunk/src/test/java/org/apache/drill/exec/store/splunk/SplunkPluginTest.java index ecc00c7ce0b..af09d3af7c7 100644 --- a/contrib/storage-splunk/src/test/java/org/apache/drill/exec/store/splunk/SplunkPluginTest.java +++ b/contrib/storage-splunk/src/test/java/org/apache/drill/exec/store/splunk/SplunkPluginTest.java @@ -152,23 +152,23 @@ public void testRawSPLQuery() throws Exception { @Test public void testExplictFieldsQuery() throws Exception { - String sql = "SELECT acceleration_id, action, add_offset, add_timestamp FROM splunk._audit LIMIT 2"; + String sql = "SELECT component, event_message, host, _time FROM splunk._internal LIMIT 3"; client.testBuilder() .sqlQuery(sql) .unOrdered() - .baselineColumns("acceleration_id", "action", "add_offset", "add_timestamp") - .expectsNumRecords(2) + .baselineColumns("component", "event_message", "host", "_time") + .expectsNumRecords(3) .go(); } @Test public void testExplicitFieldsWithLimitQuery() throws Exception { - String sql = "SELECT action, _sourcetype, _subsecond, _time FROM splunk._audit LIMIT 3"; + String sql = "SELECT `group`, _sourcetype, _subsecond, _time FROM splunk._internal LIMIT 3"; client.testBuilder() .sqlQuery(sql) .unOrdered() - .baselineColumns( "acceleration_id", "action", "add_offset", "add_timestamp") + .baselineColumns( "group", "_sourcetype", "_subsecond", "_time") .expectsNumRecords(3) .go(); } @@ -176,18 +176,18 @@ public void testExplicitFieldsWithLimitQuery() throws Exception { @Test @Ignore("the result is not consistent on system tables") public void testExplicitFieldsWithSourceType() throws Exception { - String sql = "SELECT action, _sourcetype, _subsecond, _time FROM splunk._audit WHERE sourcetype='audittrail' LIMIT 5"; + String sql = "SELECT `group`, _sourcetype, _subsecond, _time FROM splunk._internal WHERE sourcetype='splunkd' LIMIT 3"; client.testBuilder() .sqlQuery(sql) .unOrdered() - .baselineColumns( "acceleration_id", "action", "add_offset", "add_timestamp") - .expectsNumRecords(5) + .baselineColumns( "group", "_sourcetype", "_subsecond", "_time") + .expectsNumRecords(3) .go(); } @Test public void testExplicitFieldsWithOneFieldLimitQuery() throws Exception { - String sql = "SELECT `component` FROM splunk.`_introspection` ORDER BY `component` LIMIT 2"; + String sql = "SELECT distinct `component` FROM splunk.`_introspection` ORDER BY `component` LIMIT 2"; RowSet results = client.queryBuilder().sql(sql).rowSet(); TupleMetadata expectedSchema = new SchemaBuilder() @@ -233,12 +233,12 @@ public void testMultipleEqualityFilterQuery() throws Exception { @Test public void testFilterOnUnProjectedColumnQuery() throws Exception { - String sql = "SELECT action, _sourcetype, _subsecond, _time FROM splunk._audit WHERE sourcetype='audittrail' LIMIT 5"; + String sql = "SELECT `group`, _sourcetype, _subsecond, _time FROM splunk._internal WHERE sourcetype='splunkd' LIMIT 3"; client.testBuilder() .sqlQuery(sql) .unOrdered() - .baselineColumns( "acceleration_id", "action", "add_offset", "add_timestamp") - .expectsNumRecords(5) + .baselineColumns( "group", "_sourcetype", "_subsecond", "_time") + .expectsNumRecords(3) .go(); } From 8086dd475585733ab8742684024f1cc8a07b4aa8 Mon Sep 17 00:00:00 2001 From: James Turton Date: Mon, 12 Dec 2022 17:27:18 +0200 Subject: [PATCH 4/5] Patch Splunk's broken validateCertificates toggle. --- .../exec/store/splunk/SplunkConnection.java | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkConnection.java b/contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkConnection.java index 61331852bdb..c9571c6d5b6 100644 --- a/contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkConnection.java +++ b/contrib/storage-splunk/src/main/java/org/apache/drill/exec/store/splunk/SplunkConnection.java @@ -31,6 +31,15 @@ import org.slf4j.LoggerFactory; import java.util.concurrent.TimeUnit; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.X509Certificate; import java.util.Optional; /** @@ -100,9 +109,19 @@ public SplunkConnection(SplunkPluginConfig config, Service service, String query * Connects to Splunk instance * @return an active Splunk connection. */ + public Service connect() { HttpService.setSslSecurityProtocol(SSLSecurityProtocol.TLSv1_2); HttpService.setValidateCertificates(validateCertificates); + if (! validateCertificates) { + try { + HttpService.setSSLSocketFactory(createAllTrustingSSLFactory()); + } catch (KeyManagementException e) { + throw UserException.connectionError(e) + .message("Error validating SSL Certificates: " + e.getMessage()) + .build(logger); + } + } ServiceArgs loginArgs = new ServiceArgs(); if (scheme != null) { @@ -139,6 +158,7 @@ public Service connect() { throw UserException .connectionError(e) .message("Unable to connect to Splunk at %s:%s", hostname, port) + .addContext(e.getMessage()) .build(logger); } logger.info("Successfully connected to {} on port {}", hostname, port); @@ -152,4 +172,45 @@ public Service connect() { public EntityCollection getIndexes() { return service.getIndexes(); } + + /** + * As of version 1.8, Splunk's SDK introduced a boolean parameter which + * is supposed to control whether the SDK will validate SSL certificates + * or not. Unfortunately the parameter does not actually seem to have + * any effect and the end result is that when making Splunk calls, + * Splunk will always attempt to verify the SSL certificates, even when + * the parameter is set to false. This method does what the parameter + * is supposed to do in the SDK and adds and all trusting SSL Socket + * Factory to the HTTP client in Splunk's SDK. In the event Splunk + * fixes this issue, we can remove this method. + * + * @return A {@link SSLSocketFactory} which trusts any SSL certificate, + * even ones from Splunk + * @throws KeyManagementException Thros + */ + private SSLSocketFactory createAllTrustingSSLFactory() throws KeyManagementException { + SSLContext context; + try { + context = SSLContext.getInstance("TLS"); + } catch (NoSuchAlgorithmException e) { + throw UserException.validationError(e) + .message("Error establishing SSL connection: Invalid scheme: " + e.getMessage()) + .build(logger); + } + TrustManager[] trustAll = new TrustManager[]{ + new X509TrustManager() { + public X509Certificate[] getAcceptedIssuers() { + return null; + } + public void checkClientTrusted(X509Certificate[] certs, String authType) { + // No op + } + public void checkServerTrusted(X509Certificate[] certs, String authType) { + // No op + } + } + }; + context.init(null, trustAll, null); + return context.getSocketFactory(); + } } From 9006c70149ec5e35b22dfaca2367996adc374ef4 Mon Sep 17 00:00:00 2001 From: James Turton Date: Tue, 13 Dec 2022 13:40:18 +0200 Subject: [PATCH 5/5] Adjust unit test queries to work with populated indexes. --- .../apache/drill/exec/store/splunk/SplunkPluginTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/storage-splunk/src/test/java/org/apache/drill/exec/store/splunk/SplunkPluginTest.java b/contrib/storage-splunk/src/test/java/org/apache/drill/exec/store/splunk/SplunkPluginTest.java index af09d3af7c7..2109de28de1 100644 --- a/contrib/storage-splunk/src/test/java/org/apache/drill/exec/store/splunk/SplunkPluginTest.java +++ b/contrib/storage-splunk/src/test/java/org/apache/drill/exec/store/splunk/SplunkPluginTest.java @@ -152,7 +152,7 @@ public void testRawSPLQuery() throws Exception { @Test public void testExplictFieldsQuery() throws Exception { - String sql = "SELECT component, event_message, host, _time FROM splunk._internal LIMIT 3"; + String sql = "SELECT component, event_message, host, _time FROM splunk._introspection LIMIT 3"; client.testBuilder() .sqlQuery(sql) @@ -164,7 +164,7 @@ public void testExplictFieldsQuery() throws Exception { @Test public void testExplicitFieldsWithLimitQuery() throws Exception { - String sql = "SELECT `group`, _sourcetype, _subsecond, _time FROM splunk._internal LIMIT 3"; + String sql = "SELECT `group`, _sourcetype, _subsecond, _time FROM splunk._introspection LIMIT 3"; client.testBuilder() .sqlQuery(sql) .unOrdered() @@ -176,7 +176,7 @@ public void testExplicitFieldsWithLimitQuery() throws Exception { @Test @Ignore("the result is not consistent on system tables") public void testExplicitFieldsWithSourceType() throws Exception { - String sql = "SELECT `group`, _sourcetype, _subsecond, _time FROM splunk._internal WHERE sourcetype='splunkd' LIMIT 3"; + String sql = "SELECT `group`, _sourcetype, _subsecond, _time FROM splunk._introspection WHERE sourcetype='splunkd' LIMIT 3"; client.testBuilder() .sqlQuery(sql) .unOrdered() @@ -233,7 +233,7 @@ public void testMultipleEqualityFilterQuery() throws Exception { @Test public void testFilterOnUnProjectedColumnQuery() throws Exception { - String sql = "SELECT `group`, _sourcetype, _subsecond, _time FROM splunk._internal WHERE sourcetype='splunkd' LIMIT 3"; + String sql = "SELECT `group`, _sourcetype, _subsecond, _time FROM splunk._introspection WHERE sourcetype='splunk_disk_objects' LIMIT 3"; client.testBuilder() .sqlQuery(sql) .unOrdered()