From 8ddab43d09d551ec5f4f83d0d96d9075c2fe1129 Mon Sep 17 00:00:00 2001 From: Barry Nouwt Date: Fri, 23 Feb 2024 20:26:38 +0100 Subject: [PATCH 1/3] Introduce docker tests for EXPOSED URL. --- examples/runtime-tests/.gitignore | 1 + examples/runtime-tests/README.md | 5 + examples/runtime-tests/docker-compose.yml | 179 ++++++++++++++++++ .../{readme.md => README.md} | 0 smart-connector-rest-dist/pom.xml | 7 + 5 files changed, 192 insertions(+) create mode 100644 examples/runtime-tests/.gitignore create mode 100644 examples/runtime-tests/README.md create mode 100644 examples/runtime-tests/docker-compose.yml rename examples/unreachable-runtimes/{readme.md => README.md} (100%) diff --git a/examples/runtime-tests/.gitignore b/examples/runtime-tests/.gitignore new file mode 100644 index 000000000..d6e830be9 --- /dev/null +++ b/examples/runtime-tests/.gitignore @@ -0,0 +1 @@ +.python-version \ No newline at end of file diff --git a/examples/runtime-tests/README.md b/examples/runtime-tests/README.md new file mode 100644 index 000000000..603e85872 --- /dev/null +++ b/examples/runtime-tests/README.md @@ -0,0 +1,5 @@ +## Runtime tests +This docker compose example is used to test several scenario's that are impossible (or difficult) to test within Java Unit Tests. + +### Incorrect `KE_RUNTIME_EXPOSED_URL` +The `docker-compose.yml` contains the exact same scenario as the `multiple-runtimes` example, but additionally it contains several broken runtimes where the exposed URL is incorrect. With this docker project we can test whether these different types of exposed urls are breaking their own KER, other KERs or the Knowledge Directory. \ No newline at end of file diff --git a/examples/runtime-tests/docker-compose.yml b/examples/runtime-tests/docker-compose.yml new file mode 100644 index 000000000..8e915e64b --- /dev/null +++ b/examples/runtime-tests/docker-compose.yml @@ -0,0 +1,179 @@ +services: + # This is the knowledge directory, facilitating discovery between different + # runtimes. It exposes its service over port 8282. + knowledge-directory: + image: ghcr.io/tno/knowledge-engine/knowledge-directory:1.2.3 + + # These services are seperate Knowledge Engine runtime, which can host + # multiple smart connectors. Note that the REST API port is a DIFFERENT port + # number than the ones configured below. It is still the default 8280. + runtime-1: + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + environment: + KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication. + KE_RUNTIME_EXPOSED_URL: http://runtime-1:8081 # The URL where the runtime is available for inter-runtime communication from the outside. + KD_URL: http://knowledge-directory:8282 + runtime-2: + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + environment: + KE_RUNTIME_PORT: 8081 + KE_RUNTIME_EXPOSED_URL: http://runtime-2:8081 + KD_URL: http://knowledge-directory:8282 + runtime-3: + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + environment: + KE_RUNTIME_PORT: 8081 + KE_RUNTIME_EXPOSED_URL: http://runtime-3:8081 + KD_URL: http://knowledge-directory:8282 + + # These Knowledge Bases use the different runtimes, and exchange data with eachother. + kb1: + build: ../common/asking_kb + environment: + KE_URL: http://runtime-1:8280/rest + KB_ID: http://example.org/kb1 + PREFIXES: | + { + "ex": "http://example.org/" + } + GRAPH_PATTERN: | + ?a ex:relatedTo ?b . + kb2: + build: ../common/answering_kb + environment: + KE_URL: http://runtime-2:8280/rest + KB_ID: http://example.org/kb2 + PREFIXES: | + { + "ex": "http://example.org/" + } + GRAPH_PATTERN: | + ?a ex:relatedTo ?b . + KB_DATA: | + [ + { + "a": "", + "b": "" + }, + { + "a": "", + "b": "" + } + ] + kb3: + build: ../common/answering_kb + environment: + KE_URL: http://runtime-3:8280/rest + KB_ID: http://example.org/kb3 + PREFIXES: | + { + "ex": "http://example.org/" + } + GRAPH_PATTERN: | + ?a ex:relatedTo ?b . + KB_DATA: | + [ + { + "a": "", + "b": "" + }, + { + "a": "", + "b": "" + } + ] + broken-1: + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + environment: + KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication. + KE_RUNTIME_EXPOSED_URL: runtime-3:8081 # The 'http://' part of the URL with a port is missing. + KD_URL: http://knowledge-directory:8282 + brokenkb1: + build: ../common/answering_kb + environment: + KE_URL: http://broken-1:8280/rest + KB_ID: http://example.org/brokenkb1 + PREFIXES: | + { + "ex": "http://example.org/" + } + GRAPH_PATTERN: | + ?a ex:relatedTo ?b . + KB_DATA: | + [] + broken-2: + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + environment: + KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication. + KE_RUNTIME_EXPOSED_URL: 3test.runtime.nl:8081 # The 'http://' part of the URL with a port is missing and the first character is numeric. + KD_URL: http://knowledge-directory:8282 + brokenkb2: + build: ../common/answering_kb + environment: + KE_URL: http://broken-2:8280/rest + KB_ID: http://example.org/brokenkb2 + PREFIXES: | + { + "ex": "http://example.org/" + } + GRAPH_PATTERN: | + ?a ex:relatedTo ?b . + KB_DATA: | + [] + broken-3: + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + environment: + KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication. + KE_RUNTIME_EXPOSED_URL: 3test.runtime.nl # The 'http://' part of the URL without a port is missing and the first character is numeric. + KD_URL: http://knowledge-directory:8282 + brokenkb3: + build: ../common/answering_kb + environment: + KE_URL: http://broken-3:8280/rest + KB_ID: http://example.org/brokenkb3 + PREFIXES: | + { + "ex": "http://example.org/" + } + GRAPH_PATTERN: | + ?a ex:relatedTo ?b . + KB_DATA: | + [] + broken-4: + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + environment: + KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication. + KE_RUNTIME_EXPOSED_URL: http://runtime-3:8081/ # The URL ends with a '/' + KD_URL: http://knowledge-directory:8282 + brokenkb4: + build: ../common/answering_kb + environment: + KE_URL: http://broken-4:8280/rest + KB_ID: http://example.org/brokenkb4 + PREFIXES: | + { + "ex": "http://example.org/" + } + GRAPH_PATTERN: | + ?a ex:relatedTo ?b . + KB_DATA: | + [] + broken-5: + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + environment: + KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication. + KE_RUNTIME_EXPOSED_URL: runtime-3 # The 'http://' part of the URL without a port is missing. + KD_URL: http://knowledge-directory:8282 + brokenkb5: + build: ../common/answering_kb + environment: + KE_URL: http://broken-5:8280/rest + KB_ID: http://example.org/brokenkb5 + PREFIXES: | + { + "ex": "http://example.org/" + } + GRAPH_PATTERN: | + ?a ex:relatedTo ?b . + KB_DATA: | + [] \ No newline at end of file diff --git a/examples/unreachable-runtimes/readme.md b/examples/unreachable-runtimes/README.md similarity index 100% rename from examples/unreachable-runtimes/readme.md rename to examples/unreachable-runtimes/README.md diff --git a/smart-connector-rest-dist/pom.xml b/smart-connector-rest-dist/pom.xml index f7b15743c..5d524ef50 100644 --- a/smart-connector-rest-dist/pom.xml +++ b/smart-connector-rest-dist/pom.xml @@ -37,6 +37,13 @@ + + org.apache.maven.plugins + maven-surefire-plugin + + false + + org.apache.maven.plugins maven-dependency-plugin From 50808d1e9236ec4daf60e100b34bc8e39105819d Mon Sep 17 00:00:00 2001 From: Barry Nouwt Date: Tue, 27 Feb 2024 13:39:56 +0100 Subject: [PATCH 2/3] Add validation on the exposed url and exit the JVM is invalid. --- examples/runtime-tests/docker-compose.yml | 16 ++++++------- .../smartconnector/runtime/KeRuntime.java | 23 +++++++++++++++++++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/examples/runtime-tests/docker-compose.yml b/examples/runtime-tests/docker-compose.yml index 8e915e64b..5bcce9861 100644 --- a/examples/runtime-tests/docker-compose.yml +++ b/examples/runtime-tests/docker-compose.yml @@ -8,19 +8,19 @@ services: # multiple smart connectors. Note that the REST API port is a DIFFERENT port # number than the ones configured below. It is still the default 8280. runtime-1: - image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.4-SNAPSHOT environment: KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication. KE_RUNTIME_EXPOSED_URL: http://runtime-1:8081 # The URL where the runtime is available for inter-runtime communication from the outside. KD_URL: http://knowledge-directory:8282 runtime-2: - image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.4-SNAPSHOT environment: KE_RUNTIME_PORT: 8081 KE_RUNTIME_EXPOSED_URL: http://runtime-2:8081 KD_URL: http://knowledge-directory:8282 runtime-3: - image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.4-SNAPSHOT environment: KE_RUNTIME_PORT: 8081 KE_RUNTIME_EXPOSED_URL: http://runtime-3:8081 @@ -83,7 +83,7 @@ services: } ] broken-1: - image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.4-SNAPSHOT environment: KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication. KE_RUNTIME_EXPOSED_URL: runtime-3:8081 # The 'http://' part of the URL with a port is missing. @@ -102,7 +102,7 @@ services: KB_DATA: | [] broken-2: - image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.4-SNAPSHOT environment: KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication. KE_RUNTIME_EXPOSED_URL: 3test.runtime.nl:8081 # The 'http://' part of the URL with a port is missing and the first character is numeric. @@ -121,7 +121,7 @@ services: KB_DATA: | [] broken-3: - image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.4-SNAPSHOT environment: KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication. KE_RUNTIME_EXPOSED_URL: 3test.runtime.nl # The 'http://' part of the URL without a port is missing and the first character is numeric. @@ -140,7 +140,7 @@ services: KB_DATA: | [] broken-4: - image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.4-SNAPSHOT environment: KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication. KE_RUNTIME_EXPOSED_URL: http://runtime-3:8081/ # The URL ends with a '/' @@ -159,7 +159,7 @@ services: KB_DATA: | [] broken-5: - image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.3 + image: ghcr.io/tno/knowledge-engine/smart-connector:1.2.4-SNAPSHOT environment: KE_RUNTIME_PORT: 8081 # The port that the KE uses to listen for inter-KE-runtime communication. KE_RUNTIME_EXPOSED_URL: runtime-3 # The 'http://' part of the URL without a port is missing. diff --git a/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/KeRuntime.java b/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/KeRuntime.java index f74b9abf3..b308f4ec9 100644 --- a/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/KeRuntime.java +++ b/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/KeRuntime.java @@ -1,7 +1,9 @@ package eu.knowledge.engine.smartconnector.runtime; +import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; @@ -41,6 +43,27 @@ public class KeRuntime { System.exit(1); } + // execute some validation on the EXPOSED URL, because it can have severe + // consequences + String url = getConfigProperty(CONF_KEY_MY_EXPOSED_URL, null); + if (url != null) { + if (url.endsWith("/")) { + LOG.error( + "The '{}' environment variable's value '{}' should be a valid URL without a slash ('/') as the last character.", + CONF_KEY_MY_EXPOSED_URL, url); + System.exit(1); + } + try { + URL exposedUrl = new URL(url); + + } catch (MalformedURLException e) { + LOG.error( + "The '{}' environment variable with value '{}' contains a malformed URL '{}'.", + CONF_KEY_MY_EXPOSED_URL, url, e.getMessage()); + System.exit(1); + } + } + // we want to make sure that this threadpool does not keep the JVM alive. So we // set the daemon to true. executorService = Executors.newScheduledThreadPool(12, new ThreadFactory() { From 2ddaebfd64da90d0f527c66af50a2cc73b19e6cc Mon Sep 17 00:00:00 2001 From: Barry Nouwt Date: Tue, 27 Feb 2024 14:13:07 +0100 Subject: [PATCH 3/3] Make sure KERs do not crash when other KER has invalid URL. Also: - improve logging. --- .../messaging/RemoteKerConnection.java | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/messaging/RemoteKerConnection.java b/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/messaging/RemoteKerConnection.java index 734e18358..c25d6e41c 100644 --- a/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/messaging/RemoteKerConnection.java +++ b/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/messaging/RemoteKerConnection.java @@ -138,11 +138,11 @@ private void updateRemoteKerDataFromPeer() { "Failed to receive runtimedetails from {}, got status code {}. Trying KER again in {} minutes.", this.remoteKerUri, response.statusCode(), waitTime); } - } catch (IOException | URISyntaxException | InterruptedException e) { + } catch (IOException | URISyntaxException | InterruptedException | IllegalArgumentException e) { this.remoteKerDetails = null; int waitTime = errorOccurred(); - LOG.warn("Failed to receive runtimedetails from " + this.remoteKerConnectionDetails.getId() - + ". Trying KER again in " + waitTime + " minutes."); + LOG.warn("Failed to receive runtimedetails from {}, got error '{}'. Trying KER again in {} minutes.", + this.remoteKerConnectionDetails.getId(), e.getMessage(), waitTime); LOG.debug("", e); } dispatcher.notifySmartConnectorsChanged(); @@ -235,8 +235,9 @@ public void stop() { LOG.warn("Failed to say goodbye to {}, got response {}: {}", this.remoteKerUri, response.statusCode(), response.body()); } - } catch (IOException | URISyntaxException | InterruptedException e) { - LOG.warn("Failed to say goodbye to " + remoteKerConnectionDetails.getId()); + } catch (IOException | URISyntaxException | InterruptedException | IllegalArgumentException e) { + LOG.warn("Failed to say goodbye to {}, get error '{}'", remoteKerConnectionDetails.getId(), + e.getMessage()); LOG.debug("", e); } } else @@ -275,13 +276,15 @@ public void sendToRemoteSmartConnector(KnowledgeMessage message) throws IOExcept throw new IOException("Message not accepted by remote host, status code " + response.statusCode() + ", body " + response.body()); } - } catch (JsonProcessingException | URISyntaxException | InterruptedException e) { + } catch (JsonProcessingException | URISyntaxException | InterruptedException | IllegalArgumentException e) { int time = this.errorOccurred(); - LOG.warn("Ignoring KER {} for {} minutes.", this.remoteKerUri, time); + LOG.warn("Ignoring KER {} for {} minutes. Error '{}' occurred.", this.remoteKerUri, time, + e.getMessage()); throw new IOException("Could not send message to remote SmartConnector.", e); } catch (IOException e) { int time = this.errorOccurred(); - LOG.warn("Ignoring KER {} for {} minutes.", this.remoteKerUri, time); + LOG.warn("Ignoring KER {} for {} minutes. Error '{}' occurred.", this.remoteKerUri, time, + e.getMessage()); throw e; } } else { @@ -308,11 +311,11 @@ public void sendMyKerDetailsToPeer(KnowledgeEngineRuntimeDetails details) { LOG.warn("Failed to send updated KnowledgeEngineRuntimeDetails to {}, got response {}: {}", this.remoteKerUri, response.statusCode(), response.body()); } - } catch (IOException | URISyntaxException | InterruptedException e) { + } catch (IOException | URISyntaxException | InterruptedException | IllegalArgumentException e) { this.remoteKerDetails = null; this.errorOccurred(); - LOG.warn("Failed to send updated KnowledgeEngineRuntimeDetails to " - + remoteKerConnectionDetails.getId()); + LOG.warn("Failed to send updated KnowledgeEngineRuntimeDetails to {}. Got error '{}'.", + remoteKerConnectionDetails.getId(), e.getMessage()); LOG.debug("", e); } } else