diff --git a/README.md b/README.md index 3c68f3686..15a415180 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ The rest of this README is structured as follows: - [(advanced) Administering a Knowledge Engine runtime](#advanced-administering-a-knowledge-engine-runtime) - [Starting the Knowledge Engine in local mode](#starting-the-knowledge-engine-in-local-mode) - [Starting the Knowledge Engine in distributed mode](#starting-the-knowledge-engine-in-distributed-mode) + - [Configuration](#configuration) # Demonstration videos and tutorials @@ -82,10 +83,10 @@ However, running it in the above way **does not support** data exchange with rem To interact with other runtimes, it needs additional configuration: - An additional port mapping for the socket that listens for communication from other runtimes. -- `KE_RUNTIME_EXPOSED_URL`: The URL via which the above socket is available for communication from the other runtime(s). You need to make sure the traffic is correctly routed to the container's port 8081 with a reverse proxy. -- `KD_URL`: The URL on which to find the knowledge directory (to discover peers). +- `ke.runtime.exposed.url`: The URL via which the above socket is available for communication from the other runtime(s). You need to make sure the traffic is correctly routed to the container's port 8081 with a reverse proxy. +- `kd.url`: The URL on which to find the knowledge directory (to discover peers). -The configuration can be set as follows: +The configuration can be set as follows (note that the configuration properties below use [underscores and capital letters](#configuration)) ```bash docker run \ @@ -258,12 +259,15 @@ nohup java -cp "smart-connector-rest-dist-1.2.5.jar:dependency/*" eu.knowledge.e ### Starting the Knowledge Engine in distributed mode The Knowledge Engine can also start in distributed mode, where it connects with a remote knowledge directory and where different instances of the Knowledge Engine (each instance hosting one or more smart connectors) can communicate with each other. More information about starting the Knowledge Engine in distributed mode can be found in the [documentation](docs/docs/distributed_mode.md). -### Additional configuration environment variables +### Configuration +TNO Knowledge Engine uses the [MicroProfile Config 3.1](https://microprofile.io/specifications/config/) specification to configure its behaviour and we use [SmallRye](https://smallrye.io/smallrye-config/) as the implementation of this specification. The default configuration values can be found in the [microprofile-config.properties](./smart-connector/src/main/resources/META-INF/microprofile-config.properties) configuration file. And, as described in the specification, these configuration values can be overridden by [environment variables and system properties](https://download.eclipse.org/microprofile/microprofile-config-3.1/microprofile-config-spec-3.1.html#default_configsources). Note that environment variables can use underscores and capital letters to adhere to their naming conventions and the MicroProfile Config automatically maps those to corresponding configuration properties using [specific rules](https://download.eclipse.org/microprofile/microprofile-config-3.1/microprofile-config-spec-3.1.html#default_configsources.env.mapping). + +A description of all configuration properties can be found in the [`SmartConnectorConfig`](./smart-connector/src/main/java/eu/knowledge/engine/smartconnector/impl/SmartConnectorConfig.java) class. The rest of this section highlights some of these configuration properties. *Increasing the wait time for other KBs to respond* -By default, a Smart Connector waits `10` seconds max for a reply from another Smart Connector when sending an ASK/POST message. This time is configurable via the `KE_KB_WAIT_TIMEOUT` environment variable and setting it to `0` means the Smart Connector will wait indefinitely (this can be useful when dealing with Human KBs). +By default, a Smart Connector waits `10` seconds max for a reply from another Smart Connector when sending an ASK/POST message. This time is configurable via the `ke.kb.wait.timeout` property and setting it to `0` means the Smart Connector will wait indefinitely (this can be useful when dealing with Human KBs). *Increasing the HTTP timeouts* -By default, a KER waits `5` seconds max for a HTTP response from another KER when sending a message via the inter-KER protocol. The time is configurable via the `KE_HTTP_TIMEOUT` environment variable. \ No newline at end of file +By default, a KER waits `5` seconds max for a HTTP connection response from another KER when sending a message via the inter-KER protocol. The time is configurable via the `ke.http.timeout` property \ No newline at end of file diff --git a/admin-ui/src/main/java/eu/knowledge/engine/admin/AdminUI.java b/admin-ui/src/main/java/eu/knowledge/engine/admin/AdminUI.java index ab8a3e556..078231dd6 100644 --- a/admin-ui/src/main/java/eu/knowledge/engine/admin/AdminUI.java +++ b/admin-ui/src/main/java/eu/knowledge/engine/admin/AdminUI.java @@ -16,6 +16,7 @@ import org.apache.jena.update.UpdateFactory; import org.apache.jena.update.UpdateRequest; import org.apache.jena.vocabulary.RDF; +import org.eclipse.microprofile.config.ConfigProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,8 +44,6 @@ public class AdminUI implements KnowledgeBase { private static final String META_GRAPH_PATTERN_STR = "?kb rdf:type ke:KnowledgeBase . ?kb ke:hasName ?name . ?kb ke:hasDescription ?description . ?kb ke:hasKnowledgeInteraction ?ki . ?ki rdf:type ?kiType . ?ki ke:isMeta ?isMeta . ?ki ke:hasCommunicativeAct ?act . ?act rdf:type ke:CommunicativeAct . ?act ke:hasRequirement ?req . ?act ke:hasSatisfaction ?sat . ?req rdf:type ?reqType . ?sat rdf:type ?satType . ?ki ke:hasGraphPattern ?gp . ?gp rdf:type ?patternType . ?gp ke:hasPattern ?pattern ."; - private static final String CONF_KEY_INITIAL_ADMIN_UI_DELAY = "INITIAL_ADMIN_UI_DELAY"; - private SmartConnector sc; private final PrefixMapping prefixes; @@ -141,7 +140,8 @@ public void smartConnectorReady(SmartConnector aSC) { // to receive the initial state, we do a single Ask (after sleeping for a // specific amount of time) try { - Thread.sleep(Integer.parseInt(AdminUI.getConfigProperty(CONF_KEY_INITIAL_ADMIN_UI_DELAY, "5000"))); + Thread.sleep( + ConfigProvider.getConfig().getValue(AdminUIConfig.CONF_KEY_INITIAL_ADMIN_UI_DELAY, Integer.class)); } catch (InterruptedException e) { LOG.info("{}", e); } @@ -184,7 +184,8 @@ public BindingSet handleChangedKnowledgeBaseKnowledge(ReactExchangeInfo ei) { // - insert the *new* data about that knowledge base Resource kb = model.listSubjectsWithProperty(RDF.type, Vocab.KNOWLEDGE_BASE).next(); - String query = String.format("DELETE { %s } WHERE { %s FILTER (?kb = <%s>) } ", this.metaGraphPattern.getPattern(), this.metaGraphPattern.getPattern(), kb.toString()); + String query = String.format("DELETE { %s } WHERE { %s FILTER (?kb = <%s>) } ", + this.metaGraphPattern.getPattern(), this.metaGraphPattern.getPattern(), kb.toString()); UpdateRequest updateRequest = UpdateFactory.create(query); UpdateAction.execute(updateRequest, this.model); @@ -215,7 +216,8 @@ public BindingSet handleRemovedKnowledgeBaseKnowledge(ReactExchangeInfo ei) { // - delete all old data about that knowledge base Resource kb = model.listSubjectsWithProperty(RDF.type, Vocab.KNOWLEDGE_BASE).next(); - String query = String.format("DELETE { %s } WHERE { %s FILTER (?kb = <%s>) } ", this.metaGraphPattern.getPattern(), this.metaGraphPattern.getPattern(), kb.toString()); + String query = String.format("DELETE { %s } WHERE { %s FILTER (?kb = <%s>) } ", + this.metaGraphPattern.getPattern(), this.metaGraphPattern.getPattern(), kb.toString()); UpdateRequest updateRequest = UpdateFactory.create(query); UpdateAction.execute(updateRequest, this.model); @@ -328,15 +330,4 @@ public void close() { public Model getModel() { return model; } - - public static String getConfigProperty(String key, String defaultValue) { - // We might replace this with something a bit more fancy in the future... - String value = System.getenv(key); - if (value == null) { - value = defaultValue; - LOG.info("No value for the configuration parameter '" + key + "' was provided, using the default value '" - + defaultValue + "'"); - } - return value; - } } diff --git a/admin-ui/src/main/java/eu/knowledge/engine/admin/AdminUIConfig.java b/admin-ui/src/main/java/eu/knowledge/engine/admin/AdminUIConfig.java new file mode 100644 index 000000000..dfd0e2303 --- /dev/null +++ b/admin-ui/src/main/java/eu/knowledge/engine/admin/AdminUIConfig.java @@ -0,0 +1,13 @@ +package eu.knowledge.engine.admin; + +public class AdminUIConfig { + + /** + * The key to configure how long (in milliseconds) should the Admin UI wait + * until it tries to ask for all KBs in the network. This value should probably + * be higher in distributed mode, to allow the participants to reach equilibrium + * with respect to knowledge about each other. + */ + public static final String CONF_KEY_INITIAL_ADMIN_UI_DELAY = "initial.admin.ui.delay"; + +} diff --git a/admin-ui/src/main/resources/META-INF/microprofile-config.properties b/admin-ui/src/main/resources/META-INF/microprofile-config.properties new file mode 100644 index 000000000..01929b25d --- /dev/null +++ b/admin-ui/src/main/resources/META-INF/microprofile-config.properties @@ -0,0 +1 @@ +initial.admin.ui.delay = 5000 \ No newline at end of file diff --git a/admin-ui/src/main/resources/openapi-admin-ui.yaml b/admin-ui/src/main/resources/openapi-admin-ui.yaml index d23d291db..59bca4313 100644 --- a/admin-ui/src/main/resources/openapi-admin-ui.yaml +++ b/admin-ui/src/main/resources/openapi-admin-ui.yaml @@ -36,6 +36,17 @@ paths: text/plain; charset=UTF-8: schema: type: string + /rest/admin/reload: + get: + summary: Manually reload the admin-ui's smart connectors within the network. This is sometimes necessary when the initial load did not pick up all SCs correctly. + tags: + - admin API + operationId: reloadSCs + responses: + '200': + description: If the SC were reloaded. + '500': + description: If a problem occurred. components: schemas: diff --git a/smart-connector/pom.xml b/smart-connector/pom.xml index 0668f7a8b..c48b3056b 100644 --- a/smart-connector/pom.xml +++ b/smart-connector/pom.xml @@ -47,6 +47,25 @@ 5.11.3 test + + org.mockito + mockito-core + 5.14.2 + test + + + org.mockito + mockito-junit-jupiter + 5.14.2 + test + + + org.wiremock + wiremock + 3.10.0 + test + + @@ -189,6 +208,16 @@ + + org.eclipse.microprofile.config + microprofile-config-api + ${version.eclipse.microprofile.config} + + + io.smallrye.config + smallrye-config + 3.10.1 + @@ -197,6 +226,7 @@ 3.1.9 2.18.2 6.1.0 + 3.1 diff --git a/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/impl/InteractionProcessorImpl.java b/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/impl/InteractionProcessorImpl.java index d6ca242e8..f48dde3a7 100644 --- a/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/impl/InteractionProcessorImpl.java +++ b/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/impl/InteractionProcessorImpl.java @@ -28,6 +28,7 @@ import org.apache.jena.sparql.syntax.ElementData; import org.apache.jena.sparql.syntax.ElementGroup; import org.apache.jena.vocabulary.RDF; +import org.eclipse.microprofile.config.ConfigProvider; import org.slf4j.Logger; import eu.knowledge.engine.reasoner.Rule; @@ -76,8 +77,6 @@ public class InteractionProcessorImpl implements InteractionProcessor { */ private boolean reasonerEnabled = false; - private static boolean VALIDATE_OUTGOING_BINDINGS_WRT_INCOMING_BINDINGS_DEFAULT = true; - private static final Query query = QueryFactory.create( "ASK WHERE { ?req ?someClass . FILTER NOT EXISTS {?sat ?someClass .} VALUES (?req ?sat) {} }"); @@ -388,9 +387,8 @@ public CompletableFuture processPostFromMessageRouter(PostMessage } private boolean shouldValidateInputOutputBindings() { - return SmartConnectorConfig.getBoolean( - SmartConnectorConfig.CONF_KEY_VALIDATE_OUTGOING_BINDINGS_WRT_INCOMING_BINDINGS, - VALIDATE_OUTGOING_BINDINGS_WRT_INCOMING_BINDINGS_DEFAULT); + return ConfigProvider.getConfig().getValue( + SmartConnectorConfig.CONF_KEY_VALIDATE_OUTGOING_BINDINGS_WRT_INCOMING_BINDINGS, Boolean.class); } @Override diff --git a/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/impl/MessageRouterImpl.java b/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/impl/MessageRouterImpl.java index e4114a92d..5422e5b02 100644 --- a/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/impl/MessageRouterImpl.java +++ b/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/impl/MessageRouterImpl.java @@ -10,6 +10,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import org.eclipse.microprofile.config.ConfigProvider; import org.slf4j.Logger; import eu.knowledge.engine.smartconnector.messaging.AnswerMessage; @@ -30,14 +31,6 @@ public class MessageRouterImpl implements MessageRouter, SmartConnectorEndpoint */ private static final int MAX_ENTRIES = 5000; - /** - * How many seconds should the MessageRouter wait for ANSWER/REACT Message when - * sending a ASK/POST Message? 0 means wait forever (useful when working with a - * human KB) - */ - private static final String CONF_KEY_WAIT_TIMEOUT = "KE_KB_WAIT_TIMEOUT"; - private static final int DEFAULT_WAIT_TIMEOUT = 10; - private final SmartConnectorImpl smartConnector; private final Map> openAskMessages = Collections .synchronizedMap(new LinkedHashMap>() { @@ -87,7 +80,7 @@ public MessageRouterImpl(SmartConnectorImpl smartConnector) { } private int getWaitTimeout() { - return Integer.parseInt(this.getConfigProperty(CONF_KEY_WAIT_TIMEOUT, Integer.toString(DEFAULT_WAIT_TIMEOUT))); + return ConfigProvider.getConfig().getValue(SmartConnectorConfig.CONF_KEY_KE_KB_WAIT_TIMEOUT, Integer.class); } @Override @@ -286,21 +279,6 @@ public URI getKnowledgeBaseId() { return this.smartConnector.getKnowledgeBaseId(); } - public String getConfigProperty(String key, String defaultValue) { - // We might replace this with something a bit more fancy in the future... - String value = System.getenv(key); - if (value == null) { - value = defaultValue; - LOG.trace("No value for the configuration parameter '{}' was provided, using the default value '{}'", key, - defaultValue); - } - return value; - } - - public boolean hasConfigProperty(String key) { - return System.getenv(key) != null; - } - @Override public void setMessageDispatcher(MessageDispatcherEndpoint messageDispatcherEndpoint) { assert this.messageDispatcherEndpoint == null; diff --git a/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/impl/SmartConnectorConfig.java b/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/impl/SmartConnectorConfig.java index a261960c8..7f18b2a2e 100644 --- a/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/impl/SmartConnectorConfig.java +++ b/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/impl/SmartConnectorConfig.java @@ -1,14 +1,53 @@ package eu.knowledge.engine.smartconnector.impl; public class SmartConnectorConfig { - public static final String CONF_KEY_VALIDATE_OUTGOING_BINDINGS_WRT_INCOMING_BINDINGS = "SC_VALIDATE_OUTGOING_BINDINGS_WRT_INCOMING_BINDINGS"; - - public static boolean getBoolean(String key, boolean defaultValue) { - String valueString = System.getenv(key); - if (valueString == null) { - return defaultValue; - } else { - return Boolean.parseBoolean(valueString); - } - } + + /** + * Key to set whether this KER should strictly validate whether outgoing + * bindings are compatible with incoming bindings. + */ + public static final String CONF_KEY_VALIDATE_OUTGOING_BINDINGS_WRT_INCOMING_BINDINGS = "sc.validate.outgoing.bindings.wrt.incoming.bindings"; + + /** + * Key to configure the hostname of the machine this Knowledge Engine Runtime + * (KER) runs on. + * + * @deprecated Replaced by + * {@link SmartConnectorConfig#CONF_KEY_KE_RUNTIME_EXPOSED_URL} + */ + @Deprecated + public static final String CONF_KEY_KE_RUNTIME_HOSTNAME = "ke.runtime.hostname"; + + /** + * Key to configure the URL of the Knowledge Directory where this KER can find + * other KERs in the network. Note that overriding this configuration property + * will run this KER in distributed mode. + */ + public static final String CONF_KEY_KD_URL = "kd.url"; + + /** + * Key to configure the time in seconds the SCs in this KER wait for a HTTP + * connection response from another KER. Only used in distributed mode. + */ + public static final String CONF_KEY_KE_HTTP_TIMEOUT = "ke.http.timeout"; + + /** + * Key to configure the how many seconds the MessageRouter should wait for + * ANSWER/REACT Message when sending a ASK/POST Message? 0 means wait forever + * (useful when working with a human KB). + */ + public static final String CONF_KEY_KE_KB_WAIT_TIMEOUT = "ke.kb.wait.timeout"; + + /** + * Key to configure the URL that is advertised to other KERs. Other KERs can use + * this URL to reach this KER. Note that this configuration property is only + * used in distributed mode. + */ + public static final String CONF_KEY_KE_RUNTIME_EXPOSED_URL = "ke.runtime.exposed.url"; + + /** + * Key to configure the port at which the KER's peer-to-peer communication + * should happen. Only used in distributed mode. + */ + public static final String CONF_KEY_KE_RUNTIME_PORT = "ke.runtime.port"; } diff --git a/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/KeRuntime.java b/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/KeRuntime.java index b308f4ec9..1eec0ea60 100644 --- a/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/KeRuntime.java +++ b/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/KeRuntime.java @@ -8,9 +8,13 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; +import org.eclipse.microprofile.config.Config; +import org.eclipse.microprofile.config.ConfigProvider; +import org.eclipse.microprofile.config.ConfigValue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import eu.knowledge.engine.smartconnector.impl.SmartConnectorConfig; import eu.knowledge.engine.smartconnector.runtime.messaging.MessageDispatcher; /** @@ -20,14 +24,6 @@ * component of the Knowledge Engine. */ public class KeRuntime { - - private static final String CONF_KEY_MY_HOSTNAME = "KE_RUNTIME_HOSTNAME"; - private static final String CONF_KEY_MY_PORT = "KE_RUNTIME_PORT"; - private static final String CONF_KEY_KD_URL = "KD_URL"; - private static final String CONF_KEY_MY_EXPOSED_URL = "KE_RUNTIME_EXPOSED_URL"; - - private static final String EXPOSED_URL_DEFAULT_PROTOCOL = "http"; - private static final Logger LOG = LoggerFactory.getLogger(KeRuntime.class); private static LocalSmartConnectorRegistry localSmartConnectorRegistry = new LocalSmartConnectorRegistryImpl(); @@ -35,31 +31,38 @@ public class KeRuntime { private static MessageDispatcher messageDispatcher = null; static { - if (hasConfigProperty(CONF_KEY_MY_EXPOSED_URL) && hasConfigProperty(CONF_KEY_MY_HOSTNAME)) { - LOG.error("KE runtime must be configured with {} or {}, not both.", CONF_KEY_MY_EXPOSED_URL, - CONF_KEY_MY_HOSTNAME); + + Config config = ConfigProvider.getConfig(); + ConfigValue exposedUrl = config.getConfigValue(SmartConnectorConfig.CONF_KEY_KE_RUNTIME_EXPOSED_URL); + ConfigValue hostname = config.getConfigValue(SmartConnectorConfig.CONF_KEY_KE_RUNTIME_HOSTNAME); + + // Using MicroProfile Config's source ordinal to determine if default + // configuration got overridden? + if (exposedUrl.getSourceOrdinal() > 100 && hostname.getSourceOrdinal() > 100) { + LOG.error("KE runtime must be configured with {} or {}, not both.", + SmartConnectorConfig.CONF_KEY_KE_RUNTIME_EXPOSED_URL, + SmartConnectorConfig.CONF_KEY_KE_RUNTIME_HOSTNAME); LOG.info("Using {} allows the use of a reverse proxy for TLS connections, which is recommended.", - CONF_KEY_MY_EXPOSED_URL); + SmartConnectorConfig.CONF_KEY_KE_RUNTIME_EXPOSED_URL); System.exit(1); } // execute some validation on the EXPOSED URL, because it can have severe // consequences - String url = getConfigProperty(CONF_KEY_MY_EXPOSED_URL, null); - if (url != null) { + if (exposedUrl.getSourceOrdinal() > 100) { + String url = exposedUrl.getValue(); if (url.endsWith("/")) { LOG.error( "The '{}' environment variable's value '{}' should be a valid URL without a slash ('/') as the last character.", - CONF_KEY_MY_EXPOSED_URL, url); + SmartConnectorConfig.CONF_KEY_KE_RUNTIME_EXPOSED_URL, url); System.exit(1); } try { - URL exposedUrl = new URL(url); + new URL(url); } catch (MalformedURLException e) { - LOG.error( - "The '{}' environment variable with value '{}' contains a malformed URL '{}'.", - CONF_KEY_MY_EXPOSED_URL, url, e.getMessage()); + LOG.error("The '{}' environment variable with value '{}' contains a malformed URL '{}'.", + SmartConnectorConfig.CONF_KEY_KE_RUNTIME_EXPOSED_URL, url, e.getMessage()); System.exit(1); } } @@ -96,27 +99,25 @@ public static ScheduledExecutorService executorService() { } public static MessageDispatcher getMessageDispatcher() { + + Config config = ConfigProvider.getConfig(); + if (messageDispatcher == null) { try { - if (!hasConfigProperty(CONF_KEY_KD_URL)) { + ConfigValue kdUrl = config.getConfigValue(SmartConnectorConfig.CONF_KEY_KD_URL); + if (kdUrl.getSourceOrdinal() == 100) { LOG.warn( "No configuration provided for Knowledge Directory, starting Knowledge Engine in local mode"); messageDispatcher = new MessageDispatcher(); } else { - var myHostname = getConfigProperty(CONF_KEY_MY_HOSTNAME, "localhost"); - var myPort = Integer.parseInt(getConfigProperty(CONF_KEY_MY_PORT, "8081")); - - URI myExposedUrl; - if (hasConfigProperty(CONF_KEY_MY_EXPOSED_URL)) { - myExposedUrl = new URI(getConfigProperty(CONF_KEY_MY_EXPOSED_URL, null)); - } else { - // If no exposed URL config is given we build one based on the - // configured host and port. - myExposedUrl = new URI(EXPOSED_URL_DEFAULT_PROTOCOL + "://" + myHostname + ":" + myPort); - } - - messageDispatcher = new MessageDispatcher(myPort, myExposedUrl, - new URI(getConfigProperty(CONF_KEY_KD_URL, "http://localhost:8080"))); + ConfigValue port = config.getConfigValue(SmartConnectorConfig.CONF_KEY_KE_RUNTIME_PORT); + var myPort = Integer.parseInt(port.getValue()); + + ConfigValue exposedUrl = config + .getConfigValue(SmartConnectorConfig.CONF_KEY_KE_RUNTIME_EXPOSED_URL); + URI myExposedUrl = new URI(exposedUrl.getValue()); + + messageDispatcher = new MessageDispatcher(myPort, myExposedUrl, new URI(kdUrl.getValue())); } } catch (NumberFormatException | URISyntaxException e) { LOG.error("Could not parse configuration properties, cannot start Knowledge Engine", e); @@ -131,20 +132,4 @@ public static MessageDispatcher getMessageDispatcher() { } return messageDispatcher; } - - public static String getConfigProperty(String key, String defaultValue) { - // We might replace this with something a bit more fancy in the future... - String value = System.getenv(key); - if (value == null) { - value = defaultValue; - LOG.info("No value for the configuration parameter '" + key + "' was provided, using the default value '" - + defaultValue + "'"); - } - return value; - } - - public static boolean hasConfigProperty(String key) { - return System.getenv(key) != null; - } - } diff --git a/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/messaging/RemoteKerConnection.java b/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/messaging/RemoteKerConnection.java index eb2c0b393..a8da339bc 100644 --- a/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/messaging/RemoteKerConnection.java +++ b/smart-connector/src/main/java/eu/knowledge/engine/smartconnector/runtime/messaging/RemoteKerConnection.java @@ -3,7 +3,11 @@ import static eu.knowledge.engine.smartconnector.runtime.messaging.Utils.stripUserInfoFromURI; import java.io.IOException; -import java.net.*; +import java.net.Authenticator; +import java.net.PasswordAuthentication; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpRequest.BodyPublishers; @@ -15,6 +19,7 @@ import java.util.ArrayList; import java.util.List; +import org.eclipse.microprofile.config.ConfigProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,6 +27,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; +import eu.knowledge.engine.smartconnector.impl.SmartConnectorConfig; import eu.knowledge.engine.smartconnector.messaging.AnswerMessage; import eu.knowledge.engine.smartconnector.messaging.AskMessage; import eu.knowledge.engine.smartconnector.messaging.ErrorMessage; @@ -39,13 +45,6 @@ */ public class RemoteKerConnection { - /** - * How many seconds the HttpClient waits for a HTTP response when sending a HTTP - * request. Default 5 seconds. - */ - private static final String CONF_KEY_HTTP_TIMEOUT = "KE_HTTP_TIMEOUT"; - private static final int DEFAULT_HTTP_TIMEOUT = 5; - public static final Logger LOG = LoggerFactory.getLogger(RemoteKerConnection.class); private final KnowledgeEngineRuntimeConnectionDetails remoteKerConnectionDetails; @@ -98,7 +97,7 @@ protected PasswordAuthentication getPasswordAuthentication() { } private int getHttpTimeout() { - return Integer.parseInt(this.getConfigProperty(CONF_KEY_HTTP_TIMEOUT, Integer.toString(DEFAULT_HTTP_TIMEOUT))); + return ConfigProvider.getConfig().getValue(SmartConnectorConfig.CONF_KEY_KE_HTTP_TIMEOUT, Integer.class); } public URI getRemoteKerUri() { @@ -230,9 +229,9 @@ public void start() { public void stop() { if (this.isAvailable()) { try { - String ker_id = URLEncoder.encode(dispatcher.getMyKnowledgeEngineRuntimeDetails().getRuntimeId(), StandardCharsets.UTF_8); - HttpRequest request = HttpRequest - .newBuilder(new URI(this.remoteKerUri + "/runtimedetails/" + ker_id)) + String ker_id = URLEncoder.encode(dispatcher.getMyKnowledgeEngineRuntimeDetails().getRuntimeId(), + StandardCharsets.UTF_8); + HttpRequest request = HttpRequest.newBuilder(new URI(this.remoteKerUri + "/runtimedetails/" + ker_id)) .header("Content-Type", "application/json").DELETE().build(); HttpResponse response = this.httpClient.send(request, BodyHandlers.ofString()); @@ -357,20 +356,4 @@ private String getPathForMessageType(KnowledgeMessage message) { return null; } } - - public String getConfigProperty(String key, String defaultValue) { - // We might replace this with something a bit more fancy in the future... - String value = System.getenv(key); - if (value == null) { - value = defaultValue; - LOG.trace("No value for the configuration parameter '{}' was provided, using the default value '{}'", key, - defaultValue); - } - return value; - } - - public boolean hasConfigProperty(String key) { - return System.getenv(key) != null; - } - } diff --git a/smart-connector/src/main/resources/META-INF/microprofile-config.properties b/smart-connector/src/main/resources/META-INF/microprofile-config.properties new file mode 100644 index 000000000..ecb1cea5d --- /dev/null +++ b/smart-connector/src/main/resources/META-INF/microprofile-config.properties @@ -0,0 +1,8 @@ +# documentation about these configuration property can be found in the SmartConnectorConfig class +ke.runtime.port = 8081 +ke.runtime.exposed.url = http://${ke.runtime.hostname}:${ke.runtime.port} +ke.kb.wait.timeout = 10 +ke.http.timeout = 5 +kd.url = http://localhost:8080 +sc.validate.outgoing.bindings.wrt.incoming.bindings = true +ke.runtime.hostname = localhost \ No newline at end of file diff --git a/smart-connector/src/test/java/eu/knowledge/engine/smartconnector/misc/ConfigurationTest.java b/smart-connector/src/test/java/eu/knowledge/engine/smartconnector/misc/ConfigurationTest.java new file mode 100644 index 000000000..1d197b442 --- /dev/null +++ b/smart-connector/src/test/java/eu/knowledge/engine/smartconnector/misc/ConfigurationTest.java @@ -0,0 +1,212 @@ +package eu.knowledge.engine.smartconnector.misc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.util.concurrent.ExecutionException; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.knowledge.engine.smartconnector.api.AnswerExchangeInfo; +import eu.knowledge.engine.smartconnector.api.AnswerKnowledgeInteraction; +import eu.knowledge.engine.smartconnector.api.AskKnowledgeInteraction; +import eu.knowledge.engine.smartconnector.api.AskResult; +import eu.knowledge.engine.smartconnector.api.Binding; +import eu.knowledge.engine.smartconnector.api.BindingSet; +import eu.knowledge.engine.smartconnector.api.CommunicativeAct; +import eu.knowledge.engine.smartconnector.api.ExchangeInfo; +import eu.knowledge.engine.smartconnector.api.GraphPattern; +import eu.knowledge.engine.smartconnector.impl.SmartConnectorConfig; +import eu.knowledge.engine.smartconnector.util.KnowledgeNetwork; +import eu.knowledge.engine.smartconnector.util.MockedKnowledgeBase; + +public class ConfigurationTest { + + private static final Logger LOG = LoggerFactory.getLogger(ConfigurationTest.class); + + private KnowledgeNetwork kn; + private MockedKnowledgeBase kb1; + private AskKnowledgeInteraction askKI; + private MockedKnowledgeBase kb2; + private AnswerKnowledgeInteraction answerKI; + private int waitTimeout = 0; + + @BeforeEach + public void beforeTest() { + this.kn = new KnowledgeNetwork(); + + intializeKB1(); + intializeKB2(); + kn.addKB(kb1); + kn.addKB(kb2); + kn.sync(); + } + + @Test + public void testConfigValidateTrue() { + System.setProperty(SmartConnectorConfig.CONF_KEY_VALIDATE_OUTGOING_BINDINGS_WRT_INCOMING_BINDINGS, "true"); + BindingSet bs = new BindingSet(); + + var bs1 = new Binding(); + bs1.put("s", ""); + bs.add(bs1); + + var future = kb1.ask(this.askKI, bs); + + AskResult askResult = null; + try { + askResult = future.get(); + assertTrue(askResult.getExchangeInfoPerKnowledgeBase().iterator().hasNext()); + var info = askResult.getExchangeInfoPerKnowledgeBase().iterator().next(); + + assertEquals(ExchangeInfo.Status.FAILED, info.getStatus()); + assertTrue(info.getFailedMessage().contains("java.lang.IllegalArgumentException")); + + } catch (InterruptedException | ExecutionException e) { + LOG.info("{}", e); + } + + LOG.info("Result: {}", askResult); + + System.clearProperty(SmartConnectorConfig.CONF_KEY_VALIDATE_OUTGOING_BINDINGS_WRT_INCOMING_BINDINGS); + } + + @Test + public void testConfigValidateFalse() { + System.setProperty(SmartConnectorConfig.CONF_KEY_VALIDATE_OUTGOING_BINDINGS_WRT_INCOMING_BINDINGS, "false"); + + BindingSet bs = new BindingSet(); + + var bs1 = new Binding(); + bs1.put("s", ""); + bs.add(bs1); + + var future = kb1.ask(this.askKI, bs); + + AskResult askResult = null; + try { + askResult = future.get(); + assertTrue(askResult.getExchangeInfoPerKnowledgeBase().iterator().hasNext()); + var info = askResult.getExchangeInfoPerKnowledgeBase().iterator().next(); + + assertEquals(ExchangeInfo.Status.SUCCEEDED, info.getStatus()); + assertEquals(null, info.getFailedMessage()); + + } catch (InterruptedException | ExecutionException e) { + LOG.info("{}", e); + } + + LOG.info("Result: {}", askResult); + System.clearProperty(SmartConnectorConfig.CONF_KEY_VALIDATE_OUTGOING_BINDINGS_WRT_INCOMING_BINDINGS); + } + + @Test + public void testConfigWaitForKnowledgeBaseNegative() { + System.setProperty(SmartConnectorConfig.CONF_KEY_KE_KB_WAIT_TIMEOUT, "1"); + waitTimeout = 2000; + + BindingSet bs = new BindingSet(); + + var bs1 = new Binding(); + bs1.put("s", ""); + bs.add(bs1); + + var future = kb1.ask(this.askKI, bs); + + AskResult askResult = null; + try { + askResult = future.get(); + assertTrue(askResult.getExchangeInfoPerKnowledgeBase().iterator().hasNext()); + var info = askResult.getExchangeInfoPerKnowledgeBase().iterator().next(); + + assertEquals(ExchangeInfo.Status.FAILED, info.getStatus()); + assertTrue(info.getFailedMessage() != null); + assertTrue(info.getFailedMessage().contains("TimeoutException")); + + } catch (InterruptedException | ExecutionException e) { + LOG.info("{}", e); + } + + LOG.info("Result: {}", askResult); + waitTimeout = 0; + System.clearProperty(SmartConnectorConfig.CONF_KEY_KE_KB_WAIT_TIMEOUT); + } + + @Test + public void testConfigWaitForKnowledgeBasePositive() { + System.setProperty(SmartConnectorConfig.CONF_KEY_KE_KB_WAIT_TIMEOUT, "2"); + waitTimeout = 0; + BindingSet bs = new BindingSet(); + + var bs1 = new Binding(); + bs1.put("s", ""); + bs.add(bs1); + + var future = kb1.ask(this.askKI, bs); + + AskResult askResult = null; + try { + askResult = future.get(); + assertTrue(askResult.getExchangeInfoPerKnowledgeBase().iterator().hasNext()); + var info = askResult.getExchangeInfoPerKnowledgeBase().iterator().next(); + + assertEquals(ExchangeInfo.Status.SUCCEEDED, info.getStatus()); + assertEquals(null, info.getFailedMessage()); + + } catch (InterruptedException | ExecutionException e) { + LOG.info("{}", e); + } + + LOG.info("Result: {}", askResult); + System.clearProperty(SmartConnectorConfig.CONF_KEY_KE_KB_WAIT_TIMEOUT); + } + + @AfterEach + public void afterTest() { + try { + kn.stop().get(); + } catch (InterruptedException | ExecutionException e) { + fail(); + } + } + + private void intializeKB1() { + this.kb1 = new MockedKnowledgeBase("kb1"); + GraphPattern gp1 = new GraphPattern(""" + ?s a . + ?s ?n . + """); + this.askKI = new AskKnowledgeInteraction(new CommunicativeAct(), gp1); + this.kb1.register(askKI); + } + + private void intializeKB2() { + this.kb2 = new MockedKnowledgeBase("kb2"); + GraphPattern gp1 = new GraphPattern(""" + ?p a . + ?p ?name . + """); + this.answerKI = new AnswerKnowledgeInteraction(new CommunicativeAct(), gp1); + this.kb2.register(answerKI, (AnswerKnowledgeInteraction ki, AnswerExchangeInfo exchangeInfo) -> { + var bs = new BindingSet(); + var b = new Binding(); + b.put("p", ""); + b.put("name", "\"Barry Nouwt\""); + bs.add(b); + + try { + Thread.sleep(waitTimeout); + } catch (InterruptedException e) { + fail(); + } + + return bs; + }); + } + +} diff --git a/smart-connector/src/test/java/eu/knowledge/engine/smartconnector/misc/WireMockFirstConfigurationTest.java b/smart-connector/src/test/java/eu/knowledge/engine/smartconnector/misc/WireMockFirstConfigurationTest.java new file mode 100644 index 000000000..ceab06b27 --- /dev/null +++ b/smart-connector/src/test/java/eu/knowledge/engine/smartconnector/misc/WireMockFirstConfigurationTest.java @@ -0,0 +1,88 @@ +package eu.knowledge.engine.smartconnector.misc; + +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.matching; +import static com.github.tomakehurst.wiremock.client.WireMock.matchingJsonPath; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.status; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.mock; + +import java.io.IOException; +import java.net.URI; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; + +import eu.knowledge.engine.smartconnector.impl.SmartConnectorConfig; +import eu.knowledge.engine.smartconnector.runtime.KeRuntime; +import eu.knowledge.engine.smartconnector.runtime.messaging.MessageDispatcher; +import eu.knowledge.engine.smartconnector.runtime.messaging.RemoteKerConnection; +import eu.knowledge.engine.smartconnector.runtime.messaging.kd.model.KnowledgeEngineRuntimeConnectionDetails; + +@WireMockTest +public class WireMockFirstConfigurationTest { + + private static Logger LOG = LoggerFactory.getLogger(WireMockFirstConfigurationTest.class); + + @BeforeAll + public static void before(WireMockRuntimeInfo wmRuntimeInfo) { + + // set all system property for various tests below. + String host = wmRuntimeInfo.getHttpBaseUrl(); + System.setProperty(SmartConnectorConfig.CONF_KEY_KD_URL, host); + System.setProperty(SmartConnectorConfig.CONF_KEY_KE_RUNTIME_PORT, "1234"); + String value = "http://test${ke.runtime.hostname}:${ke.runtime.port}"; + System.setProperty(SmartConnectorConfig.CONF_KEY_KE_RUNTIME_EXPOSED_URL, value); + + LOG.info("Testing with exposed url: {}", value); + } + + /** + * Does a (very limited) test of the http timeout configuration option. We are + * setting it to 1 second and if this works, the test should succeed within 2 + * seconds. If setting the configuration property fails, the test would take 5 + * seconds (= default value for http timeout configuration property). + * + * @throws Exception + * + * @throws IOException + * @throws InterruptedException + */ + @Test + @Timeout(value = 3, unit = TimeUnit.SECONDS) + public void testConfigHttpConnectTimeout() throws Exception { + stubFor(post("/ker/").willReturn(status(201).withBody("{}"))); + System.setProperty(SmartConnectorConfig.CONF_KEY_KE_HTTP_TIMEOUT, "1"); + + MessageDispatcher messageDispatcher = mock(MessageDispatcher.class); + + var ker = new RemoteKerConnection(messageDispatcher, + new KnowledgeEngineRuntimeConnectionDetails().exposedUrl(URI.create("http://10.255.255.1/"))); + + ker.start(); + assertFalse(ker.isAvailable()); + System.clearProperty(SmartConnectorConfig.CONF_KEY_KE_HTTP_TIMEOUT); + } + + @AfterAll + public static void after() { + System.clearProperty(SmartConnectorConfig.CONF_KEY_KD_URL); + System.clearProperty(SmartConnectorConfig.CONF_KEY_KE_RUNTIME_PORT); + System.clearProperty(SmartConnectorConfig.CONF_KEY_KE_RUNTIME_EXPOSED_URL); + } + +}