From 09c089898707e76b20f417fed58d27c0cabb0b95 Mon Sep 17 00:00:00 2001 From: George Wu Date: Tue, 20 Aug 2024 15:50:18 -0400 Subject: [PATCH 1/6] update docs for kafka lookup extension to specify correct extension ordering --- docs/querying/kafka-extraction-namespace.md | 3 +++ docs/querying/lookups-cached-global.md | 3 +++ 2 files changed, 6 insertions(+) diff --git a/docs/querying/kafka-extraction-namespace.md b/docs/querying/kafka-extraction-namespace.md index 1cfa91aac554..b8fdbf9c9b87 100644 --- a/docs/querying/kafka-extraction-namespace.md +++ b/docs/querying/kafka-extraction-namespace.md @@ -24,6 +24,9 @@ title: "Apache Kafka Lookups" To use this Apache Druid extension, [include](../configuration/extensions.md#loading-extensions) `druid-lookups-cached-global` and `druid-kafka-extraction-namespace` in the extensions load list. +This extension provides all the functionality of `druid-lookups-cached-global`. You should only load one of the two extensions. +(load `druid-lookups-cached-global` if you only want global cached lookup functionality, load `druid-kafka-extraction-namespace` if you also want kafka lookup functionality) + If you need updates to populate as promptly as possible, it is possible to plug into a Kafka topic whose key is the old value and message is the desired new value (both in UTF-8) as a LookupExtractorFactory. ```json diff --git a/docs/querying/lookups-cached-global.md b/docs/querying/lookups-cached-global.md index 72c4189c2dad..8db1a4333d61 100644 --- a/docs/querying/lookups-cached-global.md +++ b/docs/querying/lookups-cached-global.md @@ -24,6 +24,9 @@ title: "Globally Cached Lookups" To use this Apache Druid extension, [include](../configuration/extensions.md#loading-extensions) `druid-lookups-cached-global` in the extensions load list. +The `druid-kafka-extraction-namespace` extension provides all the functionality of `druid-lookups-cached-global`. You should only load one of the two extensions. +(load `druid-lookups-cached-global` if you only want global cached lookup functionality, load `druid-kafka-extraction-namespace` if you also want kafka lookup functionality) + ## Configuration :::info Static configuration is no longer supported. Lookups can be configured through From cfa23fa5d4e4541ceaf65d7b687416d743d98538 Mon Sep 17 00:00:00 2001 From: George Wu Date: Tue, 20 Aug 2024 15:51:35 -0400 Subject: [PATCH 2/6] fix first line --- docs/querying/kafka-extraction-namespace.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/querying/kafka-extraction-namespace.md b/docs/querying/kafka-extraction-namespace.md index b8fdbf9c9b87..e7d02babdd66 100644 --- a/docs/querying/kafka-extraction-namespace.md +++ b/docs/querying/kafka-extraction-namespace.md @@ -22,7 +22,7 @@ title: "Apache Kafka Lookups" ~ under the License. --> -To use this Apache Druid extension, [include](../configuration/extensions.md#loading-extensions) `druid-lookups-cached-global` and `druid-kafka-extraction-namespace` in the extensions load list. +To use this Apache Druid extension, [include](../configuration/extensions.md#loading-extensions) `druid-kafka-extraction-namespace` in the extensions load list. This extension provides all the functionality of `druid-lookups-cached-global`. You should only load one of the two extensions. (load `druid-lookups-cached-global` if you only want global cached lookup functionality, load `druid-kafka-extraction-namespace` if you also want kafka lookup functionality) From 2ccaf611efdbf5e3256a0862671664c524ebb6d4 Mon Sep 17 00:00:00 2001 From: George Wu Date: Thu, 22 Aug 2024 12:39:50 -0400 Subject: [PATCH 3/6] test with extension dependencies --- .../kafka-extraction-namespace/pom.xml | 1 + .../guice/ExtensionFirstClassLoader.java | 56 +++++++++++++++++-- .../apache/druid/guice/ExtensionsLoader.java | 36 +++++++++++- 3 files changed, 84 insertions(+), 9 deletions(-) diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index d3cf5d3216fd..a23bd115da8f 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -44,6 +44,7 @@ org.apache.druid.extensions druid-lookups-cached-global ${project.parent.version} + provided org.apache.druid diff --git a/processing/src/main/java/org/apache/druid/guice/ExtensionFirstClassLoader.java b/processing/src/main/java/org/apache/druid/guice/ExtensionFirstClassLoader.java index 1a2944b2bdb0..0ffc8fdbeb75 100644 --- a/processing/src/main/java/org/apache/druid/guice/ExtensionFirstClassLoader.java +++ b/processing/src/main/java/org/apache/druid/guice/ExtensionFirstClassLoader.java @@ -35,11 +35,13 @@ public class ExtensionFirstClassLoader extends URLClassLoader { private final ClassLoader druidLoader; + private List extensionDependencyClassLoaders; - public ExtensionFirstClassLoader(final URL[] urls, final ClassLoader druidLoader) + public ExtensionFirstClassLoader(final URL[] urls, final ClassLoader druidLoader, final List extensionDependencyClassLoaders) { super(urls, null); this.druidLoader = Preconditions.checkNotNull(druidLoader, "druidLoader"); + this.extensionDependencyClassLoaders = Preconditions.checkNotNull(extensionDependencyClassLoaders, "extensionDependencyClassLoaders"); } @Override @@ -60,8 +62,13 @@ protected Class loadClass(final String name, final boolean resolve) throws Cl clazz = findClass(name); } catch (ClassNotFoundException e) { - // Try the Druid classloader. Will throw ClassNotFoundException if the class can't be loaded. - return druidLoader.loadClass(name); + try { + return loadClassFromExtensionDependencies(name); + } + catch (ClassNotFoundException e2) { + // Try the Druid classloader. Will throw ClassNotFoundException if the class can't be loaded. + return druidLoader.loadClass(name); + } } } @@ -73,24 +80,61 @@ protected Class loadClass(final String name, final boolean resolve) throws Cl } } + protected Class loadClassFromExtensionDependencies(final String name) throws ClassNotFoundException + { + for (ClassLoader classLoader : extensionDependencyClassLoaders) { + try { + return classLoader.loadClass(name); + } + catch (ClassNotFoundException ignored) { + } + } + throw new ClassNotFoundException(); + } + + @Override public URL getResource(final String name) { - final URL resourceFromExtension = super.getResource(name); + URL resourceFromExtension = super.getResource(name); + + if (resourceFromExtension != null) { + return resourceFromExtension; + } + resourceFromExtension = getResourceFromExtensionsDependencies(name); if (resourceFromExtension != null) { return resourceFromExtension; - } else { - return druidLoader.getResource(name); } + + return druidLoader.getResource(name); + } + protected URL getResourceFromExtensionsDependencies(final String name) + { + URL resourceFromExtension = null; + for (ClassLoader classLoader : extensionDependencyClassLoaders) { + resourceFromExtension = classLoader.getResource(name); + if (resourceFromExtension != null) { + break; + } + } + return resourceFromExtension; } + @Override public Enumeration getResources(final String name) throws IOException { final List urls = new ArrayList<>(); Iterators.addAll(urls, Iterators.forEnumeration(super.getResources(name))); Iterators.addAll(urls, Iterators.forEnumeration(druidLoader.getResources(name))); + for (ClassLoader classLoader : extensionDependencyClassLoaders) { + Iterators.addAll(urls, Iterators.forEnumeration(classLoader.getResources(name))); + } return Iterators.asEnumeration(urls.iterator()); } + + public void setExtensionDependencyClassLoaders(List extensionDependencyClassLoaders) { + this.extensionDependencyClassLoaders = Preconditions.checkNotNull(extensionDependencyClassLoaders, "extensionDependencyClassLoaders"); + } } diff --git a/processing/src/main/java/org/apache/druid/guice/ExtensionsLoader.java b/processing/src/main/java/org/apache/druid/guice/ExtensionsLoader.java index 0bdbddfa5a3f..1d30fcdf7ec9 100644 --- a/processing/src/main/java/org/apache/druid/guice/ExtensionsLoader.java +++ b/processing/src/main/java/org/apache/druid/guice/ExtensionsLoader.java @@ -20,6 +20,7 @@ package org.apache.druid.guice; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; import com.google.inject.Injector; import org.apache.commons.io.FileUtils; import org.apache.druid.initialization.DruidModule; @@ -224,7 +225,7 @@ private static URLClassLoader makeClassLoaderForExtension( } if (useExtensionClassloaderFirst) { - return new ExtensionFirstClassLoader(urls, ExtensionsLoader.class.getClassLoader()); + return new ExtensionFirstClassLoader(urls, ExtensionsLoader.class.getClassLoader(), ImmutableList.of()); } else { return new URLClassLoader(urls, ExtensionsLoader.class.getClassLoader()); } @@ -278,6 +279,7 @@ private ServiceLoadingFromExtensions(Class serviceClass) if (extensionsConfig.searchCurrentClassloader()) { addAllFromCurrentClassLoader(); } + addAllFromFileSystem(); } @@ -291,13 +293,41 @@ private void addAllFromCurrentClassLoader() private void addAllFromFileSystem() { for (File extension : getExtensionFilesToLoad()) { - log.debug("Loading extension [%s] for class [%s]", extension.getName(), serviceClass); try { - final URLClassLoader loader = getClassLoaderForExtension( + getClassLoaderForExtension( extension, extensionsConfig.isUseExtensionClassloaderFirst() ); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + ExtensionFirstClassLoader kafkaLoader = null; + ExtensionFirstClassLoader globalLoader = null; + for (Pair extension : loaders.keySet()) { + String extensionName = extension.lhs.getName(); + log.info("Extension name %s", extensionName); + if (extensionName.equals("druid-kafka-extraction-namespace")) { + kafkaLoader = (ExtensionFirstClassLoader) loaders.get(extension); + } + + if (extensionName.equals("druid-lookups-cached-global")) { + globalLoader = (ExtensionFirstClassLoader) loaders.get(extension); + } + } + + if (kafkaLoader != null && globalLoader != null) { + log.info("Setting extension dependency %s %s", kafkaLoader.getURLs(), globalLoader.getURLs()); + kafkaLoader.setExtensionDependencyClassLoaders(ImmutableList.of(globalLoader)); + } + for (File extension : getExtensionFilesToLoad()) { + try { + URLClassLoader loader = getClassLoaderForExtension( + extension, + extensionsConfig.isUseExtensionClassloaderFirst() + ); log.info( "Loading extension [%s], jars: %s", extension.getName(), From d9f969b3a7c091147745849ff7bcf042d2723e6f Mon Sep 17 00:00:00 2001 From: George Wu Date: Thu, 22 Aug 2024 12:40:27 -0400 Subject: [PATCH 4/6] Revert "test with extension dependencies" This reverts commit 2ccaf611efdbf5e3256a0862671664c524ebb6d4. --- .../kafka-extraction-namespace/pom.xml | 1 - .../guice/ExtensionFirstClassLoader.java | 56 ++----------------- .../apache/druid/guice/ExtensionsLoader.java | 36 +----------- 3 files changed, 9 insertions(+), 84 deletions(-) diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index a23bd115da8f..d3cf5d3216fd 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -44,7 +44,6 @@ org.apache.druid.extensions druid-lookups-cached-global ${project.parent.version} - provided org.apache.druid diff --git a/processing/src/main/java/org/apache/druid/guice/ExtensionFirstClassLoader.java b/processing/src/main/java/org/apache/druid/guice/ExtensionFirstClassLoader.java index 0ffc8fdbeb75..1a2944b2bdb0 100644 --- a/processing/src/main/java/org/apache/druid/guice/ExtensionFirstClassLoader.java +++ b/processing/src/main/java/org/apache/druid/guice/ExtensionFirstClassLoader.java @@ -35,13 +35,11 @@ public class ExtensionFirstClassLoader extends URLClassLoader { private final ClassLoader druidLoader; - private List extensionDependencyClassLoaders; - public ExtensionFirstClassLoader(final URL[] urls, final ClassLoader druidLoader, final List extensionDependencyClassLoaders) + public ExtensionFirstClassLoader(final URL[] urls, final ClassLoader druidLoader) { super(urls, null); this.druidLoader = Preconditions.checkNotNull(druidLoader, "druidLoader"); - this.extensionDependencyClassLoaders = Preconditions.checkNotNull(extensionDependencyClassLoaders, "extensionDependencyClassLoaders"); } @Override @@ -62,13 +60,8 @@ protected Class loadClass(final String name, final boolean resolve) throws Cl clazz = findClass(name); } catch (ClassNotFoundException e) { - try { - return loadClassFromExtensionDependencies(name); - } - catch (ClassNotFoundException e2) { - // Try the Druid classloader. Will throw ClassNotFoundException if the class can't be loaded. - return druidLoader.loadClass(name); - } + // Try the Druid classloader. Will throw ClassNotFoundException if the class can't be loaded. + return druidLoader.loadClass(name); } } @@ -80,61 +73,24 @@ protected Class loadClass(final String name, final boolean resolve) throws Cl } } - protected Class loadClassFromExtensionDependencies(final String name) throws ClassNotFoundException - { - for (ClassLoader classLoader : extensionDependencyClassLoaders) { - try { - return classLoader.loadClass(name); - } - catch (ClassNotFoundException ignored) { - } - } - throw new ClassNotFoundException(); - } - - @Override public URL getResource(final String name) { - URL resourceFromExtension = super.getResource(name); - - if (resourceFromExtension != null) { - return resourceFromExtension; - } + final URL resourceFromExtension = super.getResource(name); - resourceFromExtension = getResourceFromExtensionsDependencies(name); if (resourceFromExtension != null) { return resourceFromExtension; + } else { + return druidLoader.getResource(name); } - - return druidLoader.getResource(name); - } - protected URL getResourceFromExtensionsDependencies(final String name) - { - URL resourceFromExtension = null; - for (ClassLoader classLoader : extensionDependencyClassLoaders) { - resourceFromExtension = classLoader.getResource(name); - if (resourceFromExtension != null) { - break; - } - } - return resourceFromExtension; } - @Override public Enumeration getResources(final String name) throws IOException { final List urls = new ArrayList<>(); Iterators.addAll(urls, Iterators.forEnumeration(super.getResources(name))); Iterators.addAll(urls, Iterators.forEnumeration(druidLoader.getResources(name))); - for (ClassLoader classLoader : extensionDependencyClassLoaders) { - Iterators.addAll(urls, Iterators.forEnumeration(classLoader.getResources(name))); - } return Iterators.asEnumeration(urls.iterator()); } - - public void setExtensionDependencyClassLoaders(List extensionDependencyClassLoaders) { - this.extensionDependencyClassLoaders = Preconditions.checkNotNull(extensionDependencyClassLoaders, "extensionDependencyClassLoaders"); - } } diff --git a/processing/src/main/java/org/apache/druid/guice/ExtensionsLoader.java b/processing/src/main/java/org/apache/druid/guice/ExtensionsLoader.java index 1d30fcdf7ec9..0bdbddfa5a3f 100644 --- a/processing/src/main/java/org/apache/druid/guice/ExtensionsLoader.java +++ b/processing/src/main/java/org/apache/druid/guice/ExtensionsLoader.java @@ -20,7 +20,6 @@ package org.apache.druid.guice; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; import com.google.inject.Injector; import org.apache.commons.io.FileUtils; import org.apache.druid.initialization.DruidModule; @@ -225,7 +224,7 @@ private static URLClassLoader makeClassLoaderForExtension( } if (useExtensionClassloaderFirst) { - return new ExtensionFirstClassLoader(urls, ExtensionsLoader.class.getClassLoader(), ImmutableList.of()); + return new ExtensionFirstClassLoader(urls, ExtensionsLoader.class.getClassLoader()); } else { return new URLClassLoader(urls, ExtensionsLoader.class.getClassLoader()); } @@ -279,7 +278,6 @@ private ServiceLoadingFromExtensions(Class serviceClass) if (extensionsConfig.searchCurrentClassloader()) { addAllFromCurrentClassLoader(); } - addAllFromFileSystem(); } @@ -293,41 +291,13 @@ private void addAllFromCurrentClassLoader() private void addAllFromFileSystem() { for (File extension : getExtensionFilesToLoad()) { + log.debug("Loading extension [%s] for class [%s]", extension.getName(), serviceClass); try { - getClassLoaderForExtension( + final URLClassLoader loader = getClassLoaderForExtension( extension, extensionsConfig.isUseExtensionClassloaderFirst() ); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - ExtensionFirstClassLoader kafkaLoader = null; - ExtensionFirstClassLoader globalLoader = null; - for (Pair extension : loaders.keySet()) { - String extensionName = extension.lhs.getName(); - log.info("Extension name %s", extensionName); - if (extensionName.equals("druid-kafka-extraction-namespace")) { - kafkaLoader = (ExtensionFirstClassLoader) loaders.get(extension); - } - - if (extensionName.equals("druid-lookups-cached-global")) { - globalLoader = (ExtensionFirstClassLoader) loaders.get(extension); - } - } - - if (kafkaLoader != null && globalLoader != null) { - log.info("Setting extension dependency %s %s", kafkaLoader.getURLs(), globalLoader.getURLs()); - kafkaLoader.setExtensionDependencyClassLoaders(ImmutableList.of(globalLoader)); - } - for (File extension : getExtensionFilesToLoad()) { - try { - URLClassLoader loader = getClassLoaderForExtension( - extension, - extensionsConfig.isUseExtensionClassloaderFirst() - ); log.info( "Loading extension [%s], jars: %s", extension.getName(), From ec3ee1b4ff05c513f57dbb8521be782c2b8ac256 Mon Sep 17 00:00:00 2001 From: George Shiqi Wu Date: Thu, 22 Aug 2024 09:40:55 -0700 Subject: [PATCH 5/6] Update docs/querying/kafka-extraction-namespace.md Co-authored-by: Charles Smith --- docs/querying/kafka-extraction-namespace.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/querying/kafka-extraction-namespace.md b/docs/querying/kafka-extraction-namespace.md index e7d02babdd66..9518d38e3510 100644 --- a/docs/querying/kafka-extraction-namespace.md +++ b/docs/querying/kafka-extraction-namespace.md @@ -24,8 +24,10 @@ title: "Apache Kafka Lookups" To use this Apache Druid extension, [include](../configuration/extensions.md#loading-extensions) `druid-kafka-extraction-namespace` in the extensions load list. -This extension provides all the functionality of `druid-lookups-cached-global`. You should only load one of the two extensions. -(load `druid-lookups-cached-global` if you only want global cached lookup functionality, load `druid-kafka-extraction-namespace` if you also want kafka lookup functionality) +The `druid-kafka-extraction-namespace` extension includes all the functionality of `druid-lookups-cached-global`. +Do not load both extensions, but choose the extension that best suits your needs: +- `druid-kafka-extraction-namespace` if you need Kafka lookup functionality +- `druid-lookups-cached-global` if you only want global cached lookup functionality If you need updates to populate as promptly as possible, it is possible to plug into a Kafka topic whose key is the old value and message is the desired new value (both in UTF-8) as a LookupExtractorFactory. From 8fdb7d7bba5cd910e5017f05418bb47e7f7ca5df Mon Sep 17 00:00:00 2001 From: George Shiqi Wu Date: Thu, 22 Aug 2024 09:40:59 -0700 Subject: [PATCH 6/6] Update docs/querying/lookups-cached-global.md Co-authored-by: Charles Smith --- docs/querying/lookups-cached-global.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/querying/lookups-cached-global.md b/docs/querying/lookups-cached-global.md index 8db1a4333d61..16765194059e 100644 --- a/docs/querying/lookups-cached-global.md +++ b/docs/querying/lookups-cached-global.md @@ -24,8 +24,10 @@ title: "Globally Cached Lookups" To use this Apache Druid extension, [include](../configuration/extensions.md#loading-extensions) `druid-lookups-cached-global` in the extensions load list. -The `druid-kafka-extraction-namespace` extension provides all the functionality of `druid-lookups-cached-global`. You should only load one of the two extensions. -(load `druid-lookups-cached-global` if you only want global cached lookup functionality, load `druid-kafka-extraction-namespace` if you also want kafka lookup functionality) +The `druid-kafka-extraction-namespace` extension includes all the functionality of `druid-lookups-cached-global`. +Do not load both extensions, but choose the extension that best suits your needs: +- `druid-lookups-cached-global` if you only want global cached lookup functionality +- `druid-kafka-extraction-namespace` if you need Kafka lookup functionality ## Configuration :::info