diff --git a/docs/operations/api-reference.md b/docs/operations/api-reference.md index c184e8daf971..085a571b456a 100644 --- a/docs/operations/api-reference.md +++ b/docs/operations/api-reference.md @@ -45,7 +45,7 @@ An endpoint that always returns a boolean "true" value with a 200 OK response, u Returns the current configuration properties of the process. -* `/status/selfDiscoveredStatus` +* `/status/selfDiscovered/status` Returns a JSON map of the form `{"selfDiscovered": true/false}`, indicating whether the node has received a confirmation from the central node discovery mechanism (currently ZooKeeper) of the Druid cluster that the node has been added to the @@ -60,7 +60,7 @@ nodes will be discovered by this node timely from this point. * `/status/selfDiscovered` -Similar to `/status/selfDiscoveredStatus`, but returns 200 OK response with empty body if the node has discovered itself +Similar to `/status/selfDiscovered/status`, but returns 200 OK response with empty body if the node has discovered itself and 503 SERVICE UNAVAILABLE if the node hasn't discovered itself yet. This endpoint might be useful because some monitoring checks such as AWS load balancer health checks are not able to look at the response body. diff --git a/extensions-core/hdfs-storage/src/main/java/org/apache/druid/firehose/hdfs/HdfsFirehoseFactory.java b/extensions-core/hdfs-storage/src/main/java/org/apache/druid/firehose/hdfs/HdfsFirehoseFactory.java index 962da5e5d31d..ee5bf032b4d0 100644 --- a/extensions-core/hdfs-storage/src/main/java/org/apache/druid/firehose/hdfs/HdfsFirehoseFactory.java +++ b/extensions-core/hdfs-storage/src/main/java/org/apache/druid/firehose/hdfs/HdfsFirehoseFactory.java @@ -27,6 +27,7 @@ import org.apache.druid.data.input.InputSplit; import org.apache.druid.data.input.impl.StringInputRowParser; import org.apache.druid.data.input.impl.prefetch.PrefetchableTextFilesFirehoseFactory; +import org.apache.druid.guice.Hdfs; import org.apache.druid.inputsource.hdfs.HdfsInputSource; import org.apache.druid.storage.hdfs.HdfsDataSegmentPuller; import org.apache.druid.utils.CompressionUtils; @@ -46,7 +47,7 @@ public class HdfsFirehoseFactory extends PrefetchableTextFilesFirehoseFactory fullyQualifiedStorageDirectory; @Inject - public HdfsDataSegmentPusher(HdfsDataSegmentPusherConfig config, Configuration hadoopConfig, ObjectMapper jsonMapper) + public HdfsDataSegmentPusher( + HdfsDataSegmentPusherConfig config, + @Hdfs Configuration hadoopConfig, + ObjectMapper jsonMapper + ) { this.hadoopConfig = hadoopConfig; this.jsonMapper = jsonMapper; diff --git a/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsFileTimestampVersionFinder.java b/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsFileTimestampVersionFinder.java index d0530d246ba9..4daa94a6b02d 100644 --- a/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsFileTimestampVersionFinder.java +++ b/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsFileTimestampVersionFinder.java @@ -21,6 +21,7 @@ import com.google.inject.Inject; import org.apache.druid.data.SearchableVersionedDataFinder; +import org.apache.druid.guice.Hdfs; import org.apache.druid.java.util.common.RetryUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -36,7 +37,7 @@ public class HdfsFileTimestampVersionFinder extends HdfsDataSegmentPuller implements SearchableVersionedDataFinder { @Inject - public HdfsFileTimestampVersionFinder(Configuration config) + public HdfsFileTimestampVersionFinder(@Hdfs Configuration config) { super(config); } diff --git a/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageAuthentication.java b/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageAuthentication.java index 93fbf48efa2f..096875fdaf44 100644 --- a/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageAuthentication.java +++ b/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageAuthentication.java @@ -22,6 +22,7 @@ import com.google.common.base.Strings; import com.google.inject.Inject; +import org.apache.druid.guice.Hdfs; import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.lifecycle.LifecycleStart; @@ -40,7 +41,7 @@ public class HdfsStorageAuthentication private final Configuration hadoopConf; @Inject - public HdfsStorageAuthentication(HdfsKerberosConfig hdfsKerberosConfig, Configuration hadoopConf) + public HdfsStorageAuthentication(HdfsKerberosConfig hdfsKerberosConfig, @Hdfs Configuration hadoopConf) { this.hdfsKerberosConfig = hdfsKerberosConfig; this.hadoopConf = hadoopConf; diff --git a/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageDruidModule.java b/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageDruidModule.java index 6e88f8ee38b7..d4242bcddcfc 100644 --- a/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageDruidModule.java +++ b/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/HdfsStorageDruidModule.java @@ -28,6 +28,7 @@ import org.apache.druid.data.SearchableVersionedDataFinder; import org.apache.druid.firehose.hdfs.HdfsFirehoseFactory; import org.apache.druid.guice.Binders; +import org.apache.druid.guice.Hdfs; import org.apache.druid.guice.JsonConfigProvider; import org.apache.druid.guice.LazySingleton; import org.apache.druid.guice.LifecycleModule; @@ -108,7 +109,7 @@ public void configure(Binder binder) } } - binder.bind(Configuration.class).toInstance(conf); + binder.bind(Configuration.class).annotatedWith(Hdfs.class).toInstance(conf); JsonConfigProvider.bind(binder, "druid.storage", HdfsDataSegmentPusherConfig.class); Binders.taskLogsBinder(binder).addBinding("hdfs").to(HdfsTaskLogs.class); @@ -117,6 +118,5 @@ public void configure(Binder binder) JsonConfigProvider.bind(binder, "druid.hadoop.security.kerberos", HdfsKerberosConfig.class); binder.bind(HdfsStorageAuthentication.class).in(ManageLifecycle.class); LifecycleModule.register(binder, HdfsStorageAuthentication.class); - } } diff --git a/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/tasklog/HdfsTaskLogs.java b/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/tasklog/HdfsTaskLogs.java index 4e2b68fc67ec..7289901823b6 100644 --- a/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/tasklog/HdfsTaskLogs.java +++ b/extensions-core/hdfs-storage/src/main/java/org/apache/druid/storage/hdfs/tasklog/HdfsTaskLogs.java @@ -23,6 +23,7 @@ import com.google.common.io.ByteSource; import com.google.common.io.ByteStreams; import com.google.inject.Inject; +import org.apache.druid.guice.Hdfs; import org.apache.druid.java.util.common.IOE; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.tasklogs.TaskLogs; @@ -51,7 +52,7 @@ public class HdfsTaskLogs implements TaskLogs private final Configuration hadoopConfig; @Inject - public HdfsTaskLogs(HdfsTaskLogsConfig config, Configuration hadoopConfig) + public HdfsTaskLogs(HdfsTaskLogsConfig config, @Hdfs Configuration hadoopConfig) { this.config = config; this.hadoopConfig = hadoopConfig; diff --git a/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcExtensionsModule.java b/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcExtensionsModule.java index e57995bd26f4..78082cba5a82 100644 --- a/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcExtensionsModule.java +++ b/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcExtensionsModule.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.inject.Binder; import com.google.inject.Inject; +import org.apache.druid.data.input.orc.guice.Orc; import org.apache.druid.initialization.DruidModule; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -89,6 +90,6 @@ public void configure(Binder binder) } } - binder.bind(Configuration.class).toInstance(conf); + binder.bind(Configuration.class).annotatedWith(Orc.class).toInstance(conf); } } diff --git a/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcInputFormat.java b/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcInputFormat.java index bf60fb958966..1744fef9f6c2 100644 --- a/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcInputFormat.java +++ b/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/OrcInputFormat.java @@ -26,6 +26,7 @@ import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.impl.NestedInputFormat; +import org.apache.druid.data.input.orc.guice.Orc; import org.apache.druid.java.util.common.parsers.JSONPathSpec; import org.apache.hadoop.conf.Configuration; @@ -42,7 +43,7 @@ public class OrcInputFormat extends NestedInputFormat public OrcInputFormat( @JsonProperty("flattenSpec") @Nullable JSONPathSpec flattenSpec, @JsonProperty("binaryAsString") @Nullable Boolean binaryAsString, - @JacksonInject Configuration conf + @JacksonInject @Orc Configuration conf ) { super(flattenSpec); diff --git a/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/guice/Orc.java b/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/guice/Orc.java new file mode 100644 index 000000000000..ff1be32b6037 --- /dev/null +++ b/extensions-core/orc-extensions/src/main/java/org/apache/druid/data/input/orc/guice/Orc.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.orc.guice; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Each extension module needs to properly bind whatever it will use, but sometimes different modules need to bind the + * same class which will lead to the duplicate injection error. To avoid this problem, each module is supposed to bind + * different instances. This is a binding annotation for druid-orc-extensions extension. Any binding for the same type + * should be distinguished by using this annotation. + */ +@Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@BindingAnnotation +public @interface Orc +{ +} diff --git a/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetExtensionsModule.java b/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetExtensionsModule.java index dd2cf4c34f82..17fe50d42870 100644 --- a/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetExtensionsModule.java +++ b/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetExtensionsModule.java @@ -25,6 +25,7 @@ import com.google.inject.Binder; import com.google.inject.Inject; import org.apache.druid.data.input.parquet.avro.ParquetAvroHadoopInputRowParser; +import org.apache.druid.data.input.parquet.guice.Parquet; import org.apache.druid.data.input.parquet.simple.ParquetHadoopInputRowParser; import org.apache.druid.data.input.parquet.simple.ParquetParseSpec; import org.apache.druid.initialization.DruidModule; @@ -98,6 +99,6 @@ public void configure(Binder binder) } } - binder.bind(Configuration.class).toInstance(conf); + binder.bind(Configuration.class).annotatedWith(Parquet.class).toInstance(conf); } } diff --git a/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetInputFormat.java b/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetInputFormat.java index fbd93414a4a1..c49581a9b72a 100644 --- a/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetInputFormat.java +++ b/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetInputFormat.java @@ -19,13 +19,16 @@ package org.apache.druid.data.input.parquet; +import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.data.input.InputEntity; import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.impl.NestedInputFormat; +import org.apache.druid.data.input.parquet.guice.Parquet; import org.apache.druid.java.util.common.parsers.JSONPathSpec; +import org.apache.hadoop.conf.Configuration; import javax.annotation.Nullable; import java.io.File; @@ -35,15 +38,18 @@ public class ParquetInputFormat extends NestedInputFormat { private final boolean binaryAsString; + private final Configuration conf; @JsonCreator public ParquetInputFormat( @JsonProperty("flattenSpec") @Nullable JSONPathSpec flattenSpec, - @JsonProperty("binaryAsString") @Nullable Boolean binaryAsString + @JsonProperty("binaryAsString") @Nullable Boolean binaryAsString, + @JacksonInject @Parquet Configuration conf ) { super(flattenSpec); this.binaryAsString = binaryAsString == null ? false : binaryAsString; + this.conf = conf; } @JsonProperty @@ -65,7 +71,7 @@ public InputEntityReader createReader( File temporaryDirectory ) throws IOException { - return new ParquetReader(inputRowSchema, source, temporaryDirectory, getFlattenSpec(), binaryAsString); + return new ParquetReader(conf, inputRowSchema, source, temporaryDirectory, getFlattenSpec(), binaryAsString); } @Override diff --git a/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetReader.java b/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetReader.java index a98f929a1c03..2c76253982f2 100644 --- a/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetReader.java +++ b/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/ParquetReader.java @@ -31,6 +31,7 @@ import org.apache.druid.java.util.common.parsers.ObjectFlattener; import org.apache.druid.java.util.common.parsers.ObjectFlatteners; import org.apache.druid.java.util.common.parsers.ParseException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.parquet.example.data.Group; import org.apache.parquet.hadoop.example.GroupReadSupport; @@ -51,6 +52,7 @@ public class ParquetReader extends IntermediateRowParsingReader private final Closer closer; ParquetReader( + Configuration conf, InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory, @@ -69,7 +71,9 @@ public class ParquetReader extends IntermediateRowParsingReader final ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); - reader = closer.register(org.apache.parquet.hadoop.ParquetReader.builder(new GroupReadSupport(), path).build()); + reader = closer.register(org.apache.parquet.hadoop.ParquetReader.builder(new GroupReadSupport(), path) + .withConf(conf) + .build()); } finally { Thread.currentThread().setContextClassLoader(currentClassLoader); diff --git a/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/guice/Parquet.java b/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/guice/Parquet.java new file mode 100644 index 000000000000..a6f46c6f1ee1 --- /dev/null +++ b/extensions-core/parquet-extensions/src/main/java/org/apache/druid/data/input/parquet/guice/Parquet.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.parquet.guice; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Each extension module needs to properly bind whatever it will use, but sometimes different modules need to bind the + * same class which will lead to the duplicate injection error. To avoid this problem, each module is supposed to bind + * different instances. This is a binding annotation for druid-parquet-extensions extension. Any binding for the same + * type should be distinguished by using this annotation. + */ +@Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@BindingAnnotation +public @interface Parquet +{ +} diff --git a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/BaseParquetReaderTest.java b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/BaseParquetReaderTest.java index ac22f77f5f49..87f9229b3137 100644 --- a/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/BaseParquetReaderTest.java +++ b/extensions-core/parquet-extensions/src/test/java/org/apache/druid/data/input/parquet/BaseParquetReaderTest.java @@ -28,6 +28,7 @@ import org.apache.druid.data.input.impl.FileEntity; import org.apache.druid.java.util.common.parsers.CloseableIterator; import org.apache.druid.java.util.common.parsers.JSONPathSpec; +import org.apache.hadoop.conf.Configuration; import java.io.File; import java.io.IOException; @@ -51,7 +52,7 @@ InputEntityReader createReader( ) throws IOException { FileEntity entity = new FileEntity(new File(parquetFile)); - ParquetInputFormat parquet = new ParquetInputFormat(flattenSpec, binaryAsString); + ParquetInputFormat parquet = new ParquetInputFormat(flattenSpec, binaryAsString, new Configuration()); return parquet.createReader(schema, entity, null); } diff --git a/server/src/main/java/org/apache/druid/server/http/SelfDiscoveryResource.java b/server/src/main/java/org/apache/druid/server/http/SelfDiscoveryResource.java index 46d04a9006f7..13b7bded496e 100644 --- a/server/src/main/java/org/apache/druid/server/http/SelfDiscoveryResource.java +++ b/server/src/main/java/org/apache/druid/server/http/SelfDiscoveryResource.java @@ -19,6 +19,7 @@ package org.apache.druid.server.http; +import com.google.common.collect.Lists; import com.google.inject.Inject; import com.google.inject.Singleton; import com.sun.jersey.spi.container.ResourceFilters; @@ -36,6 +37,8 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.util.Collections; +import java.util.List; +import java.util.Set; import java.util.function.BooleanSupplier; /** @@ -44,58 +47,68 @@ * DI configuration phase. */ @Singleton +@Path("/status/selfDiscovered") public class SelfDiscoveryResource { - private BooleanSupplier selfDiscovered; + private final List selfDiscoveredRoles; @Inject public SelfDiscoveryResource( @Self DruidNode thisDruidNode, - @Self NodeRole thisNodeRole, + @Self Set thisNodeRoles, DruidNodeDiscoveryProvider nodeDiscoveryProvider, Lifecycle lifecycle ) { - Lifecycle.Handler selfDiscoveryListenerRegistrator = new Lifecycle.Handler() - { - @Override - public void start() - { - selfDiscovered = nodeDiscoveryProvider.getForNode(thisDruidNode, thisNodeRole); - } + selfDiscoveredRoles = Lists.newArrayListWithExpectedSize(thisNodeRoles.size()); + thisNodeRoles.forEach( + thisNodeRole -> { + Lifecycle.Handler selfDiscoveryListenerRegistrator = new Lifecycle.Handler() + { + @Override + public void start() + { + selfDiscoveredRoles.add(nodeDiscoveryProvider.getForNode(thisDruidNode, thisNodeRole)); + } - @Override - public void stop() - { - // do nothing - } - }; - // Using Lifecycle.Stage.SERVER because DruidNodeDiscoveryProvider should be already started when - // selfDiscoveryListenerRegistrator.start() is called. - lifecycle.addHandler(selfDiscoveryListenerRegistrator, Lifecycle.Stage.SERVER); + @Override + public void stop() + { + // do nothing + } + }; + // Using Lifecycle.Stage.SERVER because DruidNodeDiscoveryProvider should be already started when + // selfDiscoveryListenerRegistrator.start() is called. + lifecycle.addHandler(selfDiscoveryListenerRegistrator, Lifecycle.Stage.SERVER); + } + ); } /** See the description of this endpoint in api-reference.md. */ @GET - @Path("/status/selfDiscoveredStatus") + @Path("/status") @Produces(MediaType.APPLICATION_JSON) @ResourceFilters(StateResourceFilter.class) public Response getSelfDiscoveredStatus() { - return Response.ok(Collections.singletonMap("selfDiscovered", selfDiscovered.getAsBoolean())).build(); + return Response.ok(Collections.singletonMap("selfDiscovered", isDiscoveredAllRoles())).build(); } /** See the description of this endpoint in api-reference.md. */ @GET - @Path("/status/selfDiscovered") @Produces(MediaType.APPLICATION_JSON) @ResourceFilters(StateResourceFilter.class) public Response getSelfDiscovered() { - if (selfDiscovered.getAsBoolean()) { + if (isDiscoveredAllRoles()) { return Response.ok().build(); } else { return Response.status(HttpStatus.SERVICE_UNAVAILABLE_503).build(); } } + + private boolean isDiscoveredAllRoles() + { + return selfDiscoveredRoles.stream().allMatch(BooleanSupplier::getAsBoolean); + } } diff --git a/server/src/test/java/org/apache/druid/server/http/security/SecurityResourceFilterTest.java b/server/src/test/java/org/apache/druid/server/http/security/SecurityResourceFilterTest.java index 344396c98b3b..e0a5ba13334d 100644 --- a/server/src/test/java/org/apache/druid/server/http/security/SecurityResourceFilterTest.java +++ b/server/src/test/java/org/apache/druid/server/http/security/SecurityResourceFilterTest.java @@ -53,7 +53,7 @@ public class SecurityResourceFilterTest extends ResourceFilterTestHelper { @Parameterized.Parameters(name = "{index}: requestPath={0}, requestMethod={1}, resourceFilter={2}") - public static Collection data() throws NoSuchMethodException + public static Collection data() { return ImmutableList.copyOf( Iterables.concat( @@ -70,8 +70,7 @@ public static Collection data() throws NoSuchMethodException getRequestPathsWithAuthorizer(CoordinatorDynamicConfigsResource.class), getRequestPathsWithAuthorizer(QueryResource.class), getRequestPathsWithAuthorizer(StatusResource.class), - getRequestPathsWithAuthorizer(SelfDiscoveryResource.class.getDeclaredMethod("getSelfDiscoveredStatus")), - getRequestPathsWithAuthorizer(SelfDiscoveryResource.class.getDeclaredMethod("getSelfDiscovered")), + getRequestPathsWithAuthorizer(SelfDiscoveryResource.class), getRequestPathsWithAuthorizer(BrokerQueryResource.class), getRequestPathsWithAuthorizer(RouterResource.class) ) diff --git a/services/src/main/java/org/apache/druid/cli/CliBroker.java b/services/src/main/java/org/apache/druid/cli/CliBroker.java index 45864f4e2e2d..aaf46b95d5d4 100644 --- a/services/src/main/java/org/apache/druid/cli/CliBroker.java +++ b/services/src/main/java/org/apache/druid/cli/CliBroker.java @@ -43,7 +43,6 @@ import org.apache.druid.guice.LifecycleModule; import org.apache.druid.guice.QueryRunnerFactoryModule; import org.apache.druid.guice.QueryableModule; -import org.apache.druid.guice.annotations.Self; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.RetryQueryRunnerConfig; @@ -123,9 +122,7 @@ protected List getModules() LifecycleModule.register(binder, Server.class); - binder.bind(NodeRole.class).annotatedWith(Self.class).toInstance(NodeRole.BROKER); - - bindAnnouncer( + bindNodeRoleAndAnnouncer( binder, DiscoverySideEffectsProvider .builder(NodeRole.BROKER) diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index bfe5ef7da48c..6e4c27fbc292 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -46,7 +46,6 @@ import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.guice.annotations.CoordinatorIndexingServiceHelper; import org.apache.druid.guice.annotations.EscalatedGlobal; -import org.apache.druid.guice.annotations.Self; import org.apache.druid.guice.http.JettyHttpClientModule; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.concurrent.ExecutorServices; @@ -239,9 +238,7 @@ public void configure(Binder binder) DruidCoordinatorCleanupPendingSegments.class ); - binder.bind(NodeRole.class).annotatedWith(Self.class).toInstance(NodeRole.COORDINATOR); - - bindAnnouncer( + bindNodeRoleAndAnnouncer( binder, Coordinator.class, DiscoverySideEffectsProvider.builder(NodeRole.COORDINATOR).build() diff --git a/services/src/main/java/org/apache/druid/cli/CliHistorical.java b/services/src/main/java/org/apache/druid/cli/CliHistorical.java index c1bc8baadc7d..b700d828d903 100644 --- a/services/src/main/java/org/apache/druid/cli/CliHistorical.java +++ b/services/src/main/java/org/apache/druid/cli/CliHistorical.java @@ -38,7 +38,6 @@ import org.apache.druid.guice.QueryRunnerFactoryModule; import org.apache.druid.guice.QueryableModule; import org.apache.druid.guice.ServerTypeConfig; -import org.apache.druid.guice.annotations.Self; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.lookup.LookupModule; @@ -104,9 +103,7 @@ protected List getModules() JsonConfigProvider.bind(binder, "druid.historical.cache", CacheConfig.class); binder.install(new CacheModule()); - binder.bind(NodeRole.class).annotatedWith(Self.class).toInstance(NodeRole.HISTORICAL); - - bindAnnouncer( + bindNodeRoleAndAnnouncer( binder, DiscoverySideEffectsProvider .builder(NodeRole.HISTORICAL) diff --git a/services/src/main/java/org/apache/druid/cli/CliIndexer.java b/services/src/main/java/org/apache/druid/cli/CliIndexer.java index 6e51c45c065d..5ffd2a83483a 100644 --- a/services/src/main/java/org/apache/druid/cli/CliIndexer.java +++ b/services/src/main/java/org/apache/druid/cli/CliIndexer.java @@ -149,7 +149,7 @@ public void configure(Binder binder) binder.bind(SegmentLoadDropHandler.class).toProvider(Providers.of(null)); - bindAnnouncer( + bindNodeRoleAndAnnouncer( binder, DiscoverySideEffectsProvider .builder(NodeRole.INDEXER) diff --git a/services/src/main/java/org/apache/druid/cli/CliMiddleManager.java b/services/src/main/java/org/apache/druid/cli/CliMiddleManager.java index b6ce582008ec..0eeb5484919d 100644 --- a/services/src/main/java/org/apache/druid/cli/CliMiddleManager.java +++ b/services/src/main/java/org/apache/druid/cli/CliMiddleManager.java @@ -141,9 +141,7 @@ public void configure(Binder binder) LifecycleModule.register(binder, Server.class); - binder.bind(NodeRole.class).annotatedWith(Self.class).toInstance(NodeRole.MIDDLE_MANAGER); - - bindAnnouncer( + bindNodeRoleAndAnnouncer( binder, DiscoverySideEffectsProvider .builder(NodeRole.MIDDLE_MANAGER) diff --git a/services/src/main/java/org/apache/druid/cli/CliOverlord.java b/services/src/main/java/org/apache/druid/cli/CliOverlord.java index 6e00ba9d99a9..f84a5e18ee27 100644 --- a/services/src/main/java/org/apache/druid/cli/CliOverlord.java +++ b/services/src/main/java/org/apache/druid/cli/CliOverlord.java @@ -52,7 +52,6 @@ import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.guice.PolyBind; import org.apache.druid.guice.annotations.Json; -import org.apache.druid.guice.annotations.Self; import org.apache.druid.indexing.common.actions.LocalTaskActionClientFactory; import org.apache.druid.indexing.common.actions.TaskActionClientFactory; import org.apache.druid.indexing.common.actions.TaskActionToolbox; @@ -248,9 +247,7 @@ public void configure(Binder binder) LifecycleModule.register(binder, Server.class); } - binder.bind(NodeRole.class).annotatedWith(Self.class).toInstance(NodeRole.OVERLORD); - - bindAnnouncer( + bindNodeRoleAndAnnouncer( binder, IndexingService.class, DiscoverySideEffectsProvider.builder(NodeRole.OVERLORD).build() diff --git a/services/src/main/java/org/apache/druid/cli/CliRouter.java b/services/src/main/java/org/apache/druid/cli/CliRouter.java index d945222bcbdf..21ab6308ab05 100644 --- a/services/src/main/java/org/apache/druid/cli/CliRouter.java +++ b/services/src/main/java/org/apache/druid/cli/CliRouter.java @@ -109,12 +109,7 @@ protected List getModules() LifecycleModule.register(binder, Server.class); DiscoveryModule.register(binder, Self.class); - binder.bind(NodeRole.class).annotatedWith(Self.class).toInstance(NodeRole.ROUTER); - - bindAnnouncer( - binder, - DiscoverySideEffectsProvider.builder(NodeRole.ROUTER).build() - ); + bindNodeRoleAndAnnouncer(binder, DiscoverySideEffectsProvider.builder(NodeRole.ROUTER).build()); Jerseys.addResource(binder, SelfDiscoveryResource.class); LifecycleModule.registerKey(binder, Key.get(SelfDiscoveryResource.class)); diff --git a/services/src/main/java/org/apache/druid/cli/ServerRunnable.java b/services/src/main/java/org/apache/druid/cli/ServerRunnable.java index 8ac9244d7d62..4beeab8f9fee 100644 --- a/services/src/main/java/org/apache/druid/cli/ServerRunnable.java +++ b/services/src/main/java/org/apache/druid/cli/ServerRunnable.java @@ -26,6 +26,7 @@ import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Provider; +import com.google.inject.multibindings.Multibinder; import org.apache.druid.curator.discovery.ServiceAnnouncer; import org.apache.druid.discovery.DiscoveryDruidNode; import org.apache.druid.discovery.DruidNodeAnnouncer; @@ -42,6 +43,7 @@ import java.util.List; /** + * */ public abstract class ServerRunnable extends GuiceRunnable { @@ -64,7 +66,34 @@ public void run() } } - public static void bindAnnouncer( + public static void bindNodeRoleAndAnnouncer(Binder binder, DiscoverySideEffectsProvider discoverySideEffectsProvider) + { + Multibinder selfBinder = Multibinder.newSetBinder(binder, NodeRole.class, Self.class); + selfBinder.addBinding().toInstance(discoverySideEffectsProvider.nodeRole); + + bindAnnouncer( + binder, + discoverySideEffectsProvider + ); + } + + public static void bindNodeRoleAndAnnouncer( + Binder binder, + Class annotation, + DiscoverySideEffectsProvider discoverySideEffectsProvider + ) + { + Multibinder selfBinder = Multibinder.newSetBinder(binder, NodeRole.class, Self.class); + selfBinder.addBinding().toInstance(discoverySideEffectsProvider.nodeRole); + + bindAnnouncer( + binder, + annotation, + discoverySideEffectsProvider + ); + } + + private static void bindAnnouncer( final Binder binder, final DiscoverySideEffectsProvider provider ) @@ -76,7 +105,7 @@ public static void bindAnnouncer( LifecycleModule.registerKey(binder, Key.get(DiscoverySideEffectsProvider.Child.class)); } - public static void bindAnnouncer( + private static void bindAnnouncer( final Binder binder, final Class annotation, final DiscoverySideEffectsProvider provider