From f8e92cc9220d1da70cc58e2e8c810952a57edb49 Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 25 Aug 2017 18:20:21 -0500 Subject: [PATCH 1/6] Move emitters from io.druid.server.initialization to the dedicated io.druid.server.emitter package; Update emitter library to 0.6.0; Add support for ParametrizedUriEmitter; Support hierarical properties in JsonConfigurator (was needed for ParametrizedUriEmitter) --- .../java/io/druid/guice/JsonConfigurator.java | 40 ++++++++- pom.xml | 2 +- .../druid/initialization/Initialization.java | 2 +- .../ComposingEmitterConfig.java | 2 +- .../ComposingEmitterModule.java | 2 +- .../EmitterModule.java | 4 +- .../HttpEmitterConfig.java | 6 +- .../HttpEmitterModule.java | 7 +- .../LogEmitterModule.java | 2 +- .../NoopEmitterModule.java | 2 +- .../emitter/ParametrizedUriEmitterConfig.java | 34 ++++++++ .../emitter/ParametrizedUriEmitterModule.java | 72 ++++++++++++++++ .../ComposingEmitterModuleTest.java | 4 +- .../server/emitter/EmitterModuleTest.java | 82 +++++++++++++++++++ 14 files changed, 247 insertions(+), 14 deletions(-) rename server/src/main/java/io/druid/server/{initialization => emitter}/ComposingEmitterConfig.java (96%) rename server/src/main/java/io/druid/server/{initialization => emitter}/ComposingEmitterModule.java (98%) rename server/src/main/java/io/druid/server/{initialization => emitter}/EmitterModule.java (96%) rename server/src/main/java/io/druid/server/{initialization => emitter}/HttpEmitterConfig.java (90%) rename server/src/main/java/io/druid/server/{initialization => emitter}/HttpEmitterModule.java (96%) rename server/src/main/java/io/druid/server/{initialization => emitter}/LogEmitterModule.java (97%) rename server/src/main/java/io/druid/server/{initialization => emitter}/NoopEmitterModule.java (97%) create mode 100644 server/src/main/java/io/druid/server/emitter/ParametrizedUriEmitterConfig.java create mode 100644 server/src/main/java/io/druid/server/emitter/ParametrizedUriEmitterModule.java create mode 100644 server/src/test/java/io/druid/server/emitter/EmitterModuleTest.java diff --git a/api/src/main/java/io/druid/guice/JsonConfigurator.java b/api/src/main/java/io/druid/guice/JsonConfigurator.java index ff5c4fc067f5..9474cc66e098 100644 --- a/api/src/main/java/io/druid/guice/JsonConfigurator.java +++ b/api/src/main/java/io/druid/guice/JsonConfigurator.java @@ -43,6 +43,7 @@ import javax.validation.Validator; import java.io.IOException; import java.lang.reflect.Field; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -93,7 +94,7 @@ public T configurate(Properties props, String propertyPrefix, Class clazz value = propValue; } - jsonMap.put(prop.substring(propertyBase.length()), value); + hieraricalPutValue(propertyPrefix, prop, prop.substring(propertyBase.length()), value, jsonMap); } } @@ -165,6 +166,43 @@ public Message apply(String input) return config; } + private static void hieraricalPutValue( + String propertyPrefix, + String originalProperty, + String property, + Object value, + Map targetMap + ) + { + int dotIndex = property.indexOf('.'); + if (dotIndex < 0) { + targetMap.put(property, value); + return; + } + if (dotIndex == 0) { + throw new ProvisionException(String.format("Double dot in property: %s", originalProperty)); + } + if (dotIndex == property.length() - 1) { + throw new ProvisionException(String.format("Dot at the end of property: %s", originalProperty)); + } + String nestedKey = property.substring(0, dotIndex); + Object nested = targetMap.computeIfAbsent(nestedKey, k -> new HashMap()); + if (!(nested instanceof Map)) { + // Clash is possible between properties, which are used to configure different objects: e. g. + // druid.emitter=parametrized is used to configure Emitter class, and druid.emitter.parametrized.xxx=yyy is used + // to configure ParametrizedUriEmitterConfig object. So skipping xxx=yyy key-value pair when configuring Emitter + // doesn't make any difference. That is why we just log this situation, instead of throwing an exception. + log.info( + "Skipping %s property: one of it's prefixes is also used as a property key. Prefix: %s", + originalProperty, + propertyPrefix + ); + return; + } + Map nestedMap = (Map) nested; + hieraricalPutValue(propertyPrefix, originalProperty, property.substring(dotIndex + 1), value, nestedMap); + } + @VisibleForTesting public static void verifyClazzIsConfigurable(ObjectMapper mapper, Class clazz) { diff --git a/pom.xml b/pom.xml index 2b9231b9ae41..d158f4aba314 100644 --- a/pom.xml +++ b/pom.xml @@ -148,7 +148,7 @@ com.metamx emitter - 0.4.5 + 0.6.0 com.metamx diff --git a/server/src/main/java/io/druid/initialization/Initialization.java b/server/src/main/java/io/druid/initialization/Initialization.java index 53b7e2110af8..67e22c8e299f 100644 --- a/server/src/main/java/io/druid/initialization/Initialization.java +++ b/server/src/main/java/io/druid/initialization/Initialization.java @@ -60,7 +60,7 @@ import io.druid.java.util.common.ISE; import io.druid.java.util.common.logger.Logger; import io.druid.metadata.storage.derby.DerbyMetadataStorageDruidModule; -import io.druid.server.initialization.EmitterModule; +import io.druid.server.emitter.EmitterModule; import io.druid.server.initialization.jetty.JettyServerModule; import io.druid.server.metrics.MetricsModule; import org.apache.commons.io.FileUtils; diff --git a/server/src/main/java/io/druid/server/initialization/ComposingEmitterConfig.java b/server/src/main/java/io/druid/server/emitter/ComposingEmitterConfig.java similarity index 96% rename from server/src/main/java/io/druid/server/initialization/ComposingEmitterConfig.java rename to server/src/main/java/io/druid/server/emitter/ComposingEmitterConfig.java index ca7e3b3f5c90..815558102e33 100644 --- a/server/src/main/java/io/druid/server/initialization/ComposingEmitterConfig.java +++ b/server/src/main/java/io/druid/server/emitter/ComposingEmitterConfig.java @@ -17,7 +17,7 @@ * under the License. */ -package io.druid.server.initialization; +package io.druid.server.emitter; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; diff --git a/server/src/main/java/io/druid/server/initialization/ComposingEmitterModule.java b/server/src/main/java/io/druid/server/emitter/ComposingEmitterModule.java similarity index 98% rename from server/src/main/java/io/druid/server/initialization/ComposingEmitterModule.java rename to server/src/main/java/io/druid/server/emitter/ComposingEmitterModule.java index 584c074f3750..ee014f1d69b9 100644 --- a/server/src/main/java/io/druid/server/initialization/ComposingEmitterModule.java +++ b/server/src/main/java/io/druid/server/emitter/ComposingEmitterModule.java @@ -17,7 +17,7 @@ * under the License. */ -package io.druid.server.initialization; +package io.druid.server.emitter; import com.fasterxml.jackson.databind.Module; import com.google.common.base.Function; diff --git a/server/src/main/java/io/druid/server/initialization/EmitterModule.java b/server/src/main/java/io/druid/server/emitter/EmitterModule.java similarity index 96% rename from server/src/main/java/io/druid/server/initialization/EmitterModule.java rename to server/src/main/java/io/druid/server/emitter/EmitterModule.java index 613a03886be6..9969fef66b27 100644 --- a/server/src/main/java/io/druid/server/initialization/EmitterModule.java +++ b/server/src/main/java/io/druid/server/emitter/EmitterModule.java @@ -17,7 +17,7 @@ * under the License. */ -package io.druid.server.initialization; +package io.druid.server.emitter; import com.google.common.base.Strings; import com.google.common.base.Supplier; @@ -72,6 +72,7 @@ public void configure(Binder binder) binder.install(new NoopEmitterModule()); binder.install(new LogEmitterModule()); binder.install(new HttpEmitterModule()); + binder.install(new ParametrizedUriEmitterModule()); binder.install(new ComposingEmitterModule()); binder.bind(Emitter.class).toProvider(new EmitterProvider(emitterType)).in(LazySingleton.class); @@ -87,6 +88,7 @@ public ServiceEmitter getServiceEmitter(@Self Supplier configSupplier "version", Strings.nullToEmpty(version) // Version is null during `mvn test`. ); + log.info("Underlying emitter for ServiceEmitter: %s", emitter); final ServiceEmitter retVal = new ServiceEmitter( config.getServiceName(), config.getHostAndPortToUse(), diff --git a/server/src/main/java/io/druid/server/initialization/HttpEmitterConfig.java b/server/src/main/java/io/druid/server/emitter/HttpEmitterConfig.java similarity index 90% rename from server/src/main/java/io/druid/server/initialization/HttpEmitterConfig.java rename to server/src/main/java/io/druid/server/emitter/HttpEmitterConfig.java index a27cb6d6c631..55dc4f8a0929 100644 --- a/server/src/main/java/io/druid/server/initialization/HttpEmitterConfig.java +++ b/server/src/main/java/io/druid/server/emitter/HttpEmitterConfig.java @@ -17,7 +17,7 @@ * under the License. */ -package io.druid.server.initialization; +package io.druid.server.emitter; import com.fasterxml.jackson.annotation.JsonProperty; import org.joda.time.Period; @@ -27,10 +27,10 @@ public class HttpEmitterConfig extends com.metamx.emitter.core.HttpEmitterConfig { @JsonProperty - private Period timeOut = new Period("PT5M"); + private Period readTimeout = new Period("PT5M"); public Period getReadTimeout() { - return timeOut; + return readTimeout; } } diff --git a/server/src/main/java/io/druid/server/initialization/HttpEmitterModule.java b/server/src/main/java/io/druid/server/emitter/HttpEmitterModule.java similarity index 96% rename from server/src/main/java/io/druid/server/initialization/HttpEmitterModule.java rename to server/src/main/java/io/druid/server/emitter/HttpEmitterModule.java index 142fb45c7ca0..4dfe7166d57c 100644 --- a/server/src/main/java/io/druid/server/initialization/HttpEmitterModule.java +++ b/server/src/main/java/io/druid/server/emitter/HttpEmitterModule.java @@ -17,7 +17,7 @@ * under the License. */ -package io.druid.server.initialization; +package io.druid.server.emitter; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Supplier; @@ -49,6 +49,11 @@ public void configure(Binder binder) { JsonConfigProvider.bind(binder, "druid.emitter.http", HttpEmitterConfig.class); + configureSsl(binder); + } + + static void configureSsl(Binder binder) + { final SSLContext context; try { context = SSLContext.getDefault(); diff --git a/server/src/main/java/io/druid/server/initialization/LogEmitterModule.java b/server/src/main/java/io/druid/server/emitter/LogEmitterModule.java similarity index 97% rename from server/src/main/java/io/druid/server/initialization/LogEmitterModule.java rename to server/src/main/java/io/druid/server/emitter/LogEmitterModule.java index d2a2e6179020..6372321d70fb 100644 --- a/server/src/main/java/io/druid/server/initialization/LogEmitterModule.java +++ b/server/src/main/java/io/druid/server/emitter/LogEmitterModule.java @@ -17,7 +17,7 @@ * under the License. */ -package io.druid.server.initialization; +package io.druid.server.emitter; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Supplier; diff --git a/server/src/main/java/io/druid/server/initialization/NoopEmitterModule.java b/server/src/main/java/io/druid/server/emitter/NoopEmitterModule.java similarity index 97% rename from server/src/main/java/io/druid/server/initialization/NoopEmitterModule.java rename to server/src/main/java/io/druid/server/emitter/NoopEmitterModule.java index 3b80f060781e..ea5a838fe65d 100644 --- a/server/src/main/java/io/druid/server/initialization/NoopEmitterModule.java +++ b/server/src/main/java/io/druid/server/emitter/NoopEmitterModule.java @@ -17,7 +17,7 @@ * under the License. */ -package io.druid.server.initialization; +package io.druid.server.emitter; import com.google.inject.Binder; import com.google.inject.Module; diff --git a/server/src/main/java/io/druid/server/emitter/ParametrizedUriEmitterConfig.java b/server/src/main/java/io/druid/server/emitter/ParametrizedUriEmitterConfig.java new file mode 100644 index 000000000000..665c6f2fb705 --- /dev/null +++ b/server/src/main/java/io/druid/server/emitter/ParametrizedUriEmitterConfig.java @@ -0,0 +1,34 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.server.emitter; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.joda.time.Period; + +public class ParametrizedUriEmitterConfig extends com.metamx.emitter.core.ParametrizedUriEmitterConfig +{ + @JsonProperty + private Period readTimeout = new Period("PT5M"); + + public Period getReadTimeout() + { + return readTimeout; + } +} diff --git a/server/src/main/java/io/druid/server/emitter/ParametrizedUriEmitterModule.java b/server/src/main/java/io/druid/server/emitter/ParametrizedUriEmitterModule.java new file mode 100644 index 000000000000..7e9ea0946149 --- /dev/null +++ b/server/src/main/java/io/druid/server/emitter/ParametrizedUriEmitterModule.java @@ -0,0 +1,72 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.server.emitter; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.name.Named; +import com.metamx.emitter.core.Emitter; +import com.metamx.emitter.core.ParametrizedUriEmitter; +import com.metamx.http.client.HttpClientConfig; +import com.metamx.http.client.HttpClientInit; +import io.druid.guice.JsonConfigProvider; +import io.druid.guice.ManageLifecycle; +import io.druid.guice.http.LifecycleUtils; +import io.druid.java.util.common.lifecycle.Lifecycle; + +import javax.annotation.Nullable; +import javax.net.ssl.SSLContext; + +public class ParametrizedUriEmitterModule implements Module +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.emitter.parametrized", ParametrizedUriEmitterConfig.class); + HttpEmitterModule.configureSsl(binder); + } + + @Provides + @ManageLifecycle + @Named("parametrized") + public Emitter getEmitter( + Supplier config, + @Nullable SSLContext sslContext, + Lifecycle lifecycle, + ObjectMapper jsonMapper + ) + { + final HttpClientConfig.Builder builder = HttpClientConfig + .builder() + .withNumConnections(1) + .withReadTimeout(config.get().getReadTimeout().toStandardDuration()); + if (sslContext != null) { + builder.withSslContext(sslContext); + } + return new ParametrizedUriEmitter( + config.get(), + HttpClientInit.createClient(builder.build(), LifecycleUtils.asMmxLifecycle(lifecycle)), + jsonMapper + ); + } +} diff --git a/server/src/test/java/io/druid/initialization/ComposingEmitterModuleTest.java b/server/src/test/java/io/druid/initialization/ComposingEmitterModuleTest.java index b3f3ab055215..646f1fa9597f 100644 --- a/server/src/test/java/io/druid/initialization/ComposingEmitterModuleTest.java +++ b/server/src/test/java/io/druid/initialization/ComposingEmitterModuleTest.java @@ -29,8 +29,8 @@ import com.metamx.emitter.core.Emitter; import io.druid.guice.DruidGuiceExtensions; import io.druid.guice.LifecycleModule; -import io.druid.server.initialization.ComposingEmitterConfig; -import io.druid.server.initialization.ComposingEmitterModule; +import io.druid.server.emitter.ComposingEmitterConfig; +import io.druid.server.emitter.ComposingEmitterModule; import org.easymock.EasyMock; import org.junit.Before; import org.junit.Test; diff --git a/server/src/test/java/io/druid/server/emitter/EmitterModuleTest.java b/server/src/test/java/io/druid/server/emitter/EmitterModuleTest.java new file mode 100644 index 000000000000..3f0cc01d35c0 --- /dev/null +++ b/server/src/test/java/io/druid/server/emitter/EmitterModuleTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.server.emitter; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Module; +import com.metamx.emitter.core.Emitter; +import io.druid.guice.DruidGuiceExtensions; +import io.druid.guice.JsonConfigurator; +import io.druid.guice.LazySingleton; +import io.druid.guice.LifecycleModule; +import io.druid.guice.ServerModule; +import org.junit.Test; + +import javax.validation.Validation; +import javax.validation.Validator; +import java.util.Properties; + +public class EmitterModuleTest +{ + + @Test + public void testParametrizedUriEmitterConfig() + { + final Properties props = new Properties(); + props.setProperty("druid.emitter", "parametrized"); + props.setProperty("druid.emitter.parametrized.recipientBaseUrlPattern", "http://example.com:8888/{feed}"); + props.setProperty("druid.emitter.parametrized.httpEmitting.flushMillis", "1"); + props.setProperty("druid.emitter.parametrized.httpEmitting.flushCount", "2"); + props.setProperty("druid.emitter.parametrized.httpEmitting.basicAuthentication", "a:b"); + props.setProperty("druid.emitter.parametrized.httpEmitting.batchingStrategy", "NEWLINES"); + props.setProperty("druid.emitter.parametrized.httpEmitting.maxBatchSize", "4"); + props.setProperty("druid.emitter.parametrized.httpEmitting.maxBufferSize", "8"); + props.setProperty("druid.emitter.parametrized.httpEmitting.flushTimeOut", "1000"); + + final Emitter emitter = + makeInjectorWithProperties(props).getInstance(Emitter.class); + System.out.println(emitter); + } + + private Injector makeInjectorWithProperties(final Properties props) + { + return Guice.createInjector( + ImmutableList.of( + new DruidGuiceExtensions(), + new LifecycleModule(), + new ServerModule(), + new Module() + { + @Override + public void configure(Binder binder) + { + binder.bind(Validator.class).toInstance(Validation.buildDefaultValidatorFactory().getValidator()); + binder.bind(JsonConfigurator.class).in(LazySingleton.class); + binder.bind(Properties.class).toInstance(props); + } + }, + new EmitterModule(props) + ) + ); + } +} From 4ef0ce26c90e117a3ade52b8aae6b302deeeec20 Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 4 Aug 2017 17:40:58 +0300 Subject: [PATCH 2/6] Log created RequestLoggers --- .../log/ComposingRequestLoggerProvider.java | 15 ++++++++++++++- .../druid/server/log/EmittingRequestLogger.java | 9 +++++++++ .../log/EmittingRequestLoggerProvider.java | 7 ++++++- .../io/druid/server/log/FileRequestLogger.java | 8 ++++++++ .../server/log/FileRequestLoggerProvider.java | 7 ++++++- .../log/FilteredRequestLoggerProvider.java | 16 +++++++++++++++- .../druid/server/log/LoggingRequestLogger.java | 9 +++++++++ .../server/log/LoggingRequestLoggerProvider.java | 7 ++++++- .../server/log/NoopRequestLoggerProvider.java | 5 +++++ 9 files changed, 78 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/io/druid/server/log/ComposingRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/ComposingRequestLoggerProvider.java index a40b31f08057..2102742dcc09 100644 --- a/server/src/main/java/io/druid/server/log/ComposingRequestLoggerProvider.java +++ b/server/src/main/java/io/druid/server/log/ComposingRequestLoggerProvider.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Throwables; import com.google.common.collect.Lists; +import io.druid.java.util.common.logger.Logger; import io.druid.server.RequestLogLine; import javax.validation.constraints.NotNull; @@ -35,6 +36,8 @@ @JsonTypeName("composing") public class ComposingRequestLoggerProvider implements RequestLoggerProvider { + private static final Logger log = new Logger(ComposingRequestLoggerProvider.class); + @JsonProperty @NotNull private final List loggerProviders = Lists.newArrayList(); @@ -46,7 +49,9 @@ public RequestLogger get() for (RequestLoggerProvider loggerProvider : loggerProviders) { loggers.add(loggerProvider.get()); } - return new ComposingRequestLogger(loggers); + ComposingRequestLogger logger = new ComposingRequestLogger(loggers); + log.info(new Exception("Stack trace"), "Creating %s at", logger); + return logger; } public static class ComposingRequestLogger implements RequestLogger @@ -79,6 +84,14 @@ public void log(RequestLogLine requestLogLine) throws IOException throw Throwables.propagate(exception); } } + + @Override + public String toString() + { + return "ComposingRequestLogger{" + + "loggers=" + loggers + + '}'; + } } } diff --git a/server/src/main/java/io/druid/server/log/EmittingRequestLogger.java b/server/src/main/java/io/druid/server/log/EmittingRequestLogger.java index c8afc63c61b8..2cb5e5ad6b66 100644 --- a/server/src/main/java/io/druid/server/log/EmittingRequestLogger.java +++ b/server/src/main/java/io/druid/server/log/EmittingRequestLogger.java @@ -50,6 +50,15 @@ public void log(final RequestLogLine requestLogLine) throws IOException emitter.emit(new RequestLogEventBuilder(feed, requestLogLine)); } + @Override + public String toString() + { + return "EmittingRequestLogger{" + + "emitter=" + emitter + + ", feed='" + feed + '\'' + + '}'; + } + private static class RequestLogEvent implements Event { final ImmutableMap serviceDimensions; diff --git a/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java index 5d7f9ef131c5..12afed267b50 100644 --- a/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java +++ b/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java @@ -25,6 +25,7 @@ import com.google.inject.Inject; import com.google.inject.Injector; import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.common.logger.Logger; import javax.validation.constraints.NotNull; @@ -33,6 +34,8 @@ @JsonTypeName("emitter") public class EmittingRequestLoggerProvider implements RequestLoggerProvider { + private static final Logger log = new Logger(EmittingRequestLoggerProvider.class); + @JsonProperty @NotNull private String feed = null; @@ -49,6 +52,8 @@ public void injectMe(Injector injector) @Override public RequestLogger get() { - return new EmittingRequestLogger(emitter, feed); + EmittingRequestLogger logger = new EmittingRequestLogger(emitter, feed); + log.info(new Exception("Stack trace"), "Creating %s at", logger); + return logger; } } diff --git a/server/src/main/java/io/druid/server/log/FileRequestLogger.java b/server/src/main/java/io/druid/server/log/FileRequestLogger.java index 9f29f33e2361..db86c29beb93 100644 --- a/server/src/main/java/io/druid/server/log/FileRequestLogger.java +++ b/server/src/main/java/io/druid/server/log/FileRequestLogger.java @@ -134,4 +134,12 @@ public void log(RequestLogLine requestLogLine) throws IOException fileWriter.flush(); } } + + @Override + public String toString() + { + return "FileRequestLogger{" + + "baseDir=" + baseDir + + '}'; + } } diff --git a/server/src/main/java/io/druid/server/log/FileRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/FileRequestLoggerProvider.java index eed79573ce40..bba12f21e058 100644 --- a/server/src/main/java/io/druid/server/log/FileRequestLoggerProvider.java +++ b/server/src/main/java/io/druid/server/log/FileRequestLoggerProvider.java @@ -26,6 +26,7 @@ import io.druid.guice.annotations.Json; import io.druid.java.util.common.concurrent.ScheduledExecutorFactory; +import io.druid.java.util.common.logger.Logger; import javax.validation.constraints.NotNull; import java.io.File; @@ -35,6 +36,8 @@ @JsonTypeName("file") public class FileRequestLoggerProvider implements RequestLoggerProvider { + private static final Logger log = new Logger(FileRequestLoggerProvider.class); + @JsonProperty @NotNull private File dir = null; @@ -52,6 +55,8 @@ public class FileRequestLoggerProvider implements RequestLoggerProvider @Override public RequestLogger get() { - return new FileRequestLogger(jsonMapper, factory.create(1, "RequestLogger-%s"), dir); + FileRequestLogger logger = new FileRequestLogger(jsonMapper, factory.create(1, "RequestLogger-%s"), dir); + log.info(new Exception("Stack trace"), "Creating %s at", logger); + return logger; } } diff --git a/server/src/main/java/io/druid/server/log/FilteredRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/FilteredRequestLoggerProvider.java index 3e8e39df135e..d9185dcff000 100644 --- a/server/src/main/java/io/druid/server/log/FilteredRequestLoggerProvider.java +++ b/server/src/main/java/io/druid/server/log/FilteredRequestLoggerProvider.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; +import io.druid.java.util.common.logger.Logger; import io.druid.server.RequestLogLine; import javax.validation.constraints.NotNull; @@ -31,6 +32,8 @@ @JsonTypeName("filtered") public class FilteredRequestLoggerProvider implements RequestLoggerProvider { + private static final Logger log = new Logger(FilteredRequestLoggerProvider.class); + @JsonProperty @NotNull private RequestLoggerProvider delegate = null; @@ -41,7 +44,9 @@ public class FilteredRequestLoggerProvider implements RequestLoggerProvider @Override public RequestLogger get() { - return new FilteredRequestLogger(delegate.get(), queryTimeThresholdMs); + FilteredRequestLogger logger = new FilteredRequestLogger(delegate.get(), queryTimeThresholdMs); + log.info(new Exception("Stack trace"), "Creating %s at", logger); + return logger; } public static class FilteredRequestLogger implements RequestLogger @@ -64,6 +69,15 @@ public void log(RequestLogLine requestLogLine) throws IOException logger.log(requestLogLine); } } + + @Override + public String toString() + { + return "FilteredRequestLogger{" + + "queryTimeThresholdMs=" + queryTimeThresholdMs + + ", logger=" + logger + + '}'; + } } } diff --git a/server/src/main/java/io/druid/server/log/LoggingRequestLogger.java b/server/src/main/java/io/druid/server/log/LoggingRequestLogger.java index 08b7e97c92e2..bf7b91a0f1bd 100644 --- a/server/src/main/java/io/druid/server/log/LoggingRequestLogger.java +++ b/server/src/main/java/io/druid/server/log/LoggingRequestLogger.java @@ -103,4 +103,13 @@ public boolean isSetContextMDC() { return setContextMDC; } + + @Override + public String toString() + { + return "LoggingRequestLogger{" + + "setMDC=" + setMDC + + ", setContextMDC=" + setContextMDC + + '}'; + } } diff --git a/server/src/main/java/io/druid/server/log/LoggingRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/LoggingRequestLoggerProvider.java index da4a8f25ad4a..28fb360d26e6 100644 --- a/server/src/main/java/io/druid/server/log/LoggingRequestLoggerProvider.java +++ b/server/src/main/java/io/druid/server/log/LoggingRequestLoggerProvider.java @@ -24,10 +24,13 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.fasterxml.jackson.databind.ObjectMapper; import io.druid.guice.annotations.Json; +import io.druid.java.util.common.logger.Logger; @JsonTypeName("slf4j") public class LoggingRequestLoggerProvider implements RequestLoggerProvider { + private static final Logger log = new Logger(LoggingRequestLoggerProvider.class); + @JacksonInject @Json public ObjectMapper mapper; @@ -41,6 +44,8 @@ public class LoggingRequestLoggerProvider implements RequestLoggerProvider @Override public RequestLogger get() { - return new LoggingRequestLogger(mapper, setMDC, setContextMDC); + LoggingRequestLogger logger = new LoggingRequestLogger(mapper, setMDC, setContextMDC); + log.info(new Exception("Stack trace"), "Creating %s at", logger); + return logger; } } diff --git a/server/src/main/java/io/druid/server/log/NoopRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/NoopRequestLoggerProvider.java index 7fa04b6ca9f9..9a2da030076f 100644 --- a/server/src/main/java/io/druid/server/log/NoopRequestLoggerProvider.java +++ b/server/src/main/java/io/druid/server/log/NoopRequestLoggerProvider.java @@ -19,13 +19,18 @@ package io.druid.server.log; +import io.druid.java.util.common.logger.Logger; + /** */ public class NoopRequestLoggerProvider implements RequestLoggerProvider { + private static final Logger log = new Logger(NoopRequestLoggerProvider.class); + @Override public RequestLogger get() { + log.info(new Exception("Stack trace"), "Creating NoopRequestLogger at"); return new NoopRequestLogger(); } } From d9daca8b23f514e4ad201d44b497885a2325b5e3 Mon Sep 17 00:00:00 2001 From: leventov Date: Sat, 26 Aug 2017 01:33:03 -0500 Subject: [PATCH 3/6] Fix forbidden API --- api/src/main/java/io/druid/guice/JsonConfigurator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/io/druid/guice/JsonConfigurator.java b/api/src/main/java/io/druid/guice/JsonConfigurator.java index 9474cc66e098..f6d766dd9551 100644 --- a/api/src/main/java/io/druid/guice/JsonConfigurator.java +++ b/api/src/main/java/io/druid/guice/JsonConfigurator.java @@ -180,10 +180,10 @@ private static void hieraricalPutValue( return; } if (dotIndex == 0) { - throw new ProvisionException(String.format("Double dot in property: %s", originalProperty)); + throw new ProvisionException(StringUtils.format("Double dot in property: %s", originalProperty)); } if (dotIndex == property.length() - 1) { - throw new ProvisionException(String.format("Dot at the end of property: %s", originalProperty)); + throw new ProvisionException(StringUtils.format("Dot at the end of property: %s", originalProperty)); } String nestedKey = property.substring(0, dotIndex); Object nested = targetMap.computeIfAbsent(nestedKey, k -> new HashMap()); From 1770ea98cc3e4f875fb71cb2d5a8e3b01055401f Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 12 Sep 2017 11:32:37 -0500 Subject: [PATCH 4/6] Test fix --- .../test/java/io/druid/server/emitter/EmitterModuleTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/io/druid/server/emitter/EmitterModuleTest.java b/server/src/test/java/io/druid/server/emitter/EmitterModuleTest.java index 3f0cc01d35c0..d693d989d956 100644 --- a/server/src/test/java/io/druid/server/emitter/EmitterModuleTest.java +++ b/server/src/test/java/io/druid/server/emitter/EmitterModuleTest.java @@ -25,11 +25,13 @@ import com.google.inject.Injector; import com.google.inject.Module; import com.metamx.emitter.core.Emitter; +import com.metamx.emitter.core.ParametrizedUriEmitter; import io.druid.guice.DruidGuiceExtensions; import io.druid.guice.JsonConfigurator; import io.druid.guice.LazySingleton; import io.druid.guice.LifecycleModule; import io.druid.guice.ServerModule; +import org.junit.Assert; import org.junit.Test; import javax.validation.Validation; @@ -55,7 +57,8 @@ public void testParametrizedUriEmitterConfig() final Emitter emitter = makeInjectorWithProperties(props).getInstance(Emitter.class); - System.out.println(emitter); + // Testing that ParametrizedUriEmitter is successfully deserialized from the above config + Assert.assertTrue(emitter instanceof ParametrizedUriEmitter); } private Injector makeInjectorWithProperties(final Properties props) From c4cb602d42e5a3821b60cc2c0c6b6b2a5a4e5062 Mon Sep 17 00:00:00 2001 From: leventov Date: Tue, 12 Sep 2017 11:57:48 -0500 Subject: [PATCH 5/6] More Http and Parametrized Http Emitter docs --- docs/content/configuration/index.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/content/configuration/index.md b/docs/content/configuration/index.md index f17f6858a014..2204b1e381cc 100644 --- a/docs/content/configuration/index.md +++ b/docs/content/configuration/index.md @@ -204,7 +204,7 @@ The Druid servers [emit various metrics](../operations/metrics.html) and alerts |Property|Description|Default| |--------|-----------|-------| -|`druid.emitter`|Setting this value to "noop", "logging", or "http" will initialize one of the emitter modules. value "composing" can be used to initialize multiple emitter modules. |noop| +|`druid.emitter`|Setting this value to "noop", "logging", "http" or "parametrized" will initialize one of the emitter modules. value "composing" can be used to initialize multiple emitter modules. |noop| #### Logging Emitter Module @@ -217,10 +217,26 @@ The Druid servers [emit various metrics](../operations/metrics.html) and alerts |Property|Description|Default| |--------|-----------|-------| -|`druid.emitter.http.timeOut`|The timeout for data reads.|PT5M| +|`druid.emitter.http.readTimeout`|The timeout for data reads.|PT5M| |`druid.emitter.http.flushMillis`|How often the internal message buffer is flushed (data is sent).|60000| |`druid.emitter.http.flushCount`|How many messages the internal message buffer can hold before flushing (sending).|500| -|`druid.emitter.http.recipientBaseUrl`|The base URL to emit messages to. Druid will POST JSON to be consumed at the HTTP endpoint specified by this property.|none| +|`druid.emitter.http.basicAuthentication`|Login and password for authentification in "login:password" form, e. g. `druid.emitter.http.basicAuthentication=admin:adminpassword`|not specified = no authentification| +|`druid.emitter.http.flushTimeOut|The timeout after which an event should be sent to the endpoint, even if internal buffers are not filled, in milliseconds.|not specified = no timeout| +|`druid.emitter.http.batchingStrategy`|The strategy of how the batch is formatted. "ARRAY" means `[event1,event2]`, "NEWLINES" means `event1\nevent2`, ONLY_EVENTS means `event1event2`.|ARRAY| +|`druid.emitter.http.maxBatchSize`|The maximum batch size, in bytes.|5191680 (i. e. 5 MB)| +|`druid.emitter.http.recipientBaseUrl`|The base URL to emit messages to. Druid will POST JSON to be consumed at the HTTP endpoint specified by this property.|none, required config| + +#### Parametrized Http Emitter Module + +`druid.emitter.parametrized.httpEmitting.*` configs correspond to the configs of Http Emitter Modules, see above. +Except `readTimeout` and `recipientBaseUrl`. E. g. `druid.emitter.parametrized.httpEmitting.flushMillis`, +`druid.emitter.parametrized.httpEmitting.flushCount`, etc. + +The additional configs are: +|Property|Description|Default| +|--------|-----------|-------| +|`druid.emitter.parametrized.readTimeout`|The timeout for data reads.|PT5M| +|`druid.emitter.parametrized.recipientBaseUrlPattern`|The URL pattern to send an event to, based on the event's feed. E. g. `http://foo.bar/{feed}`, that will send event to `http://foo.bar/metrics` if the event's feed is "metrics".|none, required config| #### Composing Emitter Module From 0830d522f1fe2e8708d26162cf955fda337a7fd0 Mon Sep 17 00:00:00 2001 From: leventov Date: Wed, 13 Sep 2017 16:15:45 -0500 Subject: [PATCH 6/6] Switch to debug level --- .../io/druid/server/log/ComposingRequestLoggerProvider.java | 2 +- .../java/io/druid/server/log/EmittingRequestLoggerProvider.java | 2 +- .../java/io/druid/server/log/FileRequestLoggerProvider.java | 2 +- .../java/io/druid/server/log/FilteredRequestLoggerProvider.java | 2 +- .../java/io/druid/server/log/LoggingRequestLoggerProvider.java | 2 +- .../java/io/druid/server/log/NoopRequestLoggerProvider.java | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/io/druid/server/log/ComposingRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/ComposingRequestLoggerProvider.java index 2102742dcc09..a0710dfaf277 100644 --- a/server/src/main/java/io/druid/server/log/ComposingRequestLoggerProvider.java +++ b/server/src/main/java/io/druid/server/log/ComposingRequestLoggerProvider.java @@ -50,7 +50,7 @@ public RequestLogger get() loggers.add(loggerProvider.get()); } ComposingRequestLogger logger = new ComposingRequestLogger(loggers); - log.info(new Exception("Stack trace"), "Creating %s at", logger); + log.debug(new Exception("Stack trace"), "Creating %s at", logger); return logger; } diff --git a/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java index 12afed267b50..534a744e42d0 100644 --- a/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java +++ b/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java @@ -53,7 +53,7 @@ public void injectMe(Injector injector) public RequestLogger get() { EmittingRequestLogger logger = new EmittingRequestLogger(emitter, feed); - log.info(new Exception("Stack trace"), "Creating %s at", logger); + log.debug(new Exception("Stack trace"), "Creating %s at", logger); return logger; } } diff --git a/server/src/main/java/io/druid/server/log/FileRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/FileRequestLoggerProvider.java index bba12f21e058..760017e68e10 100644 --- a/server/src/main/java/io/druid/server/log/FileRequestLoggerProvider.java +++ b/server/src/main/java/io/druid/server/log/FileRequestLoggerProvider.java @@ -56,7 +56,7 @@ public class FileRequestLoggerProvider implements RequestLoggerProvider public RequestLogger get() { FileRequestLogger logger = new FileRequestLogger(jsonMapper, factory.create(1, "RequestLogger-%s"), dir); - log.info(new Exception("Stack trace"), "Creating %s at", logger); + log.debug(new Exception("Stack trace"), "Creating %s at", logger); return logger; } } diff --git a/server/src/main/java/io/druid/server/log/FilteredRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/FilteredRequestLoggerProvider.java index d9185dcff000..8d632b4fe0af 100644 --- a/server/src/main/java/io/druid/server/log/FilteredRequestLoggerProvider.java +++ b/server/src/main/java/io/druid/server/log/FilteredRequestLoggerProvider.java @@ -45,7 +45,7 @@ public class FilteredRequestLoggerProvider implements RequestLoggerProvider public RequestLogger get() { FilteredRequestLogger logger = new FilteredRequestLogger(delegate.get(), queryTimeThresholdMs); - log.info(new Exception("Stack trace"), "Creating %s at", logger); + log.debug(new Exception("Stack trace"), "Creating %s at", logger); return logger; } diff --git a/server/src/main/java/io/druid/server/log/LoggingRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/LoggingRequestLoggerProvider.java index 28fb360d26e6..3918d92440ba 100644 --- a/server/src/main/java/io/druid/server/log/LoggingRequestLoggerProvider.java +++ b/server/src/main/java/io/druid/server/log/LoggingRequestLoggerProvider.java @@ -45,7 +45,7 @@ public class LoggingRequestLoggerProvider implements RequestLoggerProvider public RequestLogger get() { LoggingRequestLogger logger = new LoggingRequestLogger(mapper, setMDC, setContextMDC); - log.info(new Exception("Stack trace"), "Creating %s at", logger); + log.debug(new Exception("Stack trace"), "Creating %s at", logger); return logger; } } diff --git a/server/src/main/java/io/druid/server/log/NoopRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/NoopRequestLoggerProvider.java index 9a2da030076f..e68394e233ed 100644 --- a/server/src/main/java/io/druid/server/log/NoopRequestLoggerProvider.java +++ b/server/src/main/java/io/druid/server/log/NoopRequestLoggerProvider.java @@ -30,7 +30,7 @@ public class NoopRequestLoggerProvider implements RequestLoggerProvider @Override public RequestLogger get() { - log.info(new Exception("Stack trace"), "Creating NoopRequestLogger at"); + log.debug(new Exception("Stack trace"), "Creating NoopRequestLogger at"); return new NoopRequestLogger(); } }