From 4f22c6520555515edad841a760b4d651be99d513 Mon Sep 17 00:00:00 2001 From: "rohit.k" Date: Sun, 23 Apr 2017 19:37:07 +0530 Subject: [PATCH 1/2] Implementing dropwizard emitter for druid --- .../extensions-contrib/dropwizard.md | 87 ++++++ extensions-contrib/dropwizard-emitter/pom.xml | 91 +++++++ .../emitter/dropwizard/DropwizardEmitter.java | 108 ++++++++ .../dropwizard/DropwizardEmitterConfig.java | 90 +++++++ .../dropwizard/DropwizardEmitterModule.java | 70 +++++ .../emitter/dropwizard/DropwizardEvent.java | 38 +++ .../dropwizard/DropwizardMetricManager.java | 32 +++ .../dropwizard/DropwizardReporter.java | 39 +++ .../DruidToDropwizardEventConverter.java | 43 +++ .../dropwizard/HistogramMetricManager.java | 46 ++++ .../SendAllDropwizardEventConverter.java | 132 +++++++++ .../WhiteListBasedConverter.java | 253 ++++++++++++++++++ .../reporters/DropwizardConsoleReporter.java | 120 +++++++++ .../reporters/DropwizardJMXReporter.java | 46 ++++ .../DropwizardEmitterConfigTest.java | 63 +++++ 15 files changed, 1258 insertions(+) create mode 100644 docs/content/development/extensions-contrib/dropwizard.md create mode 100644 extensions-contrib/dropwizard-emitter/pom.xml create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterConfig.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterModule.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEvent.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardMetricManager.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardReporter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DruidToDropwizardEventConverter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/HistogramMetricManager.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/SendAllDropwizardEventConverter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/WhiteListBasedConverter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java create mode 100644 extensions-contrib/dropwizard-emitter/src/test/java/io/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java diff --git a/docs/content/development/extensions-contrib/dropwizard.md b/docs/content/development/extensions-contrib/dropwizard.md new file mode 100644 index 000000000000..18b2db037f9f --- /dev/null +++ b/docs/content/development/extensions-contrib/dropwizard.md @@ -0,0 +1,87 @@ +--- +layout: doc_page +--- + +# Dropwizard Emitter + +To use this extension, make sure to [include](../../operations/including-extensions.html) `dropwizard-emitter` extension. + +## Introduction + +The intent of this extension is to integrate [Dropwizard](http://metrics.dropwizard.io/3.1.0/getting-started/#); metrics library with druid so that dropwizard users can easily absorb druid into their monitoring ecosystem. +It accumulates druid metrics in a dropwizard histogram and emits them to various sinks via dropwizard supported reporters. +A histogram measures the statistical distribution of values in a stream of data. In addition to minimum, maximum, mean, etc., it also measures median, 75th, 90th, 95th, 98th, 99th, and 99.9th percentiles. +Currently dropwizard metrics can be emitted to these sinks: +Console,HTTP,JMX,Graphite,CSV,Slf4jLogger,Ganglia and various other community supported [sinks](http://metrics.dropwizard.io/3.1.0/manual/third-party/);. + +## Configuration + +All the configuration parameters for Dropwizard emitter are under `druid.emitter.dropwizard`. + +|property|description|required?|default| +|--------|-----------|---------|-------| +|`druid.emitter.dropwizard.metric`|The metric manager to be used.Currently supported metric manager is histogram|no|histogram| +|`druid.emitter.dropwizard.reporter`|The dropwizard reporter to be used.|yes|none| +|`druid.emitter.dropwizard.eventConverter`| Filter and converter of druid events to dropwizard event(please see next section). |yes|none| +|`druid.emitter.dropwizard.alertEmitters`| List of emitters where alerts will be forwarded to. |no| empty list (no forwarding)| + + +### Druid to Dropwizard Event Converter + +Dropwizard Event Converter defines a mapping between druid metrics name plus dimensions to a Dropwizard metric name. +Dropwizard metric name is organized using the following schema: +`.[].[]..` +Properly naming the metrics is critical to avoid conflicts, confusing data and potentially wrong interpretation later on. + +Example `druid.historical.abc_com:8080.MyDataSourceName.GroupBy.query/time`: + + * `druid` -> namespace prefix + * `historical` -> service name + * `abc.com:8080` -> druid hostname + * `MyDataSourceName` -> dimension value + * `GroupBy` -> dimension value + * `query/time` -> metric name + +We have two different implementation of event converter: + +#### Send-All converter + +The first implementation called `all`, will send all the druid service metrics events. +The metric name will be in the form `.[].[]..` +User has control of `.[].[].` + +You can omit the hostname by setting `ignoreHostname=true` +`druid.SERVICE_NAME.dataSourceName.queryType.query.time` + +You can omit the service name by setting `ignoreServiceName=true` +`druid.HOSTNAME.dataSourceName.queryType.query.time` + +```json + +druid.emitter.dropwizard.eventConverter={"type":"all", "namespacePrefix": "druid.test", "ignoreHostname":true, "ignoreServiceName":true} + +``` + +#### White-list based converter + +The second implementation called `whiteList`, will send only the white listed metrics and dimensions. +Same as for the `all` converter user has control of `.[].[].` +White-list based converter comes with the following default white list map located under resources in `./src/main/resources/defaultWhiteListMap.json` + +Although user can override the default white list map by supplying a property called `mapPath`. +This property is a String containing the path for the file containing **white list map Json object**. +For example the following converter will read the map from the file `/pathPrefix/fileName.json`. + +```json + +druid.emitter.dropwizard.eventConverter={"type":"whiteList", "namespacePrefix": "druid.test", "ignoreHostname":true, "ignoreServiceName":true, "mapPath":"/pathPrefix/fileName.json"} + +``` + +**Druid emits a huge number of metrics we highly recommend to use the `whiteList` converter** + +### Metric Manager + +Metric manager defines the dropwizard accumulator that would be used to accumulate druid metric events. +For eg : Gauge,Counter,Histogram,Meter etc. + diff --git a/extensions-contrib/dropwizard-emitter/pom.xml b/extensions-contrib/dropwizard-emitter/pom.xml new file mode 100644 index 000000000000..e0e1ebd3944a --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/pom.xml @@ -0,0 +1,91 @@ + + + + + 4.0.0 + + + io.druid + druid + 0.10.0-SNAPSHOT + ../../pom.xml + + + io.druid.extensions.contrib + dropwizard-emitter + dropwizard-emitter + Druid emitter extension to convert druid metric to Dropwizard metrics + + + + io.druid + druid-common + ${project.parent.version} + provided + + + io.druid + druid-api + ${project.parent.version} + provided + + + com.metamx + emitter + provided + + + junit + junit + test + + + org.easymock + easymock + test + + + pl.pragmatists + JUnitParams + 1.0.4 + test + + + io.druid + druid-server + ${project.parent.version} + test-jar + test + + + io.druid + druid-processing + ${project.parent.version} + test-jar + test + + + io.dropwizard.metrics + metrics-core + 3.1.0 + + + diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitter.java new file mode 100644 index 000000000000..153c03bbf071 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitter.java @@ -0,0 +1,108 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard; + +import com.codahale.metrics.MetricRegistry; +import com.metamx.emitter.core.Emitter; +import com.metamx.emitter.core.Event; +import com.metamx.emitter.service.AlertEvent; +import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.common.ISE; +import io.druid.java.util.common.logger.Logger; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.regex.Pattern; + +public class DropwizardEmitter implements Emitter { + private static Logger log = new Logger(DropwizardEmitter.class); + private final MetricRegistry metricsRegistry = new MetricRegistry(); + private final DropwizardMetricManager dropwizardMetricManager; + private final AtomicBoolean started = new AtomicBoolean(false); + private final DruidToDropwizardEventConverter druidToDropwizardEventConverter; + private final List emitterList; + //TODO make this a list of reporters + private final DropwizardReporter dropwizardReporter; + + public DropwizardEmitter(DropwizardEmitterConfig dropwizardEmitterConfig,List emitterList) { + this.dropwizardMetricManager = dropwizardEmitterConfig.getDropwizardMetricManager(); + this.druidToDropwizardEventConverter = dropwizardEmitterConfig.getDruidToDropwizardEventConverter(); + this.emitterList = emitterList; + this.dropwizardReporter = dropwizardEmitterConfig.getDropwizardReporter(); + } + + @Override + public void start() { + try { + startReporters(); + } catch (IOException e) { + log.error(e,"Error while starting Dropwizard reporters"); + } + started.set(true); + } + + @Override + public void emit(Event event) { + if (!started.get()) { + throw new ISE("Emit was called while emitter is yet not initialized"); + } + if (event instanceof ServiceMetricEvent) { + DropwizardEvent dropwizardEvent = druidToDropwizardEventConverter.druidEventToDropwizard((ServiceMetricEvent) event); + if(dropwizardEvent!=null) { + dropwizardMetricManager.updateMetric(metricsRegistry,dropwizardEvent); + }else{ + log.debug("Dropping the service event "+event); + return; + } + } else if (!emitterList.isEmpty() && event instanceof AlertEvent) { + for (Emitter emitter : emitterList) { + emitter.emit(event); + } + } else if (event instanceof AlertEvent) { + AlertEvent alertEvent = (AlertEvent) event; + log.error( + "The following alert is dropped, description is [%s], severity is [%s]", + alertEvent.getDescription(), alertEvent.getSeverity() + ); + } else { + log.error("unknown event type [%s]", event.getClass()); + } + } + + @Override + public void flush() throws IOException { + dropwizardReporter.flush(); + } + + @Override + public void close() throws IOException { + dropwizardReporter.close(); + } + + private void startReporters() throws IOException { + dropwizardReporter.start(metricsRegistry); + } + + public static String sanitize(String namespace) + { + Pattern DOT_OR_WHITESPACE = Pattern.compile("[\\s]+|[.]+"); + return DOT_OR_WHITESPACE.matcher(namespace).replaceAll("_"); + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterConfig.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterConfig.java new file mode 100644 index 000000000000..4fd745e1e8e7 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterConfig.java @@ -0,0 +1,90 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + + +public class DropwizardEmitterConfig { + @JsonProperty("eventConverter") + final private DruidToDropwizardEventConverter druidToDropwizardEventConverter; + @JsonProperty("reporter") + final private DropwizardReporter dropwizardReporter; + @JsonProperty("metric") + final private DropwizardMetricManager dropwizardMetricManager; + @JsonProperty + final private List alertEmitters; + + @JsonCreator + public DropwizardEmitterConfig(@JsonProperty("reporter") DropwizardReporter dropwizardReporter ,@JsonProperty("eventConverter") DruidToDropwizardEventConverter druidToDropwizardEventConverter,@JsonProperty("metric") DropwizardMetricManager dropwizardMetricManager,@JsonProperty("alertEmitters") List alertEmitters ) { + this.dropwizardReporter = dropwizardReporter; + this.druidToDropwizardEventConverter = druidToDropwizardEventConverter; + this.dropwizardMetricManager = dropwizardMetricManager==null? new HistogramMetricManager():dropwizardMetricManager; + this.alertEmitters = alertEmitters; + } + + + public DruidToDropwizardEventConverter getDruidToDropwizardEventConverter() { + return druidToDropwizardEventConverter; + } + + public DropwizardReporter getDropwizardReporter() { + return dropwizardReporter; + } + + public List getAlertEmitters() { + return alertEmitters; + } + + public DropwizardMetricManager getDropwizardMetricManager() { + return dropwizardMetricManager; + } + + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + DropwizardEmitterConfig that = (DropwizardEmitterConfig) o; + + if (alertEmitters != null ? !alertEmitters.equals(that.alertEmitters) : that.alertEmitters != null) + return false; + if (dropwizardMetricManager != null ? !dropwizardMetricManager.equals(that.dropwizardMetricManager) : that.dropwizardMetricManager != null) + return false; + if (dropwizardReporter != null ? !dropwizardReporter.equals(that.dropwizardReporter) : that.dropwizardReporter != null) + return false; + if (druidToDropwizardEventConverter != null ? !druidToDropwizardEventConverter.equals(that.druidToDropwizardEventConverter) : that.druidToDropwizardEventConverter != null) + return false; + + return true; + } + + @Override + public int hashCode() { + int result = druidToDropwizardEventConverter != null ? druidToDropwizardEventConverter.hashCode() : 0; + result = 31 * result + (dropwizardReporter != null ? dropwizardReporter.hashCode() : 0); + result = 31 * result + (dropwizardMetricManager != null ? dropwizardMetricManager.hashCode() : 0); + result = 31 * result + (alertEmitters != null ? alertEmitters.hashCode() : 0); + return result; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterModule.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterModule.java new file mode 100644 index 000000000000..769acc3e0b9b --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterModule.java @@ -0,0 +1,70 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; +import com.google.common.collect.Lists; +import com.google.inject.Binder; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Provides; +import com.google.inject.name.Named; +import com.google.inject.name.Names; +import com.metamx.emitter.core.Emitter; +import io.druid.guice.JsonConfigProvider; +import io.druid.guice.ManageLifecycle; +import io.druid.initialization.DruidModule; + +import java.util.Collections; +import java.util.List; + +public class DropwizardEmitterModule implements DruidModule +{ + private static final String EMITTER_TYPE = "dropwizard"; + @Override + public List getJacksonModules() + { + return Collections.EMPTY_LIST; + } + + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.emitter." + EMITTER_TYPE, DropwizardEmitterConfig.class); + + } + + @Provides + @ManageLifecycle + @Named(EMITTER_TYPE) + public Emitter getEmitter(DropwizardEmitterConfig dropwizardEmitterConfig, ObjectMapper mapper, final Injector injector){ + List emitters = Lists.transform( + dropwizardEmitterConfig.getAlertEmitters(), + new Function() { + @Override + public Emitter apply(String s) { + return injector.getInstance(Key.get(Emitter.class, Names.named(s))); + } + } + ); + return new DropwizardEmitter(dropwizardEmitterConfig, emitters); + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEvent.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEvent.java new file mode 100644 index 000000000000..c32e9a8ab431 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEvent.java @@ -0,0 +1,38 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard; + + +public class DropwizardEvent { + private final String metricName; + private final Double value; + + public DropwizardEvent(String metricName, Double value) { + this.metricName = metricName; + this.value = value; + } + + public String getMetricName() { + return metricName; + } + + public Double getValue() { + return value; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardMetricManager.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardMetricManager.java new file mode 100644 index 000000000000..d7a7fcf92210 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardMetricManager.java @@ -0,0 +1,32 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard; + +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = HistogramMetricManager.class) +@JsonSubTypes(value= { + @JsonSubTypes.Type(name="histogram", value = HistogramMetricManager.class) +}) + +public interface DropwizardMetricManager { + public void updateMetric(MetricRegistry metricsRegistry,DropwizardEvent dropwizardEvent); +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardReporter.java new file mode 100644 index 000000000000..b2f39a65428f --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardReporter.java @@ -0,0 +1,39 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard; + +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.druid.emitter.dropwizard.reporters.DropwizardConsoleReporter; +import io.druid.emitter.dropwizard.reporters.DropwizardJMXReporter; + +import java.io.IOException; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes(value= { + @JsonSubTypes.Type(name="console", value = DropwizardConsoleReporter.class), + @JsonSubTypes.Type(name="jmx", value = DropwizardJMXReporter.class), +}) +public interface DropwizardReporter { + + public void start(MetricRegistry metricRegistry) throws IOException; + public void flush() throws IOException; + public void close() throws IOException; +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DruidToDropwizardEventConverter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DruidToDropwizardEventConverter.java new file mode 100644 index 000000000000..9042acb7bb9b --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DruidToDropwizardEventConverter.java @@ -0,0 +1,43 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.emitter.dropwizard.eventconverters.SendAllDropwizardEventConverter; +import io.druid.emitter.dropwizard.eventconverters.WhiteListBasedConverter; + + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = SendAllDropwizardEventConverter.class) +@JsonSubTypes(value= { + @JsonSubTypes.Type(name="all", value = SendAllDropwizardEventConverter.class), + @JsonSubTypes.Type(name="whiteList", value = WhiteListBasedConverter.class) +}) +public interface DruidToDropwizardEventConverter { + /** + * This function acts as a filter. It returns null if the event is not suppose to be emitted to Dropwizard + * Also This function will define the conversion between the druid event dimension's values and Dropwizard's metric name + * + * @param serviceMetricEvent Druid event ot type {@link com.metamx.emitter.service.ServiceMetricEvent} + * + * @return {@link io.druid.emitter.dropwizard.DropwizardEvent} or null + */ + DropwizardEvent druidEventToDropwizard(ServiceMetricEvent serviceMetricEvent); +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/HistogramMetricManager.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/HistogramMetricManager.java new file mode 100644 index 000000000000..4ea72b5ce886 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/HistogramMetricManager.java @@ -0,0 +1,46 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard; + +import com.codahale.metrics.MetricRegistry; +import io.druid.java.util.common.logger.Logger; +import io.druid.math.expr.Expr; + + +public class HistogramMetricManager implements DropwizardMetricManager { + private static Logger log = new Logger(HistogramMetricManager.class); + @Override + public void updateMetric(MetricRegistry metricsRegistry,DropwizardEvent dropwizardEvent) { + if(dropwizardEvent==null){ + log.error("Dropwizard event passed to " + getClass() + " is null"); + return; + } + metricsRegistry.histogram(dropwizardEvent.getMetricName()).update(dropwizardEvent.getValue().longValue()); + } + + @Override + public boolean equals(Object o){ + return true; + } + + @Override + public int hashCode() { + return 1; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/SendAllDropwizardEventConverter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/SendAllDropwizardEventConverter.java new file mode 100644 index 000000000000..dce204f15a7e --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/SendAllDropwizardEventConverter.java @@ -0,0 +1,132 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard.eventconverters; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSortedSet; +import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.emitter.dropwizard.DropwizardEmitter; +import io.druid.emitter.dropwizard.DropwizardEvent; +import io.druid.emitter.dropwizard.DruidToDropwizardEventConverter; + + +public class SendAllDropwizardEventConverter implements DruidToDropwizardEventConverter { + @JsonProperty + private boolean ignoreHostname; + @JsonProperty + private boolean ignoreServiceName; + @JsonProperty + private final String namespacePrefix; + + public String getNamespacePrefix() { + return namespacePrefix; + } + + @JsonCreator + public SendAllDropwizardEventConverter( + @JsonProperty("namespacePrefix") String namespacePrefix, + @JsonProperty("ignoreHostname") Boolean ignoreHostname, + @JsonProperty("ignoreServiceName") Boolean ignoreServiceName + ) + { + this.ignoreHostname = ignoreHostname == null ? false : ignoreHostname; + this.ignoreServiceName = ignoreServiceName == null ? false : ignoreServiceName; + this.namespacePrefix = namespacePrefix; + } + + @JsonProperty + public void setIgnoreHostname(boolean ignoreHostname) { + this.ignoreHostname = ignoreHostname; + } + + @JsonProperty + public void setIgnoreServiceName(boolean ignoreServiceName) { + this.ignoreServiceName = ignoreServiceName; + } + + @JsonProperty + public boolean isIgnoreServiceName() + { + return ignoreServiceName; + } + + @JsonProperty + public boolean isIgnoreHostname() + { + return ignoreHostname; + } + + @Override + public DropwizardEvent druidEventToDropwizard(ServiceMetricEvent serviceMetricEvent) { + ImmutableList.Builder metricPathBuilder = new ImmutableList.Builder(); + + if (!Strings.isNullOrEmpty(this.getNamespacePrefix())) + metricPathBuilder.add(this.getNamespacePrefix()); + + if (!this.isIgnoreServiceName()) { + metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getService())); + } + if (!this.isIgnoreHostname()) { + metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getHost())); + } + + ImmutableSortedSet dimNames = ImmutableSortedSet.copyOf(serviceMetricEvent.getUserDims().keySet()); + for (String dimName : dimNames) { + metricPathBuilder.add(DropwizardEmitter.sanitize(String.valueOf(serviceMetricEvent.getUserDims() + .get(dimName)))); + } + metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getMetric())); + + return new DropwizardEvent(Joiner.on(".").join(metricPathBuilder.build()), + serviceMetricEvent.getValue().doubleValue()); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()){ + return false; + } + + SendAllDropwizardEventConverter that = (SendAllDropwizardEventConverter) o; + + if (ignoreHostname != that.ignoreHostname) { + return false; + } + if (ignoreServiceName != that.ignoreServiceName) { + return false; + } + + return true; + } + + @Override + public int hashCode() { + int result = (ignoreHostname ? 1 : 0); + result = 31 * result + (ignoreServiceName ? 1 : 0); + return result; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/WhiteListBasedConverter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/WhiteListBasedConverter.java new file mode 100644 index 000000000000..59dfa6d16bb4 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/eventconverters/WhiteListBasedConverter.java @@ -0,0 +1,253 @@ +/* +* Licensed to Metamarkets Group Inc. (Metamarkets) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. Metamarkets licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +package io.druid.emitter.dropwizard.eventconverters; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedMap; +import com.google.common.io.Files; +import com.google.common.io.Resources; +import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.emitter.dropwizard.DropwizardEmitter; +import io.druid.emitter.dropwizard.DropwizardEvent; +import io.druid.emitter.dropwizard.DruidToDropwizardEventConverter; +import io.druid.java.util.common.ISE; +import io.druid.java.util.common.logger.Logger; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.nio.charset.Charset; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.SortedMap; +import java.util.concurrent.TimeUnit; + +@JsonTypeName("whiteList") +public class WhiteListBasedConverter implements DruidToDropwizardEventConverter { + private static final Logger LOGGER = new Logger(WhiteListBasedConverter.class); + /** + * @code whiteListDimsMapper is a white list of metric->dimensions mappings. + * Key is the metric name or the metric's prefix. + * Value is a list of metric's dimensions names. + * The order of the dimension name is important, it will be used to build the dropwizard metric nanme. + * For instance we have dimension type is nested under dimension dataSource -> prefix.dataSource.queryType.metricName + */ + private final ImmutableSortedMap> whiteListDimsMapper; + + @JsonProperty + private final boolean ignoreHostname; + + @JsonProperty + private final boolean ignoreServiceName; + + @JsonProperty + private final String namespacePrefix; + + @JsonProperty + private final String mapPath; + + private final ObjectMapper mapper; + + @JsonCreator + public WhiteListBasedConverter( + @JsonProperty("namespacePrefix") String namespacePrefix, + @JsonProperty("ignoreHostname") Boolean ignoreHostname, + @JsonProperty("ignoreServiceName") Boolean ignoreServiceName, + @JsonProperty("mapPath") String mapPath, + @JacksonInject ObjectMapper mapper + ) { + this.mapper = Preconditions.checkNotNull(mapper); + this.mapPath = mapPath; + this.whiteListDimsMapper = readMap(this.mapPath); + this.ignoreHostname = ignoreHostname == null ? false : ignoreHostname; + this.ignoreServiceName = ignoreServiceName == null ? false : ignoreServiceName; + this.namespacePrefix = namespacePrefix; + } + + @JsonProperty + public boolean isIgnoreHostname() { + return ignoreHostname; + } + + @JsonProperty + public boolean isIgnoreServiceName() { + return ignoreServiceName; + } + + @JsonProperty + public String getNamespacePrefix() { + return namespacePrefix; + } + + public ImmutableSortedMap> getWhiteListDimsMapper() { + return whiteListDimsMapper; + } + + /** + * @param event Event subject to filtering + * @return true if and only if the event prefix key is in the {@code whiteListDimsMapper} + */ + private boolean isInWhiteList(ServiceMetricEvent event) { + return getPrefixKey(event.getMetric(), whiteListDimsMapper) != null; + } + + /** + * @param key the metric name to lookup + * @param whiteList + * @return null if the key does not match with any of the prefixes keys in @code metricsWhiteList, + * or the prefix in @code whiteListDimsMapper + */ + private String getPrefixKey(String key, SortedMap whiteList) { + String prefixKey = null; + if (whiteList.containsKey(key)) { + return key; + } + SortedMap headMap = whiteList.headMap(key); + if (!headMap.isEmpty() && key.startsWith(headMap.lastKey())) { + prefixKey = headMap.lastKey(); + } + return prefixKey; + } + + /** + * Returns a {@link java.util.List} of the white-listed dimension's values to send. + * The list is order is the same as the order of dimensions {@code whiteListDimsMapper} + * + * @param event the event for which will filter dimensions + * @return {@link java.util.List} of the filtered dimension values to send or null if the event is not in the white list + */ + private List getOrderedDimValues(ServiceMetricEvent event) { + String prefixKey = getPrefixKey(event.getMetric(), whiteListDimsMapper); + if (prefixKey == null) { + return null; + } + ImmutableList.Builder outputList = new ImmutableList.Builder(); + Set dimensions = whiteListDimsMapper.get(prefixKey); + if (dimensions == null) { + return Collections.emptyList(); + } + for (String dimKey : dimensions) { + String dimValue = (String) event.getUserDims().get(dimKey); + if (dimValue != null) { + outputList.add(DropwizardEmitter.sanitize(dimValue)); + } + } + return outputList.build(); + } + + /** + * @param serviceMetricEvent druid metric event to convert + * @return null if the event is not white listed, otherwise return {@link io.druid.emitter.dropwizard.DropwizardEvent} + *

+ * The metric name of the dropwizard event is: + * .[].[].. + *

+ * The order of the dimension is the order returned by {@code getOrderedDimValues()} + * Note that this path will be sanitized by replacing all the `.` or space by `_` {@link io.druid.emitter.dropwizard.DropwizardEmitter#sanitize(String)} + *

+ */ + + @Override + public DropwizardEvent druidEventToDropwizard(ServiceMetricEvent serviceMetricEvent) { + if (!this.isInWhiteList(serviceMetricEvent)) { + return null; + } + final ImmutableList.Builder metricPathBuilder = new ImmutableList.Builder<>(); + if (!Strings.isNullOrEmpty(this.getNamespacePrefix())) + metricPathBuilder.add(this.getNamespacePrefix()); + if (!this.isIgnoreServiceName()) { + metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getService())); + } + if (!this.isIgnoreHostname()) { + metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getHost())); + } + metricPathBuilder.addAll(this.getOrderedDimValues(serviceMetricEvent)); + metricPathBuilder.add(DropwizardEmitter.sanitize(serviceMetricEvent.getMetric())); + + final DropwizardEvent dropwizardEvent = new DropwizardEvent( + Joiner.on(".").join(metricPathBuilder.build()), + serviceMetricEvent.getValue().doubleValue() + ); + return dropwizardEvent; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof WhiteListBasedConverter)) { + return false; + } + + WhiteListBasedConverter that = (WhiteListBasedConverter) o; + + if (isIgnoreHostname() != that.isIgnoreHostname()) { + return false; + } + if (isIgnoreServiceName() != that.isIgnoreServiceName()) { + return false; + } + if (!getNamespacePrefix().equals(that.getNamespacePrefix())) { + return false; + } + return mapPath != null ? mapPath.equals(that.mapPath) : that.mapPath == null; + + } + + @Override + public int hashCode() { + int result = (isIgnoreHostname() ? 1 : 0); + result = 31 * result + (isIgnoreServiceName() ? 1 : 0); + result = 31 * result + (Strings.isNullOrEmpty(getNamespacePrefix())?0:getNamespacePrefix().hashCode()); + result = 31 * result + (mapPath != null ? mapPath.hashCode() : 0); + return result; + } + + private ImmutableSortedMap> readMap(final String mapPath) { + String fileContent; + String actualPath = mapPath; + try { + if (Strings.isNullOrEmpty(mapPath)) { + URL resource = this.getClass().getClassLoader().getResource("defaultWhiteListMap.json"); + actualPath = resource.getFile(); + LOGGER.info("using default whiteList map located at [%s]", actualPath); + fileContent = Resources.toString(resource, Charset.defaultCharset()); + } else { + fileContent = Files.asCharSource(new File(mapPath), Charset.forName("UTF-8")).read(); + } + return mapper.reader(new TypeReference>>() { + }).readValue(fileContent); + } catch (IOException e) { + throw new ISE(e, "Got an exception while parsing file [%s]", actualPath); + } + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java new file mode 100644 index 000000000000..6f12cb8b1898 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardConsoleReporter.java @@ -0,0 +1,120 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard.reporters; + +import com.codahale.metrics.ConsoleReporter; +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.druid.emitter.dropwizard.DropwizardReporter; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +@JsonTypeName("console") +public class DropwizardConsoleReporter implements DropwizardReporter { + private long emitIntervalInSecs; + private TimeUnit rates = TimeUnit.SECONDS; + private TimeUnit durations = TimeUnit.MILLISECONDS; + private ConsoleReporter consoleReporter; + + @JsonProperty + public long getEmitIntervalInSecs() { + return emitIntervalInSecs; + } + + @JsonProperty + public void setEmitIntervalInSecs(long emitIntervalInSecs) { + this.emitIntervalInSecs = emitIntervalInSecs; + } + + @JsonProperty + public TimeUnit getRates() { + return rates; + } + + @JsonProperty + public void setRates(String rates) { + this.rates = TimeUnit.valueOf(rates); + } + + @JsonProperty + public TimeUnit getDurations() { + return durations; + } + + @JsonProperty + public void setDurations(String durations) { + this.durations = TimeUnit.valueOf(durations); + } + + @Override + public void start(MetricRegistry metricRegistry) throws IOException { + consoleReporter = com.codahale.metrics.ConsoleReporter.forRegistry(metricRegistry). + convertDurationsTo(durations).convertRatesTo(rates).build(); + consoleReporter.start(emitIntervalInSecs, TimeUnit.SECONDS); + + } + + @Override + public void flush() throws IOException { + } + + @Override + public void close() throws IOException { + + } + + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DropwizardConsoleReporter that = (DropwizardConsoleReporter) o; + + if (emitIntervalInSecs != that.emitIntervalInSecs) { + return false; + } + if (consoleReporter != null ? !consoleReporter.equals(that.consoleReporter) : that.consoleReporter != null) { + return false; + } + if (durations != that.durations) { + return false; + } + if (rates != that.rates) { + return false; + } + + return true; + } + + @Override + public int hashCode() { + int result = (int) (emitIntervalInSecs ^ (emitIntervalInSecs >>> 32)); + result = 31 * result + (rates != null ? rates.hashCode() : 0); + result = 31 * result + (durations != null ? durations.hashCode() : 0); + result = 31 * result + (consoleReporter != null ? consoleReporter.hashCode() : 0); + return result; + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java new file mode 100644 index 000000000000..790cf77954a4 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/reporters/DropwizardJMXReporter.java @@ -0,0 +1,46 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.emitter.dropwizard.reporters; + +import com.codahale.metrics.JmxReporter; +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.annotation.JsonTypeName; +import io.druid.emitter.dropwizard.DropwizardReporter; + +import java.io.IOException; + +@JsonTypeName("jmx") +public class DropwizardJMXReporter implements DropwizardReporter { + @Override + public void start(MetricRegistry metricRegistry) throws IOException { + final JmxReporter reporter = JmxReporter.forRegistry(metricRegistry).build(); + reporter.start(); + + } + + @Override + public void flush() throws IOException { + + } + + @Override + public void close() throws IOException { + + } +} diff --git a/extensions-contrib/dropwizard-emitter/src/test/java/io/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java b/extensions-contrib/dropwizard-emitter/src/test/java/io/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java new file mode 100644 index 000000000000..b3cf659176af --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/test/java/io/druid/emitter/dropwizard/DropwizardEmitterConfigTest.java @@ -0,0 +1,63 @@ +package io.druid.emitter.dropwizard; + +import com.fasterxml.jackson.databind.InjectableValues; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.emitter.dropwizard.eventconverters.SendAllDropwizardEventConverter; +import io.druid.emitter.dropwizard.eventconverters.WhiteListBasedConverter; +import io.druid.emitter.dropwizard.reporters.DropwizardConsoleReporter; +import io.druid.jackson.DefaultObjectMapper; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.*; + +public class DropwizardEmitterConfigTest { + private ObjectMapper mapper = new DefaultObjectMapper(); + + @Before + public void setUp() + { + mapper.setInjectableValues(new InjectableValues.Std().addValue( + ObjectMapper.class, + new DefaultObjectMapper() + )); + } + + @Test + public void testSerDeserDropwizardEmitterConfig() throws IOException + { + DropwizardEmitterConfig dropwizardEmitterConfig = new DropwizardEmitterConfig(new DropwizardConsoleReporter(),new SendAllDropwizardEventConverter(null,false,false),new HistogramMetricManager(),null); + String dropwizardEmitterConfigString = mapper.writeValueAsString(dropwizardEmitterConfig); + DropwizardEmitterConfig dropwizardEmitterConfigExpected = mapper.reader(DropwizardEmitterConfig.class).readValue( + dropwizardEmitterConfigString + ); + Assert.assertEquals(dropwizardEmitterConfigExpected, dropwizardEmitterConfig); + } + + @Test + public void testSerDeserDruidToDropwizardEventConverter() throws IOException + { + SendAllDropwizardEventConverter sendAllDropwizardEventConverter = new SendAllDropwizardEventConverter("prefix", true, true); + String noopDropwizardEventConverterString = mapper.writeValueAsString(sendAllDropwizardEventConverter); + DruidToDropwizardEventConverter druidToDropwizardEventConverter = mapper.reader(DruidToDropwizardEventConverter.class) + .readValue(noopDropwizardEventConverterString); + Assert.assertEquals(druidToDropwizardEventConverter, sendAllDropwizardEventConverter); + + WhiteListBasedConverter whiteListBasedConverter = new WhiteListBasedConverter( + "prefix", + true, + true, + "", + new DefaultObjectMapper() + ); + String whiteListBasedConverterString = mapper.writeValueAsString(whiteListBasedConverter); + druidToDropwizardEventConverter = mapper.reader(DruidToDropwizardEventConverter.class) + .readValue(whiteListBasedConverterString); + Assert.assertEquals(druidToDropwizardEventConverter, whiteListBasedConverter); + } + + } + + From 4c42c93ae7d2a2f2540348684abf1393a01966fd Mon Sep 17 00:00:00 2001 From: "rohit.k" Date: Sun, 23 Apr 2017 20:24:40 +0530 Subject: [PATCH 2/2] making metric manager and alert emitters as optional --- .../extensions-contrib/dropwizard.md | 17 ++++++- .../dropwizard/DropwizardEmitterConfig.java | 3 +- .../io.druid.initialization.DruidModule | 1 + .../main/resources/defaultWhiteListMap.json | 48 +++++++++++++++++++ 4 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/io.druid.initialization.DruidModule create mode 100644 extensions-contrib/dropwizard-emitter/src/main/resources/defaultWhiteListMap.json diff --git a/docs/content/development/extensions-contrib/dropwizard.md b/docs/content/development/extensions-contrib/dropwizard.md index 18b2db037f9f..17f248239c47 100644 --- a/docs/content/development/extensions-contrib/dropwizard.md +++ b/docs/content/development/extensions-contrib/dropwizard.md @@ -8,11 +8,11 @@ To use this extension, make sure to [include](../../operations/including-extensi ## Introduction -The intent of this extension is to integrate [Dropwizard](http://metrics.dropwizard.io/3.1.0/getting-started/#); metrics library with druid so that dropwizard users can easily absorb druid into their monitoring ecosystem. +The intent of this extension is to integrate [Dropwizard](http://metrics.dropwizard.io/3.1.0/getting-started/#) metrics library with druid so that dropwizard users can easily absorb druid into their monitoring ecosystem. It accumulates druid metrics in a dropwizard histogram and emits them to various sinks via dropwizard supported reporters. A histogram measures the statistical distribution of values in a stream of data. In addition to minimum, maximum, mean, etc., it also measures median, 75th, 90th, 95th, 98th, 99th, and 99.9th percentiles. Currently dropwizard metrics can be emitted to these sinks: -Console,HTTP,JMX,Graphite,CSV,Slf4jLogger,Ganglia and various other community supported [sinks](http://metrics.dropwizard.io/3.1.0/manual/third-party/);. +Console,HTTP,JMX,Graphite,CSV,Slf4jLogger,Ganglia and various other community supported [sinks](http://metrics.dropwizard.io/3.1.0/manual/third-party/). ## Configuration @@ -85,3 +85,16 @@ druid.emitter.dropwizard.eventConverter={"type":"whiteList", "namespacePrefix": Metric manager defines the dropwizard accumulator that would be used to accumulate druid metric events. For eg : Gauge,Counter,Histogram,Meter etc. +### Dropwizard reporter + +```json + +druid.emitter.dropwizard.reporter={"type":"jmx"} + +``` + +```json + +druid.emitter.dropwizard.reporter={"type":"console","emitIntervalInSecs":30}"} + +``` \ No newline at end of file diff --git a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterConfig.java b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterConfig.java index 4fd745e1e8e7..5a642cc18689 100644 --- a/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterConfig.java +++ b/extensions-contrib/dropwizard-emitter/src/main/java/io/druid/emitter/dropwizard/DropwizardEmitterConfig.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Collections; import java.util.List; @@ -39,7 +40,7 @@ public DropwizardEmitterConfig(@JsonProperty("reporter") DropwizardReporter drop this.dropwizardReporter = dropwizardReporter; this.druidToDropwizardEventConverter = druidToDropwizardEventConverter; this.dropwizardMetricManager = dropwizardMetricManager==null? new HistogramMetricManager():dropwizardMetricManager; - this.alertEmitters = alertEmitters; + this.alertEmitters = alertEmitters == null ? Collections.emptyList() : alertEmitters; } diff --git a/extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/io.druid.initialization.DruidModule new file mode 100644 index 000000000000..8c325c3daa89 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/resources/META-INF/services/io.druid.initialization.DruidModule @@ -0,0 +1 @@ +io.druid.emitter.dropwizard.DropwizardEmitterModule diff --git a/extensions-contrib/dropwizard-emitter/src/main/resources/defaultWhiteListMap.json b/extensions-contrib/dropwizard-emitter/src/main/resources/defaultWhiteListMap.json new file mode 100644 index 000000000000..87cbd8951653 --- /dev/null +++ b/extensions-contrib/dropwizard-emitter/src/main/resources/defaultWhiteListMap.json @@ -0,0 +1,48 @@ +{ + "ingest/events": [], + "ingest/handoff/failed": [], + "ingest/persists": [], + "ingest/rows/output": [], + "jvm/gc": [], + "jvm/mem": [], + "query/cpu/time": [ + "dataSource", + "type" + ], + "query/node/time": [ + "dataSource", + "type" + ], + "query/node/ttfb": [ + "dataSource", + "type" + ], + "query/partial/time": [ + "dataSource", + "type" + ], + "query/segment/time": [ + "dataSource", + "type" + ], + "query/segmentAndCache/time": [ + "dataSource", + "type" + ], + "query/time": [ + "dataSource", + "type" + ], + "query/wait/time": [ + "dataSource", + "type" + ], + "segment/count": [], + "segment/dropQueue/count": [], + "segment/loadQueue/count": [], + "segment/loadQueue/failed": [], + "segment/loadQueue/size": [], + "segment/scan/pending": [], + "segment/size": [], + "segment/usedPercent": [] +}