diff --git a/core/src/main/java/org/apache/druid/java/util/metrics/JvmMonitor.java b/core/src/main/java/org/apache/druid/java/util/metrics/JvmMonitor.java index 77f4033a3da5..7c9f6be99dfb 100644 --- a/core/src/main/java/org/apache/druid/java/util/metrics/JvmMonitor.java +++ b/core/src/main/java/org/apache/druid/java/util/metrics/JvmMonitor.java @@ -22,19 +22,13 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; -import it.unimi.dsi.fastutil.ints.IntRBTreeSet; -import it.unimi.dsi.fastutil.ints.IntSet; -import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; -import org.gridkit.lab.jvm.perfdata.JStatData; -import org.gridkit.lab.jvm.perfdata.JStatData.LongCounter; -import org.gridkit.lab.jvm.perfdata.JStatData.StringCounter; -import org.gridkit.lab.jvm.perfdata.JStatData.TickCounter; import javax.annotation.Nullable; import java.lang.management.BufferPoolMXBean; +import java.lang.management.GarbageCollectorMXBean; import java.lang.management.ManagementFactory; import java.lang.management.MemoryPoolMXBean; import java.lang.management.MemoryType; @@ -42,22 +36,15 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; public class JvmMonitor extends FeedDefiningMonitor { - private static final Logger log = new Logger(JvmMonitor.class); - private static final Pattern PATTERN_GC_GENERATION = - Pattern.compile("^sun\\.gc\\.(?:generation|collector)\\.(\\d+)\\..*"); private final Map dimensions; - private final long pid; @VisibleForTesting @Nullable - final GcCounters gcCounters; + final GcCollectors gcCollectors; @Nullable private final AllocationMetricCollector collector; @@ -73,18 +60,12 @@ public JvmMonitor(Map dimensions) } public JvmMonitor(Map dimensions, String feed) - { - this(dimensions, feed, JvmPidDiscoverer.instance()); - } - - public JvmMonitor(Map dimensions, String feed, PidDiscoverer pidDiscoverer) { super(feed); Preconditions.checkNotNull(dimensions); this.dimensions = ImmutableMap.copyOf(dimensions); - this.pid = Preconditions.checkNotNull(pidDiscoverer).getPid(); this.collector = AllocationMetricCollectors.getAllocationMetricCollector(); - this.gcCounters = tryCreateGcCounters(); + this.gcCollectors = new GcCollectors(); } @Override @@ -162,199 +143,152 @@ private void emitDirectMemMetrics(ServiceEmitter emitter) private void emitGcMetrics(ServiceEmitter emitter) { - if (gcCounters != null) { - gcCounters.emit(emitter, dimensions); - } - } - - @Nullable - private GcCounters tryCreateGcCounters() - { - try { - return new GcCounters(); - } - catch (RuntimeException e) { - // in JDK11 jdk.internal.perf.Perf is not accessible, unless - // --add-exports java.base/jdk.internal.perf=ALL-UNNAMED is set - log.warn("Cannot initialize GC counters. If running JDK11 and above," - + " add --add-exports=java.base/jdk.internal.perf=ALL-UNNAMED" - + " to the JVM arguments to enable GC counters."); - } - return null; + gcCollectors.emit(emitter, dimensions); } - @VisibleForTesting - static IntSet getGcGenerations(final Set jStatCounterNames) + private class GcCollectors { - final IntSet retVal = new IntRBTreeSet(); + private final List generationCollectors = new ArrayList<>(); + private final List spaceCollectors = new ArrayList<>(); - for (final String counterName : jStatCounterNames) { - final Matcher m = PATTERN_GC_GENERATION.matcher(counterName); - if (m.matches()) { - retVal.add(Integer.parseInt(m.group(1))); + GcCollectors() + { + List collectorMxBeans = ManagementFactory.getGarbageCollectorMXBeans(); + for (GarbageCollectorMXBean collectorMxBean : collectorMxBeans) { + generationCollectors.add(new GcGenerationCollector(collectorMxBean)); } - } - return retVal; - } + List memoryPoolMxBeans = ManagementFactory.getMemoryPoolMXBeans(); + for (MemoryPoolMXBean memoryPoolMxBean : memoryPoolMxBeans) { + MemoryUsage collectionUsage = memoryPoolMxBean.getCollectionUsage(); + if (collectionUsage != null) { + spaceCollectors.add(new GcSpaceCollector(collectionUsage, memoryPoolMxBean.getName())); + } + } - @VisibleForTesting - static String getGcGenerationName(final int genIndex) - { - switch (genIndex) { - case 0: - return "young"; - case 1: - return "old"; - case 2: - // Removed in Java 8 but still actual for previous Java versions - return "perm"; - default: - return String.valueOf(genIndex); } - } - /** - * The following GC-related code is partially based on - * https://github.com/aragozin/jvm-tools/blob/e0e37692648951440aa1a4ea5046261cb360df70/ - * sjk-core/src/main/java/org/gridkit/jvmtool/PerfCounterGcCpuUsageMonitor.java - */ - private class GcCounters - { - private final List generations = new ArrayList<>(); - - GcCounters() + void emit(ServiceEmitter emitter, Map dimensions) { - // connect to itself - final JStatData jStatData = JStatData.connect(pid); - final Map> jStatCounters = jStatData.getAllCounters(); - - for (int genIndex : getGcGenerations(jStatCounters.keySet())) { - generations.add(new GcGeneration(jStatCounters, genIndex, getGcGenerationName(genIndex))); + for (GcGenerationCollector generationCollector : generationCollectors) { + generationCollector.emit(emitter, dimensions); } - } - void emit(ServiceEmitter emitter, Map dimensions) - { - for (GcGeneration generation : generations) { - generation.emit(emitter, dimensions); + for (GcSpaceCollector spaceCollector : spaceCollectors) { + spaceCollector.emit(emitter, dimensions); } } } - private class GcGeneration + private class GcGenerationCollector { - private final String name; - @Nullable - private final GcGenerationCollector collector; - private final List spaces = new ArrayList<>(); + private final String generation; + private final String collectorName; + private final GarbageCollectorMXBean gcBean; - GcGeneration(Map> jStatCounters, long genIndex, String name) - { - this.name = StringUtils.toLowerCase(name); + private long lastInvocations = 0; + private long lastCpuNanos = 0; - final String spacesCountKey = StringUtils.format("sun.gc.generation.%d.spaces", genIndex); + private static final String GC_YOUNG_GENERATION_NAME = "young"; + private static final String GC_OLD_GENERATION_NAME = "old"; + private static final String GC_ZGC_GENERATION_NAME = "zgc"; - if (jStatCounters.containsKey(spacesCountKey)) { - final long spacesCount = ((JStatData.LongCounter) jStatCounters.get(spacesCountKey)).getLong(); - for (long spaceIndex = 0; spaceIndex < spacesCount; spaceIndex++) { - spaces.add(new GcGenerationSpace(jStatCounters, genIndex, spaceIndex)); - } - } + private static final String CMS_COLLECTOR_NAME = "cms"; + private static final String G1_COLLECTOR_NAME = "g1"; + private static final String PARALLEL_COLLECTOR_NAME = "parallel"; + private static final String SERIAL_COLLECTOR_NAME = "serial"; + private static final String ZGC_COLLECTOR_NAME = "zgc"; + private static final String SHENANDOAN_COLLECTOR_NAME = "shenandoah"; - if (jStatCounters.containsKey(StringUtils.format("sun.gc.collector.%d.name", genIndex))) { - collector = new GcGenerationCollector(jStatCounters, genIndex); - } else { - collector = null; - } + GcGenerationCollector(GarbageCollectorMXBean gcBean) + { + Pair gcNamePair = getReadableName(gcBean.getName()); + this.generation = gcNamePair.lhs; + this.collectorName = gcNamePair.rhs; + this.gcBean = gcBean; } - void emit(ServiceEmitter emitter, Map dimensions) + private Pair getReadableName(String name) { - ImmutableMap.Builder dimensionsCopyBuilder = ImmutableMap - .builder() - .putAll(dimensions) - .put("gcGen", new String[]{name}); - - if (collector != null) { - dimensionsCopyBuilder.put("gcName", new String[]{collector.name}); - } + switch (name) { + //CMS + case "ParNew": + return new Pair<>(GC_YOUNG_GENERATION_NAME, CMS_COLLECTOR_NAME); + case "ConcurrentMarkSweep": + return new Pair<>(GC_OLD_GENERATION_NAME, CMS_COLLECTOR_NAME); + + // G1 + case "G1 Young Generation": + return new Pair<>(GC_YOUNG_GENERATION_NAME, G1_COLLECTOR_NAME); + case "G1 Old Generation": + return new Pair<>(GC_OLD_GENERATION_NAME, G1_COLLECTOR_NAME); + + // Parallel + case "PS Scavenge": + return new Pair<>(GC_YOUNG_GENERATION_NAME, PARALLEL_COLLECTOR_NAME); + case "PS MarkSweep": + return new Pair<>(GC_OLD_GENERATION_NAME, PARALLEL_COLLECTOR_NAME); + + // Serial + case "Copy": + return new Pair<>(GC_YOUNG_GENERATION_NAME, SERIAL_COLLECTOR_NAME); + case "MarkSweepCompact": + return new Pair<>(GC_OLD_GENERATION_NAME, SERIAL_COLLECTOR_NAME); - Map dimensionsCopy = dimensionsCopyBuilder.build(); + //zgc + case "ZGC": + return new Pair<>(GC_ZGC_GENERATION_NAME, ZGC_COLLECTOR_NAME); - if (collector != null) { - collector.emit(emitter, dimensionsCopy); - } + //Shenandoah + case "Shenandoah Cycles": + return new Pair<>(GC_YOUNG_GENERATION_NAME, SHENANDOAN_COLLECTOR_NAME); + case "Shenandoah Pauses": + return new Pair<>(GC_OLD_GENERATION_NAME, SHENANDOAN_COLLECTOR_NAME); - for (GcGenerationSpace space : spaces) { - space.emit(emitter, dimensionsCopy); + default: + return new Pair<>(name, name); } } - } - private class GcGenerationCollector - { - private final String name; - private final LongCounter invocationsCounter; - private final TickCounter cpuCounter; - private long lastInvocations = 0; - private long lastCpuNanos = 0; - - GcGenerationCollector(Map> jStatCounters, long genIndex) + void emit(ServiceEmitter emitter, Map dimensions) { - String collectorKeyPrefix = StringUtils.format("sun.gc.collector.%d", genIndex); + ImmutableMap.Builder dimensionsCopyBuilder = ImmutableMap + .builder() + .putAll(dimensions) + .put("gcGen", new String[]{generation}); - String nameKey = StringUtils.format("%s.name", collectorKeyPrefix); - StringCounter nameCounter = (StringCounter) jStatCounters.get(nameKey); - name = getReadableName(nameCounter.getString()); + dimensionsCopyBuilder.put("gcName", new String[]{collectorName}); - invocationsCounter = (LongCounter) jStatCounters.get(StringUtils.format("%s.invocations", collectorKeyPrefix)); - cpuCounter = (TickCounter) jStatCounters.get(StringUtils.format("%s.time", collectorKeyPrefix)); - } + Map dimensionsCopy = dimensionsCopyBuilder.build(); - void emit(ServiceEmitter emitter, Map dimensions) - { final ServiceMetricEvent.Builder builder = builder(); - MonitorUtils.addDimensionsToBuilder(builder, dimensions); + MonitorUtils.addDimensionsToBuilder(builder, dimensionsCopy); - long newInvocations = invocationsCounter.getLong(); + long newInvocations = gcBean.getCollectionCount(); emitter.emit(builder.build("jvm/gc/count", newInvocations - lastInvocations)); lastInvocations = newInvocations; - long newCpuNanos = cpuCounter.getNanos(); + long newCpuNanos = gcBean.getCollectionTime(); emitter.emit(builder.build("jvm/gc/cpu", newCpuNanos - lastCpuNanos)); lastCpuNanos = newCpuNanos; + } + } - private String getReadableName(String name) + private class GcSpaceCollector + { + + private final List spaces = new ArrayList<>(); + + public GcSpaceCollector(MemoryUsage collectionUsage, String name) { - switch (name) { - // Young gen - case "Copy": - return "serial"; - case "PSScavenge": - return "parallel"; - case "PCopy": - return "cms"; - case "G1 incremental collections": - return "g1"; - case "Shenandoah partial": - return "shenandoah"; - - // Old gen - case "MCS": - return "serial"; - case "PSParallelCompact": - return "parallel"; - case "CMS": - return "cms"; - case "G1 stop-the-world full collections": - return "g1"; - case "Shenandoah full": - return "shenandoah"; + spaces.add(new GcGenerationSpace(collectionUsage, name)); + } - default: - return name; + void emit(ServiceEmitter emitter, Map dimensions) + { + for (GcGenerationSpace space : spaces) { + space.emit(emitter, dimensions); } } } @@ -362,24 +296,12 @@ private String getReadableName(String name) private class GcGenerationSpace { private final String name; + private final MemoryUsage memoryUsage; - private final LongCounter maxCounter; - private final LongCounter capacityCounter; - private final LongCounter usedCounter; - private final LongCounter initCounter; - - GcGenerationSpace(Map> jStatCounters, long genIndex, long spaceIndex) + public GcGenerationSpace(MemoryUsage memoryUsage, String name) { - String spaceKeyPrefix = StringUtils.format("sun.gc.generation.%d.space.%d", genIndex, spaceIndex); - - String nameKey = StringUtils.format("%s.name", spaceKeyPrefix); - StringCounter nameCounter = (StringCounter) jStatCounters.get(nameKey); - name = StringUtils.toLowerCase(nameCounter.toString()); - - maxCounter = (LongCounter) jStatCounters.get(StringUtils.format("%s.maxCapacity", spaceKeyPrefix)); - capacityCounter = (LongCounter) jStatCounters.get(StringUtils.format("%s.capacity", spaceKeyPrefix)); - usedCounter = (LongCounter) jStatCounters.get(StringUtils.format("%s.used", spaceKeyPrefix)); - initCounter = (LongCounter) jStatCounters.get(StringUtils.format("%s.initCapacity", spaceKeyPrefix)); + this.memoryUsage = memoryUsage; + this.name = name; } void emit(ServiceEmitter emitter, Map dimensions) @@ -389,10 +311,10 @@ void emit(ServiceEmitter emitter, Map dimensions) builder.setDimension("gcGenSpaceName", name); - emitter.emit(builder.build("jvm/gc/mem/max", maxCounter.getLong())); - emitter.emit(builder.build("jvm/gc/mem/capacity", capacityCounter.getLong())); - emitter.emit(builder.build("jvm/gc/mem/used", usedCounter.getLong())); - emitter.emit(builder.build("jvm/gc/mem/init", initCounter.getLong())); + emitter.emit(builder.build("jvm/gc/mem/max", memoryUsage.getMax())); + emitter.emit(builder.build("jvm/gc/mem/capacity", memoryUsage.getCommitted())); + emitter.emit(builder.build("jvm/gc/mem/used", memoryUsage.getUsed())); + emitter.emit(builder.build("jvm/gc/mem/init", memoryUsage.getInit())); } } } diff --git a/core/src/test/java/org/apache/druid/java/util/metrics/JvmMonitorTest.java b/core/src/test/java/org/apache/druid/java/util/metrics/JvmMonitorTest.java index d5a2890e0894..24af5db4c56f 100644 --- a/core/src/test/java/org/apache/druid/java/util/metrics/JvmMonitorTest.java +++ b/core/src/test/java/org/apache/druid/java/util/metrics/JvmMonitorTest.java @@ -19,8 +19,6 @@ package org.apache.druid.java.util.metrics; -import com.google.common.collect.ImmutableSet; -import it.unimi.dsi.fastutil.ints.IntSet; import org.apache.druid.java.util.emitter.core.Emitter; import org.apache.druid.java.util.emitter.core.Event; import org.apache.druid.java.util.emitter.service.ServiceEmitter; @@ -43,7 +41,7 @@ public void testGcCounts() throws InterruptedException serviceEmitter.start(); final JvmMonitor jvmMonitor = new JvmMonitor(); // skip tests if gc counters fail to initialize with this JDK - Assume.assumeNotNull(jvmMonitor.gcCounters); + Assume.assumeNotNull(jvmMonitor.gcCollectors); while (true) { // generate some garbage to see gc counters incremented @@ -58,41 +56,6 @@ public void testGcCounts() throws InterruptedException } } - @Test - public void testGetGcGenerations() - { - Assert.assertEquals( - IntSet.of(0, 1), - JvmMonitor.getGcGenerations( - ImmutableSet.of( - "sun.gc.collector.0.name", - "sun.gc.collector.1.name", - "sun.gc.generation.1.spaces" - ) - ) - ); - - Assert.assertEquals( - IntSet.of(1, 2), - JvmMonitor.getGcGenerations( - ImmutableSet.of( - "sun.gc.generation.1.spaces", - "sun.gc.collector.2.name", - "sun.gc.somethingelse.3.name" - ) - ) - ); - } - - @Test - public void testGetGcGenerationName() - { - Assert.assertEquals("young", JvmMonitor.getGcGenerationName(0)); - Assert.assertEquals("old", JvmMonitor.getGcGenerationName(1)); - Assert.assertEquals("perm", JvmMonitor.getGcGenerationName(2)); - Assert.assertEquals("3", JvmMonitor.getGcGenerationName(3)); - } - private static class GcTrackingEmitter implements Emitter { private Number oldGcCount; diff --git a/pom.xml b/pom.xml index fce189172bba..5d29c54cb883 100644 --- a/pom.xml +++ b/pom.xml @@ -1757,7 +1757,6 @@ - ${jdk.surefire.argLine} -Xmx768m -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError diff --git a/processing/pom.xml b/processing/pom.xml index 80a8c8001681..da5e9a763803 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -300,7 +300,6 @@ maven-surefire-plugin - ${jdk.surefire.argLine} -server -Xms3G -Xmx3G -Djub.consumers=CONSOLE,H2 -Djub.db.file=benchmarks/benchmarks -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError