From fac9d3496c612c12f69d8be618492c582c81cc93 Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Wed, 3 Jan 2018 16:59:06 -0800 Subject: [PATCH 01/82] Fix lookup logging on node start (#5206) (#5207) * Add better logging messages in lookups startup on query nodes * Make sure list is mutable * Move list to be with other `final` variables --- .../query/lookup/LookupReferencesManager.java | 36 ++++++++----------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java b/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java index 6200961fad24..df2229f9229b 100644 --- a/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java +++ b/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java @@ -65,7 +65,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.LockSupport; import java.util.function.Function; -import java.util.stream.Collectors; /** * This class provide a basic {@link LookupExtractorFactory} references manager. @@ -472,37 +471,30 @@ private List getLookupBeanList(Map lookupBeanList) { - ImmutableMap.Builder builder = ImmutableMap.builder(); - ExecutorService executorService = Execs.multiThreaded( + final ImmutableMap.Builder builder = ImmutableMap.builder(); + final ExecutorService executorService = Execs.multiThreaded( lookupConfig.getNumLookupLoadingThreads(), "LookupReferencesManager-Startup-%s" ); - CompletionService> completionService = + final CompletionService> completionService = new ExecutorCompletionService<>(executorService); + final List remainingLookups = new ArrayList<>(lookupBeanList); try { LOG.info("Starting lookup loading process"); - List remainingLookups = lookupBeanList; - for (int i = 0; i < lookupConfig.getLookupStartRetries(); i++) { + for (int i = 0; i < lookupConfig.getLookupStartRetries() && !remainingLookups.isEmpty(); i++) { LOG.info("Round of attempts #%d, [%d] lookups", i + 1, remainingLookups.size()); - Map successfulLookups = + final Map successfulLookups = startLookups(remainingLookups, completionService); builder.putAll(successfulLookups); - List failedLookups = remainingLookups - .stream() - .filter(l -> !successfulLookups.containsKey(l.getName())) - .collect(Collectors.toList()); - if (failedLookups.isEmpty()) { - break; - } else { - // next round - remainingLookups = failedLookups; - } + remainingLookups.removeIf(l -> successfulLookups.containsKey(l.getName())); + } + if (!remainingLookups.isEmpty()) { + LOG.warn( + "Failed to start the following lookups after [%d] attempts: [%s]", + lookupConfig.getLookupStartRetries(), + remainingLookups + ); } - LOG.info( - "Failed to start the following lookups after [%d] attempts: [%s]", - lookupConfig.getLookupStartRetries(), - remainingLookups - ); stateRef.set(new LookupUpdateState(builder.build(), ImmutableList.of(), ImmutableList.of())); } catch (InterruptedException | RuntimeException e) { From e64092aa31afcf5e0942d7867c7a5c3918520054 Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Wed, 3 Jan 2018 17:53:31 -0800 Subject: [PATCH 02/82] Fix broken KafkaEmitterConfig parsing (#5201) (#5208) * Fix broken KafkaEmitterConfig parsing This was a regression introduced in https://github.com/druid-io/druid/pull/4722 KafkaEmitterConfig property names have dot(.) in the name of properties and JsonConfigurator behavior was changed to not support that. Added a test and fixed parsing of properties that have dot(.) in property names * Fix test failure --- .../java/io/druid/guice/JsonConfigurator.java | 6 ++-- .../io/druid/guice/JsonConfiguratorTest.java | 34 ++++++++++++++++--- .../server/emitter/EmitterModuleTest.java | 2 ++ 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/api/src/main/java/io/druid/guice/JsonConfigurator.java b/api/src/main/java/io/druid/guice/JsonConfigurator.java index f6d766dd9551..09721a932d59 100644 --- a/api/src/main/java/io/druid/guice/JsonConfigurator.java +++ b/api/src/main/java/io/druid/guice/JsonConfigurator.java @@ -93,7 +93,6 @@ public T configurate(Properties props, String propertyPrefix, Class clazz log.info(e, "Unable to parse [%s]=[%s] as a json object, using as is.", prop, propValue); value = propValue; } - hieraricalPutValue(propertyPrefix, prop, prop.substring(propertyBase.length()), value, jsonMap); } } @@ -175,8 +174,11 @@ private static void hieraricalPutValue( ) { int dotIndex = property.indexOf('.'); + // Always put property with name even if it is of form a.b. This will make sure the property is available for classes + // where JsonProperty names are of the form a.b + // Note:- this will cause more than required properties to be present in the jsonMap. + targetMap.put(property, value); if (dotIndex < 0) { - targetMap.put(property, value); return; } if (dotIndex == 0) { diff --git a/api/src/test/java/io/druid/guice/JsonConfiguratorTest.java b/api/src/test/java/io/druid/guice/JsonConfiguratorTest.java index 0ce4f77a79aa..acfadf57131f 100644 --- a/api/src/test/java/io/druid/guice/JsonConfiguratorTest.java +++ b/api/src/test/java/io/druid/guice/JsonConfiguratorTest.java @@ -94,10 +94,13 @@ public ExecutableValidator forExecutables() public void testTest() { Assert.assertEquals( - new MappableObject("p1", ImmutableList.of("p2")), - new MappableObject("p1", ImmutableList.of("p2")) + new MappableObject("p1", ImmutableList.of("p2"), "p2"), + new MappableObject("p1", ImmutableList.of("p2"), "p2") + ); + Assert.assertEquals( + new MappableObject("p1", null, null), + new MappableObject("p1", ImmutableList.of(), null) ); - Assert.assertEquals(new MappableObject("p1", null), new MappableObject("p1", ImmutableList.of())); } @Test @@ -140,6 +143,19 @@ public void testQuotedConfig() Assert.assertEquals("testing \"prop1\"", obj.prop1); Assert.assertEquals(ImmutableList.of(), obj.prop1List); } + + @Test + public void testPropertyWithDot() + { + final JsonConfigurator configurator = new JsonConfigurator(mapper, validator); + properties.setProperty(PROP_PREFIX + "prop2.prop.2", "testing"); + properties.setProperty(PROP_PREFIX + "prop1", "prop1"); + final MappableObject obj = configurator.configurate(properties, PROP_PREFIX, MappableObject.class); + Assert.assertEquals("testing", obj.prop2); + Assert.assertEquals(ImmutableList.of(), obj.prop1List); + Assert.assertEquals("prop1", obj.prop1); + + } } class MappableObject @@ -148,15 +164,19 @@ class MappableObject final String prop1; @JsonProperty("prop1List") final List prop1List; + @JsonProperty("prop2.prop.2") + final String prop2; @JsonCreator protected MappableObject( @JsonProperty("prop1") final String prop1, - @JsonProperty("prop1List") final List prop1List + @JsonProperty("prop1List") final List prop1List, + @JsonProperty("prop2.prop.2") final String prop2 ) { this.prop1 = prop1; this.prop1List = prop1List == null ? ImmutableList.of() : prop1List; + this.prop2 = prop2; } @@ -172,6 +192,12 @@ public String getProp1() return prop1; } + @JsonProperty + public String getProp2() + { + return prop2; + } + @Override public boolean equals(Object o) { diff --git a/server/src/test/java/io/druid/server/emitter/EmitterModuleTest.java b/server/src/test/java/io/druid/server/emitter/EmitterModuleTest.java index 52a6876a5b62..0cb94f6d447f 100644 --- a/server/src/test/java/io/druid/server/emitter/EmitterModuleTest.java +++ b/server/src/test/java/io/druid/server/emitter/EmitterModuleTest.java @@ -31,6 +31,7 @@ import io.druid.guice.LazySingleton; import io.druid.guice.LifecycleModule; import io.druid.guice.ServerModule; +import io.druid.jackson.JacksonModule; import org.junit.Assert; import org.junit.Test; @@ -67,6 +68,7 @@ private Injector makeInjectorWithProperties(final Properties props) new DruidGuiceExtensions(), new LifecycleModule(), new ServerModule(), + new JacksonModule(), new Module() { @Override From 14da754594d99724c03303b8789e4683006110aa Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 3 Jan 2018 20:29:32 -0800 Subject: [PATCH 03/82] Support replaceExisting parameter for segments pushers (#5187) (#5209) * support replaceExisting parameter for segments pushers * code review changes * code review changes --- .../segment/loading/DataSegmentPusher.java | 25 +++++- .../storage/azure/AzureDataSegmentPusher.java | 13 +-- .../io/druid/storage/azure/AzureStorage.java | 15 +++- .../io/druid/storage/azure/AzureTaskLogs.java | 2 +- .../azure/AzureDataSegmentPusherTest.java | 9 +- .../storage/azure/AzureTaskLogsTest.java | 2 +- .../cassandra/CassandraDataSegmentPusher.java | 51 +++++++---- .../CloudFilesDataSegmentPusher.java | 32 ++++--- .../cloudfiles/CloudFilesObjectApiProxy.java | 5 ++ .../CloudFilesDataSegmentPusherTest.java | 2 +- .../google/GoogleDataSegmentPusher.java | 16 ++-- .../google/GoogleDataSegmentPusherTest.java | 18 +++- .../storage/hdfs/HdfsDataSegmentPusher.java | 11 +-- .../org/apache/hadoop/fs/HadoopFsWrapper.java | 20 +++-- .../hdfs/HdfsDataSegmentPusherTest.java | 8 +- .../druid/storage/s3/S3DataSegmentPusher.java | 50 ++++++----- .../storage/s3/S3DataSegmentPusherTest.java | 2 +- .../common/index/YeOldePlumberSchool.java | 5 +- .../common/task/ConvertSegmentTask.java | 6 +- .../indexing/common/task/MergeTaskBase.java | 6 +- .../indexing/common/task/IndexTaskTest.java | 2 +- .../task/SameIntervalMergeTaskTest.java | 2 +- .../IngestSegmentFirehoseFactoryTest.java | 2 +- .../indexing/overlord/TaskLifecycleTest.java | 4 +- .../indexing/test/TestDataSegmentPusher.java | 2 +- .../java/util/common/CompressionUtils.java | 61 +++++++++---- .../loading/LocalDataSegmentPusher.java | 90 ++++++++++++------- .../appenderator/AppenderatorImpl.java | 15 +++- .../realtime/plumber/RealtimePlumber.java | 10 ++- .../loading/LocalDataSegmentPusherTest.java | 58 ++++++++++-- .../appenderator/AppenderatorTester.java | 2 +- .../java/io/druid/cli/CliRealtimeExample.java | 2 +- 32 files changed, 379 insertions(+), 169 deletions(-) diff --git a/api/src/main/java/io/druid/segment/loading/DataSegmentPusher.java b/api/src/main/java/io/druid/segment/loading/DataSegmentPusher.java index b9bf810f72ce..2ce046665977 100644 --- a/api/src/main/java/io/druid/segment/loading/DataSegmentPusher.java +++ b/api/src/main/java/io/druid/segment/loading/DataSegmentPusher.java @@ -39,7 +39,30 @@ public interface DataSegmentPusher @Deprecated String getPathForHadoop(String dataSource); String getPathForHadoop(); - DataSegment push(File file, DataSegment segment) throws IOException; + + /** + * Pushes index files and segment descriptor to deep storage. + * @param file directory containing index files + * @param segment segment descriptor + * @param replaceExisting overwrites existing objects if true, else leaves existing objects unchanged on conflict. + * The behavior of the indexer determines whether this should be true or false. For example, + * since Tranquility does not guarantee that replica tasks will generate indexes with the same + * data, the first segment pushed should be favored since otherwise multiple historicals may + * load segments with the same identifier but different contents which is a bad situation. On + * the other hand, indexers that maintain exactly-once semantics by storing checkpoint data can + * lose or repeat data if it fails to write a segment because it already exists and overwriting + * is not permitted. This situation can occur if a task fails after pushing to deep storage but + * before writing to the metadata storage, see: https://github.com/druid-io/druid/issues/5161. + * + * If replaceExisting is true, existing objects MUST be overwritten, since failure to do so + * will break exactly-once semantics. If replaceExisting is false, existing objects SHOULD be + * prioritized but it is acceptable if they are overwritten (deep storages may be eventually + * consistent or otherwise unable to support transactional writes). + * @return segment descriptor + * @throws IOException + */ + DataSegment push(File file, DataSegment segment, boolean replaceExisting) throws IOException; + //use map instead of LoadSpec class to avoid dependency pollution. Map makeLoadSpec(URI finalIndexZipFilePath); diff --git a/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPusher.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPusher.java index 97bc1a0167a7..39ab34258e0d 100644 --- a/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPusher.java +++ b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPusher.java @@ -109,12 +109,13 @@ public DataSegment uploadDataSegment( final long size, final File compressedSegmentData, final File descriptorFile, - final Map azurePaths + final Map azurePaths, + final boolean replaceExisting ) throws StorageException, IOException, URISyntaxException { - azureStorage.uploadBlob(compressedSegmentData, config.getContainer(), azurePaths.get("index")); - azureStorage.uploadBlob(descriptorFile, config.getContainer(), azurePaths.get("descriptor")); + azureStorage.uploadBlob(compressedSegmentData, config.getContainer(), azurePaths.get("index"), replaceExisting); + azureStorage.uploadBlob(descriptorFile, config.getContainer(), azurePaths.get("descriptor"), replaceExisting); final DataSegment outSegment = segment .withSize(size) @@ -131,9 +132,9 @@ public DataSegment uploadDataSegment( } @Override - public DataSegment push(final File indexFilesDir, final DataSegment segment) throws IOException + public DataSegment push(final File indexFilesDir, final DataSegment segment, final boolean replaceExisting) + throws IOException { - log.info("Uploading [%s] to Azure.", indexFilesDir); final int version = SegmentUtils.getVersionFromDir(indexFilesDir); @@ -153,7 +154,7 @@ public DataSegment push(final File indexFilesDir, final DataSegment segment) thr @Override public DataSegment call() throws Exception { - return uploadDataSegment(segment, version, size, outFile, descFile, azurePaths); + return uploadDataSegment(segment, version, size, outFile, descFile, azurePaths, replaceExisting); } }, config.getMaxTries() diff --git a/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureStorage.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureStorage.java index 8585bf043797..28fa9dad56c6 100644 --- a/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureStorage.java +++ b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureStorage.java @@ -25,7 +25,6 @@ import com.microsoft.azure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlockBlob; import com.microsoft.azure.storage.blob.ListBlobItem; - import io.druid.java.util.common.logger.Logger; import java.io.File; @@ -81,14 +80,24 @@ public List emptyCloudBlobDirectory(final String containerName, final St } - public void uploadBlob(final File file, final String containerName, final String blobPath) + public void uploadBlob( + final File file, + final String containerName, + final String blobPath, + final boolean replaceExisting + ) throws IOException, StorageException, URISyntaxException { CloudBlobContainer container = getCloudBlobContainer(containerName); try (FileInputStream stream = new FileInputStream(file)) { CloudBlockBlob blob = container.getBlockBlobReference(blobPath); - blob.upload(stream, file.length()); + + if (!replaceExisting && blob.exists()) { + log.info("Skipping push because blob [%s] exists && replaceExisting == false", blobPath); + } else { + blob.upload(stream, file.length()); + } } } diff --git a/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureTaskLogs.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureTaskLogs.java index f4b6093ed760..50092ceff62c 100644 --- a/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureTaskLogs.java +++ b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureTaskLogs.java @@ -59,7 +59,7 @@ public void pushTaskLog(final String taskid, final File logFile) throws IOExcept try { AzureUtils.retryAzureOperation( (Callable) () -> { - azureStorage.uploadBlob(logFile, config.getContainer(), taskKey); + azureStorage.uploadBlob(logFile, config.getContainer(), taskKey, true); return null; }, config.getMaxTries() diff --git a/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentPusherTest.java b/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentPusherTest.java index af76f3571424..30c571a4f163 100644 --- a/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentPusherTest.java +++ b/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentPusherTest.java @@ -104,7 +104,7 @@ public void testPush() throws Exception size ); - DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush); + DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush, true); Assert.assertEquals(segmentToPush.getSize(), segment.getSize()); } @@ -133,9 +133,9 @@ public void uploadDataSegmentTest() throws StorageException, IOException, URISyn final File descriptorFile = new File("descriptor.json"); final Map azurePaths = pusher.getAzurePaths(dataSegment); - azureStorage.uploadBlob(compressedSegmentData, containerName, azurePaths.get("index")); + azureStorage.uploadBlob(compressedSegmentData, containerName, azurePaths.get("index"), true); expectLastCall(); - azureStorage.uploadBlob(descriptorFile, containerName, azurePaths.get("descriptor")); + azureStorage.uploadBlob(descriptorFile, containerName, azurePaths.get("descriptor"), true); expectLastCall(); replayAll(); @@ -146,7 +146,8 @@ public void uploadDataSegmentTest() throws StorageException, IOException, URISyn 0, // empty file compressedSegmentData, descriptorFile, - azurePaths + azurePaths, + true ); assertEquals(compressedSegmentData.length(), pushedDataSegment.getSize()); diff --git a/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureTaskLogsTest.java b/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureTaskLogsTest.java index 26ff677af177..5f8394a735a4 100644 --- a/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureTaskLogsTest.java +++ b/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureTaskLogsTest.java @@ -65,7 +65,7 @@ public void testPushTaskLog() throws Exception try { final File logFile = new File(tmpDir, "log"); - azureStorage.uploadBlob(logFile, container, prefix + "/" + taskid + "/log"); + azureStorage.uploadBlob(logFile, container, prefix + "/" + taskid + "/log", true); expectLastCall(); replayAll(); diff --git a/extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentPusher.java b/extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentPusher.java index 3595147ace85..3b3ad8e5539b 100644 --- a/extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentPusher.java +++ b/extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentPusher.java @@ -24,8 +24,9 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.netflix.astyanax.MutationBatch; +import com.netflix.astyanax.connectionpool.exceptions.NotFoundException; import com.netflix.astyanax.recipes.storage.ChunkedStorage; - +import com.netflix.astyanax.recipes.storage.ChunkedStorageProvider; import io.druid.java.util.common.CompressionUtils; import io.druid.java.util.common.logger.Logger; import io.druid.segment.SegmentUtils; @@ -53,7 +54,8 @@ public class CassandraDataSegmentPusher extends CassandraStorage implements Data @Inject public CassandraDataSegmentPusher( CassandraDataSegmentConfig config, - ObjectMapper jsonMapper) + ObjectMapper jsonMapper + ) { super(config); this.jsonMapper = jsonMapper; @@ -73,13 +75,14 @@ public String getPathForHadoop(String dataSource) } @Override - public DataSegment push(final File indexFilesDir, DataSegment segment) throws IOException + public DataSegment push(final File indexFilesDir, DataSegment segment, final boolean replaceExisting) + throws IOException { log.info("Writing [%s] to C*", indexFilesDir); String key = JOINER.join( config.getKeyspace().isEmpty() ? null : config.getKeyspace(), this.getStorageDir(segment) - ); + ); // Create index final File compressedIndexFile = File.createTempFile("druid", "index.zip"); @@ -89,26 +92,28 @@ public DataSegment push(final File indexFilesDir, DataSegment segment) throws IO int version = SegmentUtils.getVersionFromDir(indexFilesDir); try { - long start = System.currentTimeMillis(); - ChunkedStorage.newWriter(indexStorage, key, new FileInputStream(compressedIndexFile)) - .withConcurrencyLevel(CONCURRENCY).call(); - byte[] json = jsonMapper.writeValueAsBytes(segment); - MutationBatch mutation = this.keyspace.prepareMutationBatch(); - mutation.withRow(descriptorStorage, key) - .putColumn("lastmodified", System.currentTimeMillis(), null) - .putColumn("descriptor", json, null); - mutation.execute(); - log.info("Wrote index to C* in [%s] ms", System.currentTimeMillis() - start); + if (!replaceExisting && doesObjectExist(indexStorage, key)) { + log.info("Skipping push because key [%s] exists && replaceExisting == false", key); + } else { + long start = System.currentTimeMillis(); + ChunkedStorage.newWriter(indexStorage, key, new FileInputStream(compressedIndexFile)) + .withConcurrencyLevel(CONCURRENCY).call(); + byte[] json = jsonMapper.writeValueAsBytes(segment); + MutationBatch mutation = this.keyspace.prepareMutationBatch(); + mutation.withRow(descriptorStorage, key) + .putColumn("lastmodified", System.currentTimeMillis(), null) + .putColumn("descriptor", json, null); + mutation.execute(); + log.info("Wrote index to C* in [%s] ms", System.currentTimeMillis() - start); + } } catch (Exception e) { throw new IOException(e); } segment = segment.withSize(indexSize) - .withLoadSpec( - ImmutableMap. of("type", "c*", "key", key) - ) - .withBinaryVersion(version); + .withLoadSpec(ImmutableMap.of("type", "c*", "key", key)) + .withBinaryVersion(version); log.info("Deleting zipped index File[%s]", compressedIndexFile); compressedIndexFile.delete(); @@ -120,4 +125,14 @@ public Map makeLoadSpec(URI uri) { throw new UnsupportedOperationException("not supported"); } + + private boolean doesObjectExist(ChunkedStorageProvider provider, String objectName) throws Exception + { + try { + return ChunkedStorage.newInfoReader(provider, objectName).call().isValidForRead(); + } + catch (NotFoundException e) { + return false; + } + } } diff --git a/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusher.java b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusher.java index a08105b0575e..e224a5b83b78 100644 --- a/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusher.java +++ b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusher.java @@ -74,7 +74,8 @@ public String getPathForHadoop(final String dataSource) } @Override - public DataSegment push(final File indexFilesDir, final DataSegment inSegment) throws IOException + public DataSegment push(final File indexFilesDir, final DataSegment inSegment, final boolean replaceExisting) + throws IOException { final String segmentPath = CloudFilesUtils.buildCloudFilesPath(this.config.getBasePath(), getStorageDir(inSegment)); @@ -98,18 +99,23 @@ public DataSegment call() throws Exception segmentPath, outFile, objectApi.getRegion(), objectApi.getContainer() ); - log.info("Pushing %s.", segmentData.getPath()); - objectApi.put(segmentData); - - // Avoid using Guava in DataSegmentPushers because they might be used with very diverse Guava versions in - // runtime, and because Guava deletes methods over time, that causes incompatibilities. - Files.write(descFile.toPath(), jsonMapper.writeValueAsBytes(inSegment)); - CloudFilesObject descriptorData = new CloudFilesObject( - segmentPath, descFile, - objectApi.getRegion(), objectApi.getContainer() - ); - log.info("Pushing %s.", descriptorData.getPath()); - objectApi.put(descriptorData); + + if (!replaceExisting && objectApi.exists(segmentData.getPath())) { + log.info("Skipping push because object [%s] exists && replaceExisting == false", segmentData.getPath()); + } else { + log.info("Pushing %s.", segmentData.getPath()); + objectApi.put(segmentData); + + // Avoid using Guava in DataSegmentPushers because they might be used with very diverse Guava versions in + // runtime, and because Guava deletes methods over time, that causes incompatibilities. + Files.write(descFile.toPath(), jsonMapper.writeValueAsBytes(inSegment)); + CloudFilesObject descriptorData = new CloudFilesObject( + segmentPath, descFile, + objectApi.getRegion(), objectApi.getContainer() + ); + log.info("Pushing %s.", descriptorData.getPath()); + objectApi.put(descriptorData); + } final DataSegment outSegment = inSegment .withSize(indexSize) diff --git a/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesObjectApiProxy.java b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesObjectApiProxy.java index 7f41497b37c9..d495fcc1c77f 100644 --- a/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesObjectApiProxy.java +++ b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesObjectApiProxy.java @@ -58,4 +58,9 @@ public CloudFilesObject get(String path) Payload payload = swiftObject.getPayload(); return new CloudFilesObject(payload, this.region, this.container, path); } + + public boolean exists(String path) + { + return objectApi.getWithoutBody(path) != null; + } } diff --git a/extensions-contrib/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusherTest.java b/extensions-contrib/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusherTest.java index b257efb1282a..73be6ce0e1c1 100644 --- a/extensions-contrib/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusherTest.java +++ b/extensions-contrib/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusherTest.java @@ -84,7 +84,7 @@ public void testPush() throws Exception size ); - DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush); + DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush, true); Assert.assertEquals(segmentToPush.getSize(), segment.getSize()); diff --git a/extensions-contrib/google-extensions/src/main/java/io/druid/storage/google/GoogleDataSegmentPusher.java b/extensions-contrib/google-extensions/src/main/java/io/druid/storage/google/GoogleDataSegmentPusher.java index 527e7e14025e..e8489ea8bd86 100644 --- a/extensions-contrib/google-extensions/src/main/java/io/druid/storage/google/GoogleDataSegmentPusher.java +++ b/extensions-contrib/google-extensions/src/main/java/io/druid/storage/google/GoogleDataSegmentPusher.java @@ -93,7 +93,8 @@ public File createDescriptorFile(final ObjectMapper jsonMapper, final DataSegmen return descriptorFile; } - public void insert(final File file, final String contentType, final String path) throws IOException + public void insert(final File file, final String contentType, final String path, final boolean replaceExisting) + throws IOException { LOG.info("Inserting [%s] to [%s]", file, path); @@ -102,11 +103,16 @@ public void insert(final File file, final String contentType, final String path) InputStreamContent mediaContent = new InputStreamContent(contentType, fileSteam); mediaContent.setLength(file.length()); - storage.insert(config.getBucket(), path, mediaContent); + if (!replaceExisting && storage.exists(config.getBucket(), path)) { + LOG.info("Skipping push because path [%s] exists && replaceExisting == false", path); + } else { + storage.insert(config.getBucket(), path, mediaContent); + } } @Override - public DataSegment push(final File indexFilesDir, final DataSegment segment) throws IOException + public DataSegment push(final File indexFilesDir, final DataSegment segment, final boolean replaceExisting) + throws IOException { LOG.info("Uploading [%s] to Google.", indexFilesDir); @@ -128,8 +134,8 @@ public DataSegment push(final File indexFilesDir, final DataSegment segment) thr descriptorFile = createDescriptorFile(jsonMapper, outSegment); - insert(indexFile, "application/zip", indexPath); - insert(descriptorFile, "application/json", descriptorPath); + insert(indexFile, "application/zip", indexPath, replaceExisting); + insert(descriptorFile, "application/json", descriptorPath, replaceExisting); return outSegment; } diff --git a/extensions-contrib/google-extensions/src/test/java/io/druid/storage/google/GoogleDataSegmentPusherTest.java b/extensions-contrib/google-extensions/src/test/java/io/druid/storage/google/GoogleDataSegmentPusherTest.java index 6c845d433c77..11372c243aa0 100644 --- a/extensions-contrib/google-extensions/src/test/java/io/druid/storage/google/GoogleDataSegmentPusherTest.java +++ b/extensions-contrib/google-extensions/src/test/java/io/druid/storage/google/GoogleDataSegmentPusherTest.java @@ -103,20 +103,30 @@ public void testPush() throws Exception storage, googleAccountConfig, jsonMapper - ).addMockedMethod("insert", File.class, String.class, String.class).createMock(); + ).addMockedMethod("insert", File.class, String.class, String.class, boolean.class).createMock(); final String storageDir = pusher.getStorageDir(segmentToPush); final String indexPath = prefix + "/" + storageDir + "/" + "index.zip"; final String descriptorPath = prefix + "/" + storageDir + "/" + "descriptor.json"; - pusher.insert(EasyMock.anyObject(File.class), EasyMock.eq("application/zip"), EasyMock.eq(indexPath)); + pusher.insert( + EasyMock.anyObject(File.class), + EasyMock.eq("application/zip"), + EasyMock.eq(indexPath), + EasyMock.eq(true) + ); expectLastCall(); - pusher.insert(EasyMock.anyObject(File.class), EasyMock.eq("application/json"), EasyMock.eq(descriptorPath)); + pusher.insert( + EasyMock.anyObject(File.class), + EasyMock.eq("application/json"), + EasyMock.eq(descriptorPath), + EasyMock.eq(true) + ); expectLastCall(); replayAll(); - DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush); + DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush, true); Assert.assertEquals(segmentToPush.getSize(), segment.getSize()); Assert.assertEquals(segmentToPush, segment); diff --git a/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPusher.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPusher.java index 33a85ac2147a..3374d4333329 100644 --- a/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPusher.java +++ b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPusher.java @@ -89,7 +89,7 @@ public String getPathForHadoop() } @Override - public DataSegment push(File inDir, DataSegment segment) throws IOException + public DataSegment push(File inDir, DataSegment segment, boolean replaceExisting) throws IOException { final String storageDir = this.getStorageDir(segment); @@ -145,8 +145,8 @@ public DataSegment push(File inDir, DataSegment segment) throws IOException // Create parent if it does not exist, recreation is not an error fs.mkdirs(outIndexFile.getParent()); - copyFilesWithChecks(fs, tmpDescriptorFile, outDescriptorFile); - copyFilesWithChecks(fs, tmpIndexFile, outIndexFile); + copyFilesWithChecks(fs, tmpDescriptorFile, outDescriptorFile, replaceExisting); + copyFilesWithChecks(fs, tmpIndexFile, outIndexFile, replaceExisting); } finally { try { @@ -162,9 +162,10 @@ public DataSegment push(File inDir, DataSegment segment) throws IOException return dataSegment; } - private void copyFilesWithChecks(final FileSystem fs, final Path from, final Path to) throws IOException + private void copyFilesWithChecks(final FileSystem fs, final Path from, final Path to, final boolean replaceExisting) + throws IOException { - if (!HadoopFsWrapper.rename(fs, from, to)) { + if (!HadoopFsWrapper.rename(fs, from, to, replaceExisting)) { if (fs.exists(to)) { log.info( "Unable to rename temp Index file[%s] to final segment path [%s]. " diff --git a/extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java b/extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java index df00fdb7a0fa..4604241dd76a 100644 --- a/extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java +++ b/extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java @@ -36,23 +36,25 @@ public class HadoopFsWrapper private HadoopFsWrapper() {} /** - * Same as FileSystem.rename(from, to, Options.Rename.NONE) . That is, - * it returns "false" when "to" directory already exists. It is different from FileSystem.rename(from, to) - * which moves "from" directory inside "to" directory if it already exists. + * Same as FileSystem.rename(from, to, Options.Rename). It is different from FileSystem.rename(from, to) which moves + * "from" directory inside "to" directory if it already exists. * * @param from * @param to - * @return - * @throws IOException + * @param replaceExisting if existing files should be overwritten + * + * @return true if operation succeeded, false if replaceExisting == false and destination already exists + * + * @throws IOException if trying to overwrite a non-empty directory */ - public static boolean rename(FileSystem fs, Path from, Path to) throws IOException + public static boolean rename(FileSystem fs, Path from, Path to, boolean replaceExisting) throws IOException { try { - fs.rename(from, to, Options.Rename.NONE); + fs.rename(from, to, replaceExisting ? Options.Rename.OVERWRITE : Options.Rename.NONE); return true; } - catch (IOException ex) { - log.warn(ex, "Failed to rename [%s] to [%s].", from, to); + catch (FileAlreadyExistsException ex) { + log.info(ex, "Destination exists while renaming [%s] to [%s]", from, to); return false; } } diff --git a/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java b/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java index 95dd85bf2b9e..bfc119f6498a 100644 --- a/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java +++ b/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java @@ -161,7 +161,7 @@ private void testUsingScheme(final String scheme) throws Exception size ); - DataSegment segment = pusher.push(segmentDir, segmentToPush); + DataSegment segment = pusher.push(segmentDir, segmentToPush, true); String indexUri = StringUtils.format( @@ -201,7 +201,7 @@ private void testUsingScheme(final String scheme) throws Exception File outDir = new File(StringUtils.format("%s/%s", config.getStorageDirectory(), segmentPath)); outDir.setReadOnly(); try { - pusher.push(segmentDir, segmentToPush); + pusher.push(segmentDir, segmentToPush, true); } catch (IOException e) { Assert.fail("should not throw exception"); @@ -246,7 +246,7 @@ private void testUsingSchemeForMultipleSegments(final String scheme, final int n } for (int i = 0; i < numberOfSegments; i++) { - final DataSegment pushedSegment = pusher.push(segmentDir, segments[i]); + final DataSegment pushedSegment = pusher.push(segmentDir, segments[i], true); String indexUri = StringUtils.format( "%s/%s/%d_index.zip", @@ -308,7 +308,7 @@ private void testUsingSchemeForMultipleSegments(final String scheme, final int n File outDir = new File(StringUtils.format("%s/%s", config.getStorageDirectory(), segmentPath)); outDir.setReadOnly(); try { - pusher.push(segmentDir, segments[i]); + pusher.push(segmentDir, segments[i], true); } catch (IOException e) { Assert.fail("should not throw exception"); diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java index 1100287cb966..333e7fae5dd1 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java @@ -88,7 +88,8 @@ public List getAllowedPropertyPrefixesForHadoop() } @Override - public DataSegment push(final File indexFilesDir, final DataSegment inSegment) throws IOException + public DataSegment push(final File indexFilesDir, final DataSegment inSegment, final boolean replaceExisting) + throws IOException { final String s3Path = S3Utils.constructSegmentPath(config.getBaseKey(), getStorageDir(inSegment)); @@ -105,21 +106,10 @@ public DataSegment push(final File indexFilesDir, final DataSegment inSegment) t public DataSegment call() throws Exception { S3Object toPush = new S3Object(zipOutFile); - - final String outputBucket = config.getBucket(); - final String s3DescriptorPath = S3Utils.descriptorPathForSegmentPath(s3Path); - - toPush.setBucketName(outputBucket); - toPush.setKey(s3Path); - if (!config.getDisableAcl()) { - toPush.setAcl(GSAccessControlList.REST_CANNED_BUCKET_OWNER_FULL_CONTROL); - } - - log.info("Pushing %s.", toPush); - s3Client.putObject(outputBucket, toPush); + putObject(config.getBucket(), s3Path, toPush, replaceExisting); final DataSegment outSegment = inSegment.withSize(indexSize) - .withLoadSpec(makeLoadSpec(outputBucket, toPush.getKey())) + .withLoadSpec(makeLoadSpec(config.getBucket(), toPush.getKey())) .withBinaryVersion(SegmentUtils.getVersionFromDir(indexFilesDir)); File descriptorFile = File.createTempFile("druid", "descriptor.json"); @@ -127,14 +117,13 @@ public DataSegment call() throws Exception // runtime, and because Guava deletes methods over time, that causes incompatibilities. Files.write(descriptorFile.toPath(), jsonMapper.writeValueAsBytes(outSegment)); S3Object descriptorObject = new S3Object(descriptorFile); - descriptorObject.setBucketName(outputBucket); - descriptorObject.setKey(s3DescriptorPath); - if (!config.getDisableAcl()) { - descriptorObject.setAcl(GSAccessControlList.REST_CANNED_BUCKET_OWNER_FULL_CONTROL); - } - log.info("Pushing %s", descriptorObject); - s3Client.putObject(outputBucket, descriptorObject); + putObject( + config.getBucket(), + S3Utils.descriptorPathForSegmentPath(s3Path), + descriptorObject, + replaceExisting + ); log.info("Deleting zipped index File[%s]", zipOutFile); zipOutFile.delete(); @@ -164,7 +153,6 @@ public Map makeLoadSpec(URI finalIndexZipFilePath) /** * Any change in loadSpec need to be reflected {@link io.druid.indexer.JobHelper#getURIFromSegment()} - * */ @SuppressWarnings("JavadocReference") private Map makeLoadSpec(String bucket, String key) @@ -180,4 +168,22 @@ private Map makeLoadSpec(String bucket, String key) config.isUseS3aSchema() ? "s3a" : "s3n" ); } + + private void putObject(String bucketName, String path, S3Object object, boolean replaceExisting) + throws ServiceException + { + object.setBucketName(bucketName); + object.setKey(path); + if (!config.getDisableAcl()) { + object.setAcl(GSAccessControlList.REST_CANNED_BUCKET_OWNER_FULL_CONTROL); + } + + log.info("Pushing %s.", object); + + if (!replaceExisting && S3Utils.isObjectInBucket(s3Client, bucketName, object.getKey())) { + log.info("Skipping push because key [%s] exists && replaceExisting == false", object.getKey()); + } else { + s3Client.putObject(bucketName, object); + } + } } diff --git a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java index 32818b17e487..f26bd1610398 100644 --- a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java +++ b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java @@ -113,7 +113,7 @@ public S3Object answer() throws Throwable size ); - DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush); + DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush, true); Assert.assertEquals(segmentToPush.getSize(), segment.getSize()); Assert.assertEquals(1, (int) segment.getBinaryVersion()); diff --git a/indexing-service/src/main/java/io/druid/indexing/common/index/YeOldePlumberSchool.java b/indexing-service/src/main/java/io/druid/indexing/common/index/YeOldePlumberSchool.java index b6e2a26d88fc..0bba29fbe881 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/index/YeOldePlumberSchool.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/index/YeOldePlumberSchool.java @@ -199,7 +199,10 @@ public void finishJob() .withDimensions(ImmutableList.copyOf(mappedSegment.getAvailableDimensions())) .withBinaryVersion(SegmentUtils.getVersionFromDir(fileToUpload)); - dataSegmentPusher.push(fileToUpload, segmentToUpload); + // This plumber is only used in batch ingestion situations where you do not have replica tasks pushing + // segments with the same identifier but potentially different contents. In case of conflict, favor the most + // recently pushed segment (replaceExisting == true). + dataSegmentPusher.push(fileToUpload, segmentToUpload, true); log.info( "Uploaded segment[%s]", diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/ConvertSegmentTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/ConvertSegmentTask.java index 4605995ee613..0e752cb988ce 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/ConvertSegmentTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/ConvertSegmentTask.java @@ -440,7 +440,11 @@ private void convertSegment(TaskToolbox toolbox) throws SegmentLoadingException, // Appending to the version makes a new version that inherits most comparability parameters of the original // version, but is "newer" than said original version. DataSegment updatedSegment = segment.withVersion(StringUtils.format("%s_v%s", segment.getVersion(), outVersion)); - updatedSegment = toolbox.getSegmentPusher().push(outLocation, updatedSegment); + + // The convert segment task does not support replicas where different tasks could generate segments with the + // same identifier but potentially different contents. In case of conflict, favor the most recently pushed + // segment (replaceExisting == true). + updatedSegment = toolbox.getSegmentPusher().push(outLocation, updatedSegment, true); actionClient.submit(new SegmentInsertAction(Sets.newHashSet(updatedSegment))); } else { diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java b/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java index 8a6cb91dea52..2b5574127d65 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java @@ -185,7 +185,11 @@ public String apply(DataSegment input) long uploadStart = System.currentTimeMillis(); // Upload file - final DataSegment uploadedSegment = toolbox.getSegmentPusher().push(fileToUpload, mergedSegment); + + // The merge task does not support replicas where different tasks could generate segments with the + // same identifier but potentially different contents. In case of conflict, favor the most recently pushed + // segment (replaceExisting == true). + final DataSegment uploadedSegment = toolbox.getSegmentPusher().push(fileToUpload, mergedSegment, true); emitter.emit(builder.build("merger/uploadTime", System.currentTimeMillis() - uploadStart)); emitter.emit(builder.build("merger/mergeSize", uploadedSegment.getSize())); diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java index 79310c9771de..ece5c55c7eb1 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java @@ -1006,7 +1006,7 @@ public String getPathForHadoop() } @Override - public DataSegment push(File file, DataSegment segment) throws IOException + public DataSegment push(File file, DataSegment segment, boolean replaceExisting) throws IOException { segments.add(segment); return segment; diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/SameIntervalMergeTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/SameIntervalMergeTaskTest.java index 63389a804bae..e42f55d9f605 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/SameIntervalMergeTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/SameIntervalMergeTaskTest.java @@ -201,7 +201,7 @@ public String getPathForHadoop() } @Override - public DataSegment push(File file, DataSegment segment) throws IOException + public DataSegment push(File file, DataSegment segment, boolean replaceExisting) throws IOException { // the merged segment is pushed to storage segments.add(segment); diff --git a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java index 624ddc4cc425..3a255ea1fd55 100644 --- a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java @@ -249,7 +249,7 @@ public String getPathForHadoop() } @Override - public DataSegment push(File file, DataSegment segment) throws IOException + public DataSegment push(File file, DataSegment segment, boolean replaceExisting) throws IOException { return segment; } diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java index e408bd28c2db..50912834680b 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java @@ -485,7 +485,7 @@ public String getPathForHadoop(String dataSource) } @Override - public DataSegment push(File file, DataSegment segment) throws IOException + public DataSegment push(File file, DataSegment segment, boolean replaceExisting) throws IOException { pushedSegments++; return segment; @@ -1034,7 +1034,7 @@ public String getPathForHadoop() } @Override - public DataSegment push(File file, DataSegment dataSegment) throws IOException + public DataSegment push(File file, DataSegment dataSegment, boolean replaceExisting) throws IOException { throw new RuntimeException("FAILURE"); } diff --git a/indexing-service/src/test/java/io/druid/indexing/test/TestDataSegmentPusher.java b/indexing-service/src/test/java/io/druid/indexing/test/TestDataSegmentPusher.java index 923a80299605..d518931a5faa 100644 --- a/indexing-service/src/test/java/io/druid/indexing/test/TestDataSegmentPusher.java +++ b/indexing-service/src/test/java/io/druid/indexing/test/TestDataSegmentPusher.java @@ -48,7 +48,7 @@ public String getPathForHadoop() } @Override - public DataSegment push(File file, DataSegment segment) throws IOException + public DataSegment push(File file, DataSegment segment, boolean replaceExisting) throws IOException { pushedSegments.add(segment); return segment; diff --git a/java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java b/java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java index 08614ff8d759..fd7bc1e6ec57 100644 --- a/java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/CompressionUtils.java @@ -48,36 +48,59 @@ public class CompressionUtils { private static final Logger log = new Logger(CompressionUtils.class); private static final int DEFAULT_RETRY_COUNT = 3; - - public static final String GZ_SUFFIX = ".gz"; - public static final String ZIP_SUFFIX = ".zip"; + private static final String GZ_SUFFIX = ".gz"; + private static final String ZIP_SUFFIX = ".zip"; /** * Zip the contents of directory into the file indicated by outputZipFile. Sub directories are skipped * * @param directory The directory whose contents should be added to the zip in the output stream. * @param outputZipFile The output file to write the zipped data to + * @param fsync True if the output file should be fsynced to disk * * @return The number of bytes (uncompressed) read from the input directory. * * @throws IOException */ - public static long zip(File directory, File outputZipFile) throws IOException + public static long zip(File directory, File outputZipFile, boolean fsync) throws IOException { if (!isZip(outputZipFile.getName())) { log.warn("No .zip suffix[%s], putting files from [%s] into it anyway.", outputZipFile, directory); } try (final FileOutputStream out = new FileOutputStream(outputZipFile)) { - return zip(directory, out); + long bytes = zip(directory, out); + + // For explanation of why fsyncing here is a good practice: + // https://github.com/druid-io/druid/pull/5187#pullrequestreview-85188984 + if (fsync) { + out.getChannel().force(true); + } + + return bytes; } } + /** + * Zip the contents of directory into the file indicated by outputZipFile. Sub directories are skipped + * + * @param directory The directory whose contents should be added to the zip in the output stream. + * @param outputZipFile The output file to write the zipped data to + * + * @return The number of bytes (uncompressed) read from the input directory. + * + * @throws IOException + */ + public static long zip(File directory, File outputZipFile) throws IOException + { + return zip(directory, outputZipFile, false); + } + /** * Zips the contents of the input directory to the output stream. Sub directories are skipped * * @param directory The directory whose contents should be added to the zip in the output stream. - * @param out The output stream to write the zip data to. It is closed in the process + * @param out The output stream to write the zip data to. Caller is responsible for closing this stream. * * @return The number of bytes (uncompressed) read from the input directory. * @@ -88,23 +111,23 @@ public static long zip(File directory, OutputStream out) throws IOException if (!directory.isDirectory()) { throw new IOE("directory[%s] is not a directory", directory); } - final File[] files = directory.listFiles(); + + final ZipOutputStream zipOut = new ZipOutputStream(out); long totalSize = 0; - try (final ZipOutputStream zipOut = new ZipOutputStream(out)) { - for (File file : files) { - log.info("Adding file[%s] with size[%,d]. Total size so far[%,d]", file, file.length(), totalSize); - if (file.length() >= Integer.MAX_VALUE) { - zipOut.finish(); - throw new IOE("file[%s] too large [%,d]", file, file.length()); - } - zipOut.putNextEntry(new ZipEntry(file.getName())); - totalSize += Files.asByteSource(file).copyTo(zipOut); + for (File file : directory.listFiles()) { + log.info("Adding file[%s] with size[%,d]. Total size so far[%,d]", file, file.length(), totalSize); + if (file.length() >= Integer.MAX_VALUE) { + zipOut.finish(); + throw new IOE("file[%s] too large [%,d]", file, file.length()); } - zipOut.closeEntry(); - // Workarround for http://hg.openjdk.java.net/jdk8/jdk8/jdk/rev/759aa847dcaf - zipOut.flush(); + zipOut.putNextEntry(new ZipEntry(file.getName())); + totalSize += Files.asByteSource(file).copyTo(zipOut); } + zipOut.closeEntry(); + // Workaround for http://hg.openjdk.java.net/jdk8/jdk8/jdk/rev/759aa847dcaf + zipOut.flush(); + zipOut.finish(); return totalSize; } diff --git a/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java index b6e52bd551d3..0c00e0aeb334 100644 --- a/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java +++ b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import io.druid.java.util.common.CompressionUtils; +import io.druid.java.util.common.IOE; import io.druid.java.util.common.logger.Logger; import io.druid.segment.SegmentUtils; import io.druid.timeline.DataSegment; @@ -33,23 +34,22 @@ import java.net.URI; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; +import java.nio.file.StandardOpenOption; import java.util.Map; import java.util.UUID; -/** - */ public class LocalDataSegmentPusher implements DataSegmentPusher { private static final Logger log = new Logger(LocalDataSegmentPusher.class); + private static final String INDEX_FILENAME = "index.zip"; + private static final String DESCRIPTOR_FILENAME = "descriptor.json"; + private final LocalDataSegmentPusherConfig config; private final ObjectMapper jsonMapper; @Inject - public LocalDataSegmentPusher( - LocalDataSegmentPusherConfig config, - ObjectMapper jsonMapper - ) + public LocalDataSegmentPusher(LocalDataSegmentPusherConfig config, ObjectMapper jsonMapper) { this.config = config; this.jsonMapper = jsonMapper; @@ -71,7 +71,7 @@ public String getPathForHadoop(String dataSource) } @Override - public DataSegment push(File dataSegmentFile, DataSegment segment) throws IOException + public DataSegment push(File dataSegmentFile, DataSegment segment, boolean replaceExisting) throws IOException { final String storageDir = this.getStorageDir(segment); final File baseStorageDir = config.getStorageDirectory(); @@ -95,27 +95,53 @@ public DataSegment push(File dataSegmentFile, DataSegment segment) throws IOExce final File tmpOutDir = new File(baseStorageDir, intermediateDirFor(storageDir)); log.info("Creating intermediate directory[%s] for segment[%s]", tmpOutDir.toString(), segment.getIdentifier()); - final long size = compressSegment(dataSegmentFile, tmpOutDir); - - final DataSegment dataSegment = createDescriptorFile( - segment.withLoadSpec(makeLoadSpec(new File(outDir, "index.zip").toURI())) - .withSize(size) - .withBinaryVersion(SegmentUtils.getVersionFromDir(dataSegmentFile)), - tmpOutDir - ); + FileUtils.forceMkdir(tmpOutDir); - // moving the temporary directory to the final destination, once success the potentially concurrent push operations - // will be failed and will read the descriptor.json created by current push operation directly - FileUtils.forceMkdir(outDir.getParentFile()); try { - Files.move(tmpOutDir.toPath(), outDir.toPath()); + final File tmpIndexFile = new File(tmpOutDir, INDEX_FILENAME); + final long size = compressSegment(dataSegmentFile, tmpIndexFile); + + final File tmpDescriptorFile = new File(tmpOutDir, DESCRIPTOR_FILENAME); + DataSegment dataSegment = createDescriptorFile( + segment.withLoadSpec(makeLoadSpec(new File(outDir, INDEX_FILENAME).toURI())) + .withSize(size) + .withBinaryVersion(SegmentUtils.getVersionFromDir(dataSegmentFile)), + tmpDescriptorFile + ); + + FileUtils.forceMkdir(outDir); + if (replaceExisting) { + final File indexFileTarget = new File(outDir, tmpIndexFile.getName()); + final File descriptorFileTarget = new File(outDir, tmpDescriptorFile.getName()); + + if (!tmpIndexFile.renameTo(indexFileTarget)) { + throw new IOE("Failed to rename [%s] to [%s]", tmpIndexFile, indexFileTarget); + } + + if (!tmpDescriptorFile.renameTo(descriptorFileTarget)) { + throw new IOE("Failed to rename [%s] to [%s]", tmpDescriptorFile, descriptorFileTarget); + } + } else { + try { + Files.move(tmpIndexFile.toPath(), outDir.toPath().resolve(tmpIndexFile.toPath().getFileName())); + } + catch (FileAlreadyExistsException e) { + log.info("[%s] already exists at [%s], ignore if replication is configured", INDEX_FILENAME, outDir); + } + try { + Files.move(tmpDescriptorFile.toPath(), outDir.toPath().resolve(tmpDescriptorFile.toPath().getFileName())); + } + catch (FileAlreadyExistsException e) { + log.info("[%s] already exists at [%s], ignore if replication is configured", DESCRIPTOR_FILENAME, outDir); + dataSegment = jsonMapper.readValue(new File(outDir, DESCRIPTOR_FILENAME), DataSegment.class); + } + } + + return dataSegment; } - catch (FileAlreadyExistsException e) { - log.warn("Push destination directory[%s] exists, ignore this message if replication is configured.", outDir); + finally { FileUtils.deleteDirectory(tmpOutDir); - return jsonMapper.readValue(new File(outDir, "descriptor.json"), DataSegment.class); } - return dataSegment; } @Override @@ -129,21 +155,21 @@ private String intermediateDirFor(String storageDir) return "intermediate_pushes/" + storageDir + "." + UUID.randomUUID().toString(); } - private long compressSegment(File dataSegmentFile, File outDir) throws IOException + private long compressSegment(File dataSegmentFile, File dest) throws IOException { - FileUtils.forceMkdir(outDir); - File outFile = new File(outDir, "index.zip"); - log.info("Compressing files from[%s] to [%s]", dataSegmentFile, outFile); - return CompressionUtils.zip(dataSegmentFile, outFile); + log.info("Compressing files from[%s] to [%s]", dataSegmentFile, dest); + return CompressionUtils.zip(dataSegmentFile, dest, true); } - private DataSegment createDescriptorFile(DataSegment segment, File outDir) throws IOException + private DataSegment createDescriptorFile(DataSegment segment, File dest) throws IOException { - File descriptorFile = new File(outDir, "descriptor.json"); - log.info("Creating descriptor file at[%s]", descriptorFile); + log.info("Creating descriptor file at[%s]", dest); // Avoid using Guava in DataSegmentPushers because they might be used with very diverse Guava versions in // runtime, and because Guava deletes methods over time, that causes incompatibilities. - Files.write(descriptorFile.toPath(), jsonMapper.writeValueAsBytes(segment)); + Files.write( + dest.toPath(), jsonMapper.writeValueAsBytes(segment), StandardOpenOption.CREATE, StandardOpenOption.SYNC + ); + return segment; } } diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java index aec0f751a3b6..c8fd9c52d41d 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java @@ -556,6 +556,10 @@ private ListenableFuture pushBarrier() * Merge segment, push to deep storage. Should only be used on segments that have been fully persisted. Must only * be run in the single-threaded pushExecutor. * + * Note that this calls DataSegmentPusher.push() with replaceExisting == true which is appropriate for the indexing + * tasks it is currently being used for (local indexing and Kafka indexing). If this is going to be used by an + * indexing task type that requires replaceExisting == false, this setting will need to be pushed to the caller. + * * @param identifier sink identifier * @param sink sink to push * @@ -633,9 +637,18 @@ private DataSegment mergeAndPush(final SegmentIdentifier identifier, final Sink // Retry pushing segments because uploading to deep storage might fail especially for cloud storage types final DataSegment segment = RetryUtils.retry( + // The appenderator is currently being used for the local indexing task and the Kafka indexing task. For the + // Kafka indexing task, pushers MUST overwrite any existing objects in deep storage with the same identifier + // in order to maintain exactly-once semantics. If they do not and instead favor existing objects, in case of + // failure during publishing, the indexed data may not represent the checkpointed state and data loss or + // duplication may occur. See: https://github.com/druid-io/druid/issues/5161. The local indexing task does not + // support replicas where different tasks could generate segments with the same identifier but potentially + // different contents so it is okay if existing objects are overwritten. In both of these cases, we want to + // favor the most recently pushed segment so replaceExisting == true. () -> dataSegmentPusher.push( mergedFile, - sink.getSegment().withDimensions(IndexMerger.getMergedDimensionsFromQueryableIndexes(indexes)) + sink.getSegment().withDimensions(IndexMerger.getMergedDimensionsFromQueryableIndexes(indexes)), + true ), exception -> exception instanceof Exception, 5 diff --git a/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java b/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java index 3a19ed632161..345e687cf2e5 100644 --- a/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java +++ b/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java @@ -444,9 +444,17 @@ public void doRun() log.info("Pushing [%s] to deep storage", sink.getSegment().getIdentifier()); + // The realtime plumber can generate segments with the same identifier (i.e. replica tasks) but does not + // have any strict requirement that the contents of these segments be identical. It is possible that two + // tasks generate a segment with the same identifier containing different data, and in this situation we + // want to favor the data from the task which pushed first. This is because it is possible that one + // historical could load the segment after the first task pushed and another historical load the same + // segment after the second task pushed. If the second task's segment overwrote the first one, the second + // historical node would be serving different data from the first. Hence set replaceExisting == false. DataSegment segment = dataSegmentPusher.push( mergedFile, - sink.getSegment().withDimensions(IndexMerger.getMergedDimensionsFromQueryableIndexes(indexes)) + sink.getSegment().withDimensions(IndexMerger.getMergedDimensionsFromQueryableIndexes(indexes)), + false ); log.info("Inserting [%s] to the metadata store", sink.getSegment().getIdentifier()); segmentPublisher.publishSegment(segment); diff --git a/server/src/test/java/io/druid/segment/loading/LocalDataSegmentPusherTest.java b/server/src/test/java/io/druid/segment/loading/LocalDataSegmentPusherTest.java index 2309735424d5..e8168cb668fe 100644 --- a/server/src/test/java/io/druid/segment/loading/LocalDataSegmentPusherTest.java +++ b/server/src/test/java/io/druid/segment/loading/LocalDataSegmentPusherTest.java @@ -23,11 +23,13 @@ import com.google.common.collect.ImmutableList; import com.google.common.io.Files; import com.google.common.primitives.Ints; +import io.druid.java.util.common.CompressionUtils; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.StringUtils; import io.druid.segment.TestHelper; import io.druid.timeline.DataSegment; import io.druid.timeline.partition.NoneShardSpec; +import org.apache.commons.io.FileUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -54,7 +56,18 @@ public class LocalDataSegmentPusherTest Intervals.utc(0, 1), "v1", null, + ImmutableList.of("dim1"), null, + NoneShardSpec.instance(), + null, + -1 + ); + DataSegment dataSegment2 = new DataSegment( + "ds", + Intervals.utc(0, 1), + "v1", + null, + ImmutableList.of("dim2"), null, NoneShardSpec.instance(), null, @@ -79,8 +92,8 @@ public void testPush() throws IOException */ final DataSegment dataSegment2 = dataSegment.withVersion("v2"); - DataSegment returnSegment1 = localDataSegmentPusher.push(dataSegmentFiles, dataSegment); - DataSegment returnSegment2 = localDataSegmentPusher.push(dataSegmentFiles, dataSegment2); + DataSegment returnSegment1 = localDataSegmentPusher.push(dataSegmentFiles, dataSegment, true); + DataSegment returnSegment2 = localDataSegmentPusher.push(dataSegmentFiles, dataSegment2, true); Assert.assertNotNull(returnSegment1); Assert.assertEquals(dataSegment, returnSegment1); @@ -106,14 +119,45 @@ public void testPush() throws IOException } @Test - public void testFirstPushWinsForConcurrentPushes() throws IOException + public void testFirstPushWinsForConcurrentPushesWhenReplaceExistingFalse() throws IOException { File replicatedDataSegmentFiles = temporaryFolder.newFolder(); Files.asByteSink(new File(replicatedDataSegmentFiles, "version.bin")).write(Ints.toByteArray(0x8)); - DataSegment returnSegment1 = localDataSegmentPusher.push(dataSegmentFiles, dataSegment); - DataSegment returnSegment2 = localDataSegmentPusher.push(replicatedDataSegmentFiles, dataSegment); + DataSegment returnSegment1 = localDataSegmentPusher.push(dataSegmentFiles, dataSegment, false); + DataSegment returnSegment2 = localDataSegmentPusher.push(replicatedDataSegmentFiles, dataSegment2, false); + + Assert.assertEquals(dataSegment.getDimensions(), returnSegment1.getDimensions()); + Assert.assertEquals(dataSegment.getDimensions(), returnSegment2.getDimensions()); + + File unzipDir = new File(config.storageDirectory, "unzip"); + FileUtils.forceMkdir(unzipDir); + CompressionUtils.unzip( + new File(config.storageDirectory, "/ds/1970-01-01T00:00:00.000Z_1970-01-01T00:00:00.001Z/v1/0/index.zip"), + unzipDir + ); + + Assert.assertEquals(0x9, Ints.fromByteArray(Files.toByteArray(new File(unzipDir, "version.bin")))); + } + + @Test + public void testLastPushWinsForConcurrentPushesWhenReplaceExistingTrue() throws IOException + { + File replicatedDataSegmentFiles = temporaryFolder.newFolder(); + Files.asByteSink(new File(replicatedDataSegmentFiles, "version.bin")).write(Ints.toByteArray(0x8)); + DataSegment returnSegment1 = localDataSegmentPusher.push(dataSegmentFiles, dataSegment, true); + DataSegment returnSegment2 = localDataSegmentPusher.push(replicatedDataSegmentFiles, dataSegment2, true); + + Assert.assertEquals(dataSegment.getDimensions(), returnSegment1.getDimensions()); + Assert.assertEquals(dataSegment2.getDimensions(), returnSegment2.getDimensions()); + + File unzipDir = new File(config.storageDirectory, "unzip"); + FileUtils.forceMkdir(unzipDir); + CompressionUtils.unzip( + new File(config.storageDirectory, "/ds/1970-01-01T00:00:00.000Z_1970-01-01T00:00:00.001Z/v1/0/index.zip"), + unzipDir + ); - Assert.assertEquals(returnSegment1, returnSegment2); + Assert.assertEquals(0x8, Ints.fromByteArray(Files.toByteArray(new File(unzipDir, "version.bin")))); } @Test @@ -124,7 +168,7 @@ public void testPushCannotCreateDirectory() throws IOException config.storageDirectory = new File(config.storageDirectory, "xxx"); Assert.assertTrue(config.storageDirectory.mkdir()); config.storageDirectory.setWritable(false); - localDataSegmentPusher.push(dataSegmentFiles, dataSegment); + localDataSegmentPusher.push(dataSegmentFiles, dataSegment, true); } @Test diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTester.java b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTester.java index 555e05d20673..186434c01167 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTester.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTester.java @@ -193,7 +193,7 @@ public String getPathForHadoop() } @Override - public DataSegment push(File file, DataSegment segment) throws IOException + public DataSegment push(File file, DataSegment segment, boolean replaceExisting) throws IOException { if (enablePushFailure && mustFail) { mustFail = false; diff --git a/services/src/main/java/io/druid/cli/CliRealtimeExample.java b/services/src/main/java/io/druid/cli/CliRealtimeExample.java index b4ebfc3eeee7..a50f51bf0204 100644 --- a/services/src/main/java/io/druid/cli/CliRealtimeExample.java +++ b/services/src/main/java/io/druid/cli/CliRealtimeExample.java @@ -155,7 +155,7 @@ public String getPathForHadoop(String dataSource) } @Override - public DataSegment push(File file, DataSegment segment) throws IOException + public DataSegment push(File file, DataSegment segment, boolean replaceExisting) throws IOException { return segment; } From f3dec31a107663af23a957fb1fb06e14fc738eff Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Thu, 4 Jan 2018 15:39:31 -0800 Subject: [PATCH 04/82] Upgrade to Calcite 1.15.0 (#5210) (#5214) * Upgrade to Calcite 1.15.0 * Use Filtration.eternity() --- docs/content/misc/math-expr.md | 2 +- pom.xml | 2 +- .../expression/TimestampExtractExprMacro.java | 2 +- .../calcite/expression/DruidExpression.java | 2 +- .../builtin/ExtractOperatorConversion.java | 6 +- .../druid/sql/calcite/CalciteQueryTest.java | 101 ++++++++++++++++-- 6 files changed, 96 insertions(+), 19 deletions(-) diff --git a/docs/content/misc/math-expr.md b/docs/content/misc/math-expr.md index 07dcc5018082..abcebdd3b5e3 100644 --- a/docs/content/misc/math-expr.md +++ b/docs/content/misc/math-expr.md @@ -62,7 +62,7 @@ The following built-in functions are available. |timestamp_ceil|timestamp_ceil(expr, period, \[origin, \[timezone\]\]) rounds up a timestamp, returning it as a new timestamp. Period can be any ISO8601 period, like P3M (quarters) or PT12H (half-days). The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00".| |timestamp_floor|timestamp_floor(expr, period, \[origin, [timezone\]\]) rounds down a timestamp, returning it as a new timestamp. Period can be any ISO8601 period, like P3M (quarters) or PT12H (half-days). The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00".| |timestamp_shift|timestamp_shift(expr, period, step, \[timezone\]) shifts a timestamp by a period (step times), returning it as a new timestamp. Period can be any ISO8601 period. Step may be negative. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00".| -|timestamp_extract|timestamp_extract(expr, unit, \[timezone\]) extracts a time part from expr, returning it as a number. Unit can be EPOCH, SECOND, MINUTE, HOUR, DAY (day of month), DOW (day of week), DOY (day of year), WEEK (week of [week year](https://en.wikipedia.org/wiki/ISO_week_date)), MONTH (1 through 12), QUARTER (1 through 4), or YEAR. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00"| +|timestamp_extract|timestamp_extract(expr, unit, \[timezone\]) extracts a time part from expr, returning it as a number. Unit can be EPOCH (number of seconds since 1970-01-01 00:00:00 UTC), SECOND, MINUTE, HOUR, DAY (day of month), DOW (day of week), DOY (day of year), WEEK (week of [week year](https://en.wikipedia.org/wiki/ISO_week_date)), MONTH (1 through 12), QUARTER (1 through 4), or YEAR. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00"| |timestamp_parse|timestamp_parse(string expr, \[pattern, [timezone\]\]) parses a string into a timestamp using a given [Joda DateTimeFormat pattern](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html). If the pattern is not provided, this parses time strings in either ISO8601 or SQL format. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00", and will be used as the time zone for strings that do not include a time zone offset. Pattern and time zone must be literals. Strings that cannot be parsed as timestamps will be returned as nulls.| |timestamp_format|timestamp_format(expr, \[pattern, \[timezone\]\]) formats a timestamp as a string with a given [Joda DateTimeFormat pattern](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html), or ISO8601 if the pattern is not provided. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00". Pattern and time zone must be literals.| diff --git a/pom.xml b/pom.xml index a4c039532dfc..fe9546068c3d 100644 --- a/pom.xml +++ b/pom.xml @@ -62,7 +62,7 @@ 4.0.0 2.12.0 1.10.0 - 1.14.0 + 1.15.0 16.0.1 4.1.0 9.3.19.v20170502 diff --git a/processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java b/processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java index 19c64415e5fa..bb1b2ca5f243 100644 --- a/processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java +++ b/processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java @@ -91,7 +91,7 @@ public ExprEval eval(final ObjectBinding bindings) final DateTime dateTime = new DateTime(arg.eval(bindings).asLong(), chronology); switch (unit) { case EPOCH: - return ExprEval.of(dateTime.getMillis()); + return ExprEval.of(dateTime.getMillis() / 1000); case SECOND: return ExprEval.of(dateTime.secondOfMinute().get()); case MINUTE: diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java b/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java index 86475a390492..b779fdd03a0c 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/DruidExpression.java @@ -35,7 +35,7 @@ import java.util.function.Function; /** - * Represents three kinds of expression-like concepts that native Druid queries support: + * Represents two kinds of expression-like concepts that native Druid queries support: * * (1) SimpleExtractions, which are direct column access, possibly with an extractionFn * (2) native Druid expressions. diff --git a/sql/src/main/java/io/druid/sql/calcite/expression/builtin/ExtractOperatorConversion.java b/sql/src/main/java/io/druid/sql/calcite/expression/builtin/ExtractOperatorConversion.java index 8b6de78be82b..e2cd31005e5a 100644 --- a/sql/src/main/java/io/druid/sql/calcite/expression/builtin/ExtractOperatorConversion.java +++ b/sql/src/main/java/io/druid/sql/calcite/expression/builtin/ExtractOperatorConversion.java @@ -39,6 +39,7 @@ public class ExtractOperatorConversion implements SqlOperatorConversion { private static final Map EXTRACT_UNIT_MAP = ImmutableMap.builder() + .put(TimeUnitRange.EPOCH, TimestampExtractExprMacro.Unit.EPOCH) .put(TimeUnitRange.SECOND, TimestampExtractExprMacro.Unit.SECOND) .put(TimeUnitRange.MINUTE, TimestampExtractExprMacro.Unit.MINUTE) .put(TimeUnitRange.HOUR, TimestampExtractExprMacro.Unit.HOUR) @@ -75,11 +76,6 @@ public DruidExpression toDruidExpression( return null; } - if (call.getOperator().getName().equals("EXTRACT_DATE")) { - // Arg will be in number of days since the epoch. Can't translate. - return null; - } - final TimestampExtractExprMacro.Unit druidUnit = EXTRACT_UNIT_MAP.get(calciteUnit); if (druidUnit == null) { // Don't know how to extract this time unit. diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index 97502ea8c8cd..7939ebec1acc 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -2514,7 +2514,6 @@ public void testCaseFilteredAggregationWithGroupBy() throws Exception } @Test - @Ignore // https://issues.apache.org/jira/browse/CALCITE-1910 public void testFilteredAggregationWithNotIn() throws Exception { testQuery( @@ -2527,12 +2526,26 @@ public void testFilteredAggregationWithNotIn() throws Exception .dataSource(CalciteTests.DATASOURCE1) .intervals(QSS(Filtration.eternity())) .granularity(Granularities.ALL) - .aggregators(AGGS()) + .aggregators( + AGGS( + new FilteredAggregatorFactory( + new CountAggregatorFactory("a0"), + NOT(SELECTOR("dim1", "1", null)) + ), + new FilteredAggregatorFactory( + new CountAggregatorFactory("a1"), + AND( + NOT(SELECTOR("dim2", null, null)), + NOT(SELECTOR("dim1", "1", null)) + ) + ) + ) + ) .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), ImmutableList.of( - new Object[]{1L, 5L} + new Object[]{5L, 2L} ) ); } @@ -2817,6 +2830,27 @@ public void testCountStarWithNotOfDegenerateFilter() throws Exception ); } + @Test + public void testCountStarWithBoundFilterSimplifyOnMetric() throws Exception + { + testQuery( + "SELECT COUNT(*) FROM druid.foo WHERE 2.5 < m1 AND m1 < 3.5", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters(BOUND("m1", "2.5", "3.5", true, true, null, StringComparators.NUMERIC)) + .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{1L} + ) + ); + } + @Test public void testCountStarWithBoundFilterSimplifyOr() throws Exception { @@ -3201,6 +3235,39 @@ public void testCountStarWithTimeFilterOnLongColumnUsingExtractEpoch() throws Ex ); } + @Test + public void testCountStarWithTimeFilterOnLongColumnUsingExtractEpochFromDate() throws Exception + { + testQuery( + "SELECT COUNT(*) FROM druid.foo WHERE " + + "cnt >= EXTRACT(EPOCH FROM DATE '1970-01-01') * 1000 " + + "AND cnt < EXTRACT(EPOCH FROM DATE '1970-01-02') * 1000", + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(QSS(Filtration.eternity())) + .granularity(Granularities.ALL) + .filters( + BOUND( + "cnt", + String.valueOf(DateTimes.of("1970-01-01").getMillis()), + String.valueOf(DateTimes.of("1970-01-02").getMillis()), + false, + true, + null, + StringComparators.NUMERIC + ) + ) + .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .context(TIMESERIES_CONTEXT_DEFAULT) + .build() + ), + ImmutableList.of( + new Object[]{6L} + ) + ); + } + @Test public void testCountStarWithTimeFilterOnLongColumnUsingTimestampToMillis() throws Exception { @@ -4918,7 +4985,6 @@ public void testFilterOnTimeFloorComparisonMisaligned() throws Exception } @Test - @Ignore // https://issues.apache.org/jira/browse/CALCITE-1601 public void testFilterOnTimeExtract() throws Exception { testQuery( @@ -4928,9 +4994,15 @@ public void testFilterOnTimeExtract() throws Exception ImmutableList.of( Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) - .intervals(QSS(Intervals.of("2000/P1M"))) + .intervals(QSS(Filtration.eternity())) .granularity(Granularities.ALL) .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .filters( + AND( + EXPRESSION_FILTER("(timestamp_extract(\"__time\",'YEAR','UTC') == 2000)"), + EXPRESSION_FILTER("(timestamp_extract(\"__time\",'MONTH','UTC') == 1)") + ) + ) .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), @@ -4941,24 +5013,33 @@ public void testFilterOnTimeExtract() throws Exception } @Test - @Ignore // https://issues.apache.org/jira/browse/CALCITE-1601 - public void testFilterOnTimeExtractWithMultipleMonths() throws Exception + public void testFilterOnTimeExtractWithMultipleDays() throws Exception { testQuery( "SELECT COUNT(*) FROM druid.foo\n" + "WHERE EXTRACT(YEAR FROM __time) = 2000\n" - + "AND EXTRACT(MONTH FROM __time) IN (2, 3, 5)", + + "AND EXTRACT(DAY FROM __time) IN (2, 3, 5)", ImmutableList.of( Druids.newTimeseriesQueryBuilder() .dataSource(CalciteTests.DATASOURCE1) - .intervals(QSS(Intervals.of("2000-02-01/P2M"), Intervals.of("2000-05-01/P1M"))) + .intervals(QSS(Filtration.eternity())) .granularity(Granularities.ALL) .aggregators(AGGS(new CountAggregatorFactory("a0"))) + .filters( + AND( + EXPRESSION_FILTER("(timestamp_extract(\"__time\",'YEAR','UTC') == 2000)"), + OR( + EXPRESSION_FILTER("(timestamp_extract(\"__time\",'DAY','UTC') == 2)"), + EXPRESSION_FILTER("(timestamp_extract(\"__time\",'DAY','UTC') == 3)"), + EXPRESSION_FILTER("(timestamp_extract(\"__time\",'DAY','UTC') == 5)") + ) + ) + ) .context(TIMESERIES_CONTEXT_DEFAULT) .build() ), ImmutableList.of( - new Object[]{3L} + new Object[]{2L} ) ); } From a15becb47b7a7370c7a5d4ba907d31230afa0bb9 Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Fri, 5 Jan 2018 14:34:44 -0800 Subject: [PATCH 05/82] Add missing auth doc links (#5224) (#5225) --- docs/content/development/extensions.md | 1 + docs/content/toc.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/content/development/extensions.md b/docs/content/development/extensions.md index bc7283060425..a9e1d1ee2f05 100644 --- a/docs/content/development/extensions.md +++ b/docs/content/development/extensions.md @@ -22,6 +22,7 @@ Core extensions are maintained by Druid committers. |Name|Description|Docs| |----|-----------|----| |druid-avro-extensions|Support for data in Apache Avro data format.|[link](../development/extensions-core/avro.html)| +|druid-basic-security|Support for Basic HTTP authentication and role-based access control.|[link](../development/extensions-core/druid-basic-security.html)| |druid-caffeine-cache|A local cache implementation backed by Caffeine.|[link](../development/extensions-core/caffeine-cache.html)| |druid-datasketches|Support for approximate counts and set operations with [DataSketches](http://datasketches.github.io/).|[link](../development/extensions-core/datasketches-aggregators.html)| |druid-hdfs-storage|HDFS deep storage.|[link](../development/extensions-core/hdfs.html)| diff --git a/docs/content/toc.md b/docs/content/toc.md index d18de0e7c734..e51f95c05d46 100644 --- a/docs/content/toc.md +++ b/docs/content/toc.md @@ -90,6 +90,7 @@ layout: toc * [Broker](/docs/VERSION/configuration/broker.html) * [Realtime](/docs/VERSION/configuration/realtime.html) * [Configuring Logging](/docs/VERSION/configuration/logging.html) + * [Configuring Authentication and Authorization](/docs/VERSION/configuration/auth.html) ## Development * [Overview](/docs/VERSION/development/overview.html) From 555580921f9cada9f1d6f95c532437aa86a79b01 Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Fri, 5 Jan 2018 17:34:48 -0800 Subject: [PATCH 06/82] Exclude sketches-core from druid-sql (#5223) (#5226) --- sql/pom.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sql/pom.xml b/sql/pom.xml index 265390e8e7a6..da49d38fc62b 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -41,6 +41,18 @@ org.apache.calcite calcite-core + + + + com.yahoo.datasketches + sketches-core + + org.apache.calcite From ee96a6a6604f2f650597191ba10b4e782f7ef1c9 Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Tue, 9 Jan 2018 11:03:33 -0800 Subject: [PATCH 07/82] Throw away rows with timestamps beyond long bounds in kafka indexing (#5215) (#5232) * Throw away rows with timestamps beyond long bounds in kafka indexing * PR comments --- .../druid/indexing/kafka/KafkaIndexTask.java | 14 +++++++++++++ .../indexing/kafka/KafkaIndexTaskTest.java | 21 ++++++++++--------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java index d4fee282bde5..5082a8060cb4 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java +++ b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java @@ -65,6 +65,7 @@ import io.druid.indexing.kafka.supervisor.KafkaSupervisor; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.ISE; +import io.druid.java.util.common.Intervals; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.collect.Utils; import io.druid.java.util.common.concurrent.Execs; @@ -2015,6 +2016,19 @@ private boolean withinMinMaxRecordTime(final InputRow row) final boolean afterMaximumMessageTime = ioConfig.getMaximumMessageTime().isPresent() && ioConfig.getMaximumMessageTime().get().isBefore(row.getTimestamp()); + if (!Intervals.ETERNITY.contains(row.getTimestamp())) { + final String errorMsg = StringUtils.format( + "Encountered row with timestamp that cannot be represented as a long: [%s]", + row + ); + log.debug(errorMsg); + if (tuningConfig.isReportParseExceptions()) { + throw new ParseException(errorMsg); + } else { + return false; + } + } + if (log.isDebugEnabled()) { if (beforeMinimumMessageTime) { log.debug( diff --git a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java index 298d1490f882..c2e1700df8a3 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java @@ -241,6 +241,7 @@ private static List> generateRecords(String topic new ProducerRecord(topic, 0, null, JB("2010", "c", "y", 1.0f)), new ProducerRecord(topic, 0, null, JB("2011", "d", "y", 1.0f)), new ProducerRecord(topic, 0, null, JB("2011", "e", "y", 1.0f)), + new ProducerRecord(topic, 0, null, JB("246140482-04-24T15:36:27.903Z", "x", "z", 1.0f)), new ProducerRecord(topic, 0, null, StringUtils.toUtf8("unparseable")), new ProducerRecord(topic, 0, null, null), new ProducerRecord(topic, 0, null, JB("2013", "f", "y", 1.0f)), @@ -456,7 +457,7 @@ public void testIncrementalHandOff() throws Exception // of events fetched across two partitions from Kafka final KafkaPartitions checkpoint1 = new KafkaPartitions(topic, ImmutableMap.of(0, 5L, 1, 0L)); final KafkaPartitions checkpoint2 = new KafkaPartitions(topic, ImmutableMap.of(0, 4L, 1, 2L)); - final KafkaPartitions endPartitions = new KafkaPartitions(topic, ImmutableMap.of(0, 8L, 1, 2L)); + final KafkaPartitions endPartitions = new KafkaPartitions(topic, ImmutableMap.of(0, 9L, 1, 2L)); final KafkaIndexTask task = createTask( null, new KafkaIOConfig( @@ -494,7 +495,7 @@ public void testIncrementalHandOff() throws Exception // Check metrics Assert.assertEquals(8, task.getFireDepartmentMetrics().processed()); Assert.assertEquals(1, task.getFireDepartmentMetrics().unparseable()); - Assert.assertEquals(1, task.getFireDepartmentMetrics().thrownAway()); + Assert.assertEquals(2, task.getFireDepartmentMetrics().thrownAway()); // Check published metadata SegmentDescriptor desc1 = SD(task, "2008/P1D", 0); @@ -506,7 +507,7 @@ public void testIncrementalHandOff() throws Exception SegmentDescriptor desc7 = SD(task, "2013/P1D", 0); Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3, desc4, desc5, desc6, desc7), publishedDescriptors()); Assert.assertEquals( - new KafkaDataSourceMetadata(new KafkaPartitions(topic, ImmutableMap.of(0, 8L, 1, 2L))), + new KafkaDataSourceMetadata(new KafkaPartitions(topic, ImmutableMap.of(0, 9L, 1, 2L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); @@ -970,7 +971,7 @@ public void testRunConflicting() throws Exception new KafkaIOConfig( "sequence1", new KafkaPartitions(topic, ImmutableMap.of(0, 3L)), - new KafkaPartitions(topic, ImmutableMap.of(0, 8L)), + new KafkaPartitions(topic, ImmutableMap.of(0, 9L)), kafkaServer.consumerProperties(), true, false, @@ -1001,7 +1002,7 @@ public void testRunConflicting() throws Exception Assert.assertEquals(0, task1.getFireDepartmentMetrics().thrownAway()); Assert.assertEquals(3, task2.getFireDepartmentMetrics().processed()); Assert.assertEquals(1, task2.getFireDepartmentMetrics().unparseable()); - Assert.assertEquals(1, task2.getFireDepartmentMetrics().thrownAway()); + Assert.assertEquals(2, task2.getFireDepartmentMetrics().thrownAway()); // Check published segments & metadata, should all be from the first task SegmentDescriptor desc1 = SD(task1, "2010/P1D", 0); @@ -1039,7 +1040,7 @@ public void testRunConflictingWithoutTransactions() throws Exception new KafkaIOConfig( "sequence1", new KafkaPartitions(topic, ImmutableMap.of(0, 3L)), - new KafkaPartitions(topic, ImmutableMap.of(0, 8L)), + new KafkaPartitions(topic, ImmutableMap.of(0, 9L)), kafkaServer.consumerProperties(), false, false, @@ -1076,7 +1077,7 @@ public void testRunConflictingWithoutTransactions() throws Exception Assert.assertEquals(0, task1.getFireDepartmentMetrics().thrownAway()); Assert.assertEquals(3, task2.getFireDepartmentMetrics().processed()); Assert.assertEquals(1, task2.getFireDepartmentMetrics().unparseable()); - Assert.assertEquals(1, task2.getFireDepartmentMetrics().thrownAway()); + Assert.assertEquals(2, task2.getFireDepartmentMetrics().thrownAway()); // Check published segments & metadata SegmentDescriptor desc3 = SD(task2, "2011/P1D", 1); @@ -1449,7 +1450,7 @@ public void testRunAndPauseAfterReadWithModifiedEndOffsets() throws Exception Assert.assertEquals(KafkaIndexTask.Status.PAUSED, task.getStatus()); // try again but with resume flag == true - newEndOffsets = ImmutableMap.of(0, 6L); + newEndOffsets = ImmutableMap.of(0, 7L); task.setEndOffsets(newEndOffsets, true, true); Assert.assertEquals(newEndOffsets, task.getEndOffsets()); Assert.assertNotEquals(KafkaIndexTask.Status.PAUSED, task.getStatus()); @@ -1468,7 +1469,7 @@ public void testRunAndPauseAfterReadWithModifiedEndOffsets() throws Exception // Check metrics Assert.assertEquals(4, task.getFireDepartmentMetrics().processed()); Assert.assertEquals(1, task.getFireDepartmentMetrics().unparseable()); - Assert.assertEquals(0, task.getFireDepartmentMetrics().thrownAway()); + Assert.assertEquals(1, task.getFireDepartmentMetrics().thrownAway()); // Check published metadata SegmentDescriptor desc1 = SD(task, "2009/P1D", 0); @@ -1476,7 +1477,7 @@ public void testRunAndPauseAfterReadWithModifiedEndOffsets() throws Exception SegmentDescriptor desc3 = SD(task, "2011/P1D", 0); Assert.assertEquals(ImmutableSet.of(desc1, desc2, desc3), publishedDescriptors()); Assert.assertEquals( - new KafkaDataSourceMetadata(new KafkaPartitions(topic, ImmutableMap.of(0, 6L))), + new KafkaDataSourceMetadata(new KafkaPartitions(topic, ImmutableMap.of(0, 7L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) ); From d14b261270ec74aeb381fdf1098b28aa92c179a3 Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Wed, 10 Jan 2018 13:32:19 -0800 Subject: [PATCH 08/82] Support for encryption of MySQL connections (#5122) (#5247) * Encrypting MySQL connections * Update docs * Make verifyServerCertificate a configurable parameter * Change password parameter and doc update * Make server certificate verification disabled by default * Update tostring * Update docs * Add check for trust store passwords * Add warning for null password --- .../development/extensions-core/mysql.md | 20 +++ .../storage/mysql/MySQLConnector.java | 76 ++++++++++- .../storage/mysql/MySQLConnectorConfig.java | 123 ++++++++++++++++++ .../mysql/MySQLMetadataStorageModule.java | 3 + 4 files changed, 218 insertions(+), 4 deletions(-) create mode 100644 extensions-core/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnectorConfig.java diff --git a/docs/content/development/extensions-core/mysql.md b/docs/content/development/extensions-core/mysql.md index eb03af6af867..5314c07eef59 100644 --- a/docs/content/development/extensions-core/mysql.md +++ b/docs/content/development/extensions-core/mysql.md @@ -53,3 +53,23 @@ Make sure to [include](../../operations/including-extensions.html) `mysql-metada packaged in a separate tarball that can be downloaded from [here](http://druid.io/downloads.html). You can also get it using [pull-deps](../../operations/pull-deps.html), or you can build it from source code; see [Build from Source](../build.html). + + +## Encrypting MySQL connections + This extension provides support for encrypting MySQL connections. To get more information about encrypting MySQL connections using TLS/SSL in general, please refer to this [guide](https://dev.mysql.com/doc/refman/5.7/en/using-encrypted-connections.html). + +## Configuration + +|Property|Description|Default|Required| +|--------|-----------|-------|--------| +|`druid.metadata.mysql.ssl.useSSL`|Enable SSL|`false`|no| +|`druid.metadata.mysql.ssl.clientCertificateKeyStoreUrl`|The file path URL to the client certificate key store.|none|no| +|`druid.metadata.mysql.ssl.clientCertificateKeyStoreType`|The type of the key store where the client certificate is stored.|none|no| +|`druid.metadata.mysql.ssl.clientCertificateKeyStorePassword`|The [Password Provider](../operations/password-provider.html) or String password for the client key store.|none|no| +|`druid.metadata.mysql.ssl.verifyServerCertificate`|Enables server certificate verification.|false|no| +|`druid.metadata.mysql.ssl.trustCertificateKeyStoreUrl`|The file path to the trusted root certificate key store.|Default trust store provided by MySQL|yes if `verifyServerCertificate` is set to true and a custom trust store is used| +|`druid.metadata.mysql.ssl.trustCertificateKeyStoreType`|The type of the key store where trusted root certificates are stored.|JKS|yes if `verifyServerCertificate` is set to true and keystore type is not JKS| +|`druid.metadata.mysql.ssl.trustCertificateKeyStorePassword`|The [Password Provider](../operations/password-provider.html) or String password for the trust store.|none|yes if `verifyServerCertificate` is set to true and password is not null| +|`druid.metadata.mysql.ssl.enabledSSLCipherSuites`|Overrides the existing cipher suites with these cipher suites.|none|no| +|`druid.metadata.mysql.ssl.enabledTLSProtocols`|Overrides the TLS protocols with these protocols.|none|no| + diff --git a/extensions-core/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnector.java b/extensions-core/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnector.java index 3079ad427eac..910f397f447b 100644 --- a/extensions-core/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnector.java +++ b/extensions-core/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnector.java @@ -19,6 +19,7 @@ package io.druid.metadata.storage.mysql; +import com.google.common.base.Joiner; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.inject.Inject; @@ -36,6 +37,7 @@ import org.skife.jdbi.v2.tweak.HandleCallback; import org.skife.jdbi.v2.util.BooleanMapper; +import java.io.File; import java.sql.SQLException; public class MySQLConnector extends SQLMetadataConnector @@ -48,7 +50,11 @@ public class MySQLConnector extends SQLMetadataConnector private final DBI dbi; @Inject - public MySQLConnector(Supplier config, Supplier dbTables) + public MySQLConnector( + Supplier config, + Supplier dbTables, + MySQLConnectorConfig connectorConfig + ) { super(config, dbTables); @@ -57,6 +63,68 @@ public MySQLConnector(Supplier config, Supplier< // so we need to help JDBC find the driver datasource.setDriverClassLoader(getClass().getClassLoader()); datasource.setDriverClassName("com.mysql.jdbc.Driver"); + datasource.addConnectionProperty("useSSL", String.valueOf(connectorConfig.isUseSSL())); + if (connectorConfig.isUseSSL()) { + log.info("SSL is enabled on this MySQL connection. "); + + datasource.addConnectionProperty( + "verifyServerCertificate", + String.valueOf(connectorConfig.isVerifyServerCertificate()) + ); + if (connectorConfig.isVerifyServerCertificate()) { + log.info("Server certificate verification is enabled. "); + + if (connectorConfig.getTrustCertificateKeyStoreUrl() != null) { + datasource.addConnectionProperty( + "trustCertificateKeyStoreUrl", + new File(connectorConfig.getTrustCertificateKeyStoreUrl()).toURI().toString() + ); + } + if (connectorConfig.getTrustCertificateKeyStoreType() != null) { + datasource.addConnectionProperty( + "trustCertificateKeyStoreType", + connectorConfig.getTrustCertificateKeyStoreType() + ); + } + if (connectorConfig.getTrustCertificateKeyStorePassword() == null) { + log.warn( + "Trust store password is empty. Ensure that the trust store has been configured with an empty password."); + } else { + datasource.addConnectionProperty( + "trustCertificateKeyStorePassword", + connectorConfig.getTrustCertificateKeyStorePassword() + ); + } + } + if (connectorConfig.getClientCertificateKeyStoreUrl() != null) { + datasource.addConnectionProperty( + "clientCertificateKeyStoreUrl", + new File(connectorConfig.getClientCertificateKeyStoreUrl()).toURI().toString() + ); + } + if (connectorConfig.getClientCertificateKeyStoreType() != null) { + datasource.addConnectionProperty( + "clientCertificateKeyStoreType", + connectorConfig.getClientCertificateKeyStoreType() + ); + } + if (connectorConfig.getClientCertificateKeyStorePassword() != null) { + datasource.addConnectionProperty( + "clientCertificateKeyStorePassword", + connectorConfig.getClientCertificateKeyStorePassword() + ); + } + Joiner joiner = Joiner.on(",").skipNulls(); + if (connectorConfig.getEnabledSSLCipherSuites() != null) { + datasource.addConnectionProperty( + "enabledSSLCipherSuites", + joiner.join(connectorConfig.getEnabledSSLCipherSuites()) + ); + } + if (connectorConfig.getEnabledTLSProtocols() != null) { + datasource.addConnectionProperty("enabledTLSProtocols", joiner.join(connectorConfig.getEnabledTLSProtocols())); + } + } // use double-quotes for quoting columns, so we can write SQL that works with most databases datasource.setConnectionInitSqls(ImmutableList.of("SET sql_mode='ANSI_QUOTES'")); @@ -97,9 +165,9 @@ public boolean tableExists(Handle handle, String tableName) { // ensure database defaults to utf8, otherwise bail boolean isUtf8 = handle - .createQuery("SELECT @@character_set_database = 'utf8'") - .map(BooleanMapper.FIRST) - .first(); + .createQuery("SELECT @@character_set_database = 'utf8'") + .map(BooleanMapper.FIRST) + .first(); if (!isUtf8) { throw new ISE( diff --git a/extensions-core/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnectorConfig.java b/extensions-core/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnectorConfig.java new file mode 100644 index 000000000000..77fc9dcd60b6 --- /dev/null +++ b/extensions-core/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLConnectorConfig.java @@ -0,0 +1,123 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.metadata.storage.mysql; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.metadata.PasswordProvider; + +import java.util.List; + +public class MySQLConnectorConfig +{ + @JsonProperty + private boolean useSSL = false; + + @JsonProperty + private String trustCertificateKeyStoreUrl; + + @JsonProperty + private String trustCertificateKeyStoreType; + + @JsonProperty("trustCertificateKeyStorePassword") + private PasswordProvider trustCertificateKeyStorePasswordProvider; + + @JsonProperty + private String clientCertificateKeyStoreUrl; + + @JsonProperty + private String clientCertificateKeyStoreType; + + @JsonProperty("clientCertificateKeyStorePassword") + private PasswordProvider clientCertificateKeyStorePasswordProvider; + + @JsonProperty + private List enabledSSLCipherSuites; + + @JsonProperty + private List enabledTLSProtocols; + + @JsonProperty + private boolean verifyServerCertificate = false; + + public boolean isUseSSL() + { + return useSSL; + } + + public String getTrustCertificateKeyStoreUrl() + { + return trustCertificateKeyStoreUrl; + } + + public String getTrustCertificateKeyStoreType() + { + return trustCertificateKeyStoreType; + } + + public String getTrustCertificateKeyStorePassword() + { + return trustCertificateKeyStorePasswordProvider == null ? null : trustCertificateKeyStorePasswordProvider.getPassword(); + } + + public String getClientCertificateKeyStoreUrl() + { + return clientCertificateKeyStoreUrl; + } + + public String getClientCertificateKeyStoreType() + { + return clientCertificateKeyStoreType; + } + + public String getClientCertificateKeyStorePassword() + { + return clientCertificateKeyStorePasswordProvider == null ? null : clientCertificateKeyStorePasswordProvider.getPassword(); + } + + public List getEnabledSSLCipherSuites() + { + return enabledSSLCipherSuites; + } + + public List getEnabledTLSProtocols() + { + return enabledTLSProtocols; + } + + public boolean isVerifyServerCertificate() + { + return verifyServerCertificate; + } + + @Override + public String toString() + { + return "MySQLConnectorConfig{" + + "useSSL='" + useSSL + '\'' + + ", clientCertificateKeyStoreUrl='" + clientCertificateKeyStoreUrl + '\'' + + ", clientCertificateKeyStoreType='" + clientCertificateKeyStoreType + '\'' + + ", verifyServerCertificate='" + verifyServerCertificate + '\'' + + ", trustCertificateKeyStoreUrl='" + trustCertificateKeyStoreUrl + '\'' + + ", trustCertificateKeyStoreType='" + trustCertificateKeyStoreType + '\'' + + ", enabledSSLCipherSuites=" + enabledSSLCipherSuites + + ", enabledTLSProtocols=" + enabledTLSProtocols + + '}'; + } +} diff --git a/extensions-core/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLMetadataStorageModule.java b/extensions-core/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLMetadataStorageModule.java index 211d5ca0b0e4..4ba1cb8177bb 100644 --- a/extensions-core/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLMetadataStorageModule.java +++ b/extensions-core/mysql-metadata-storage/src/main/java/io/druid/metadata/storage/mysql/MySQLMetadataStorageModule.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Binder; import com.google.inject.Key; +import io.druid.guice.JsonConfigProvider; import io.druid.guice.LazySingleton; import io.druid.guice.PolyBind; import io.druid.guice.SQLMetadataStorageDruidModule; @@ -56,6 +57,8 @@ public void configure(Binder binder) { super.configure(binder); + JsonConfigProvider.bind(binder, "druid.metadata.mysql.ssl", MySQLConnectorConfig.class); + PolyBind .optionBinder(binder, Key.get(MetadataStorageProvider.class)) .addBinding(TYPE) From 61bef7dc82b61b9d27632470ac2633d1ac3af4a3 Mon Sep 17 00:00:00 2001 From: Parag Jain Date: Wed, 10 Jan 2018 16:13:54 -0600 Subject: [PATCH 09/82] Fix state check bug in Kafka Index Task (#5204) (#5248) [Backport] Fix state check bug in Kafka Index Task #5204 --- .../druid/indexing/kafka/KafkaIndexTask.java | 5 +- .../kafka/supervisor/KafkaSupervisor.java | 2 +- .../indexing/kafka/KafkaIndexTaskTest.java | 116 ++++++++++++++++++ .../overlord/supervisor/Supervisor.java | 10 +- 4 files changed, 125 insertions(+), 8 deletions(-) diff --git a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java index 5082a8060cb4..4a2ecbbae10c 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java +++ b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java @@ -512,13 +512,14 @@ public TaskStatus run(final TaskToolbox toolbox) throws Exception final Object restoredMetadata = driver.startJob(); if (restoredMetadata == null) { // no persist has happened so far + // so either this is a brand new task or replacement of a failed task Preconditions.checkState(sequences.get(0).startOffsets.entrySet().stream().allMatch( partitionOffsetEntry -> Longs.compare( partitionOffsetEntry.getValue(), ioConfig.getStartPartitions() .getPartitionOffsetMap() .get(partitionOffsetEntry.getKey()) - ) == 0 + ) >= 0 ), "Sequence offsets are not compatible with start offsets of task"); nextOffsets.putAll(sequences.get(0).startOffsets); } else { @@ -545,7 +546,7 @@ public TaskStatus run(final TaskToolbox toolbox) throws Exception ioConfig.getStartPartitions().getPartitionOffsetMap().keySet() ); } - // sequences size can 0 only when all sequences got published and task stopped before it could finish + // sequences size can be 0 only when all sequences got published and task stopped before it could finish // which is super rare if (sequences.size() == 0 || sequences.get(sequences.size() - 1).isCheckpointed()) { this.endOffsets.putAll(sequences.size() == 0 diff --git a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisor.java b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisor.java index d0b8ea55c241..2a36f251d0a0 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisor.java +++ b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisor.java @@ -670,7 +670,7 @@ public void handle() throws ExecutionException, InterruptedException, TimeoutExc // as when the task starts they are sent existing checkpoints Preconditions.checkState( checkpoints.size() <= 1, - "Got checkpoint request with null as previous check point, however found more than one checkpoints in metadata store" + "Got checkpoint request with null as previous check point, however found more than one checkpoints" ); if (checkpoints.size() == 1) { log.info("Already checkpointed with dataSourceMetadata [%s]", checkpoints.get(0)); diff --git a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java index c2e1700df8a3..40986a95aab3 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java @@ -155,10 +155,12 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.Callable; import java.util.concurrent.Executor; import java.util.concurrent.Executors; @@ -1557,6 +1559,72 @@ public void testRunWithOffsetOutOfRangeExceptionAndNextOffsetGreaterThanLeastAva } } + @Test(timeout = 60_000L) + public void testRunContextSequenceAheadOfStartingOffsets() throws Exception + { + // This tests the case when a replacement task is created in place of a failed test + // which has done some incremental handoffs, thus the context will contain starting + // sequence offsets from which the task should start reading and ignore the start offsets + if (!isIncrementalHandoffSupported) { + return; + } + // Insert data + try (final KafkaProducer kafkaProducer = kafkaServer.newProducer()) { + for (ProducerRecord record : records) { + kafkaProducer.send(record).get(); + } + } + + final TreeMap> sequences = new TreeMap<>(); + // Here the sequence number is 1 meaning that one incremental handoff was done by the failed task + // and this task should start reading from offset 2 for partition 0 + sequences.put(1, ImmutableMap.of(0, 2L)); + final Map context = new HashMap<>(); + context.put("checkpoints", objectMapper.writerWithType(new TypeReference>>() + { + }).writeValueAsString(sequences)); + + final KafkaIndexTask task = createTask( + null, + new KafkaIOConfig( + "sequence0", + // task should ignore these and use sequence info sent in the context + new KafkaPartitions(topic, ImmutableMap.of(0, 0L)), + new KafkaPartitions(topic, ImmutableMap.of(0, 5L)), + kafkaServer.consumerProperties(), + true, + false, + null, + null, + false + ), + context + ); + + final ListenableFuture future = runTask(task); + + // Wait for task to exit + Assert.assertEquals(TaskState.SUCCESS, future.get().getStatusCode()); + + // Check metrics + Assert.assertEquals(3, task.getFireDepartmentMetrics().processed()); + Assert.assertEquals(0, task.getFireDepartmentMetrics().unparseable()); + Assert.assertEquals(0, task.getFireDepartmentMetrics().thrownAway()); + + // Check published metadata + SegmentDescriptor desc1 = SD(task, "2010/P1D", 0); + SegmentDescriptor desc2 = SD(task, "2011/P1D", 0); + Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors()); + Assert.assertEquals( + new KafkaDataSourceMetadata(new KafkaPartitions(topic, ImmutableMap.of(0, 5L))), + metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()) + ); + + // Check segments in deep storage + Assert.assertEquals(ImmutableList.of("c"), readSegmentColumn("dim1", desc1)); + Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentColumn("dim1", desc2)); + } + private ListenableFuture runTask(final Task task) { try { @@ -1615,6 +1683,15 @@ private KafkaIndexTask createTask( return createTask(taskId, DATA_SCHEMA, ioConfig); } + private KafkaIndexTask createTask( + final String taskId, + final KafkaIOConfig ioConfig, + final Map context + ) + { + return createTask(taskId, DATA_SCHEMA, ioConfig, context); + } + private KafkaIndexTask createTask( final String taskId, final DataSchema dataSchema, @@ -1651,6 +1728,45 @@ private KafkaIndexTask createTask( return task; } + + private KafkaIndexTask createTask( + final String taskId, + final DataSchema dataSchema, + final KafkaIOConfig ioConfig, + final Map context + ) + { + final KafkaTuningConfig tuningConfig = new KafkaTuningConfig( + 1000, + maxRowsPerSegment, + new Period("P1Y"), + null, + null, + null, + true, + reportParseExceptions, + handoffConditionTimeout, + resetOffsetAutomatically, + null + ); + if (isIncrementalHandoffSupported) { + context.put(KafkaSupervisor.IS_INCREMENTAL_HANDOFF_SUPPORTED, true); + } + + final KafkaIndexTask task = new KafkaIndexTask( + taskId, + null, + cloneDataSchema(dataSchema), + tuningConfig, + ioConfig, + context, + null, + null + ); + task.setPollRetryMs(POLL_RETRY_MS); + return task; + } + private static DataSchema cloneDataSchema(final DataSchema dataSchema) { return new DataSchema( diff --git a/server/src/main/java/io/druid/indexing/overlord/supervisor/Supervisor.java b/server/src/main/java/io/druid/indexing/overlord/supervisor/Supervisor.java index c6c411480683..681421b48002 100644 --- a/server/src/main/java/io/druid/indexing/overlord/supervisor/Supervisor.java +++ b/server/src/main/java/io/druid/indexing/overlord/supervisor/Supervisor.java @@ -40,14 +40,14 @@ public interface Supervisor void reset(DataSourceMetadata dataSourceMetadata); /** - * The definition of checkpoint is not very strict as currently it does not affect data or control path + * The definition of checkpoint is not very strict as currently it does not affect data or control path. * On this call Supervisor can potentially checkpoint data processed so far to some durable storage * for example - Kafka Supervisor uses this to merge and handoff segments containing at least the data - * represented by dataSourceMetadata + * represented by {@param currentCheckpoint} DataSourceMetadata * - * @param sequenceName unique Identifier to figure out for which sequence to do check pointing - * @param previousCheckPoint DataSourceMetadata check pointed in previous call - * @param currentCheckPoint current DataSourceMetadata to be check pointed + * @param sequenceName unique Identifier to figure out for which sequence to do checkpointing + * @param previousCheckPoint DataSourceMetadata checkpointed in previous call + * @param currentCheckPoint current DataSourceMetadata to be checkpointed */ void checkpoint( @Nullable String sequenceName, From 87a1170d3b83e815e36c6d044c1774db9aae38cf Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Fri, 12 Jan 2018 05:39:17 -0800 Subject: [PATCH 10/82] =?UTF-8?q?[Backport]=20fix=20timewarp=20query=20res?= =?UTF-8?q?ults=20when=20using=20timezones=20and=20crossing=20DST=20tran?= =?UTF-8?q?=E2=80=A6=20(#5252)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix timewarp query results when using timezones and crossing DST transitions (#5157) * timewarp and timezones changes: * `TimewarpOperator` will now compensate for daylight savings time shifts between date translation ranges for queries using a `PeriodGranularity` with a timezone defined * introduces a new abstract query type `TimeBucketedQuery` for all queries which have a `Granularity` (100% not attached to this name). `GroupByQuery`, `SearchQuery`, `SelectQuery`, `TimeseriesQuery`, and `TopNQuery` all extend `TimeBucke tedQuery`, cutting down on some duplicate code and providing a mechanism for `TimewarpOperator` (and anything else) that needs to be aware of granularity * move precondition check to TimeBucketedQuery, add Granularities.nullToAll, add getTimezone to TimeBucketQuery * formatting * more formatting * unused import * changes: * add 'getGranularity' and 'getTimezone' to 'Query' interface * merge 'TimeBucketedQuery' into 'BaseQuery' * fixup tests from resulting serialization changes * dedupe * fix after merge * suppress warning * Fix compile issue --- .../common/granularity/Granularities.java | 4 + .../main/java/io/druid/query/BaseQuery.java | 71 +++++---- .../src/main/java/io/druid/query/Query.java | 8 + .../java/io/druid/query/TimewarpOperator.java | 8 +- .../io/druid/query/groupby/GroupByQuery.java | 22 +-- .../io/druid/query/search/SearchQuery.java | 30 ++-- .../io/druid/query/select/SelectQuery.java | 34 ++--- .../query/timeseries/TimeseriesQuery.java | 15 +- .../java/io/druid/query/topn/TopNQuery.java | 14 +- .../io/druid/query/TimewarpOperatorTest.java | 138 +++++++++++++++++- .../druid/query/scan/ScanQuerySpecTest.java | 3 +- .../druid/sql/calcite/CalciteQueryTest.java | 8 +- 12 files changed, 233 insertions(+), 122 deletions(-) diff --git a/java-util/src/main/java/io/druid/java/util/common/granularity/Granularities.java b/java-util/src/main/java/io/druid/java/util/common/granularity/Granularities.java index 599110550f9b..de8b6f688d2f 100644 --- a/java-util/src/main/java/io/druid/java/util/common/granularity/Granularities.java +++ b/java-util/src/main/java/io/druid/java/util/common/granularity/Granularities.java @@ -41,4 +41,8 @@ public class Granularities public static final Granularity ALL = GranularityType.ALL.getDefaultGranularity(); public static final Granularity NONE = GranularityType.NONE.getDefaultGranularity(); + public static Granularity nullToAll(Granularity granularity) + { + return granularity == null ? Granularities.ALL : granularity; + } } diff --git a/processing/src/main/java/io/druid/query/BaseQuery.java b/processing/src/main/java/io/druid/query/BaseQuery.java index 7b511097a832..86587d47ae84 100644 --- a/processing/src/main/java/io/druid/query/BaseQuery.java +++ b/processing/src/main/java/io/druid/query/BaseQuery.java @@ -25,12 +25,17 @@ import com.google.common.collect.Maps; import com.google.common.collect.Ordering; import io.druid.guice.annotations.ExtensionPoint; +import io.druid.java.util.common.granularity.Granularities; +import io.druid.java.util.common.granularity.Granularity; +import io.druid.java.util.common.granularity.PeriodGranularity; import io.druid.query.spec.QuerySegmentSpec; +import org.joda.time.DateTimeZone; import org.joda.time.Duration; import org.joda.time.Interval; import java.util.List; import java.util.Map; +import java.util.Objects; /** */ @@ -50,6 +55,7 @@ public static void checkInterrupted() private final Map context; private final QuerySegmentSpec querySegmentSpec; private volatile Duration duration; + private final Granularity granularity; public BaseQuery( DataSource dataSource, @@ -57,14 +63,27 @@ public BaseQuery( boolean descending, Map context ) + { + this(dataSource, querySegmentSpec, descending, context, Granularities.ALL); + } + + public BaseQuery( + DataSource dataSource, + QuerySegmentSpec querySegmentSpec, + boolean descending, + Map context, + Granularity granularity + ) { Preconditions.checkNotNull(dataSource, "dataSource can't be null"); Preconditions.checkNotNull(querySegmentSpec, "querySegmentSpec can't be null"); + Preconditions.checkNotNull(granularity, "Must specify a granularity"); this.dataSource = dataSource; this.context = context; this.querySegmentSpec = querySegmentSpec; this.descending = descending; + this.granularity = granularity; } @JsonProperty @@ -115,6 +134,21 @@ public Duration getDuration() return duration; } + @Override + @JsonProperty + public Granularity getGranularity() + { + return granularity; + } + + @Override + public DateTimeZone getTimezone() + { + return granularity instanceof PeriodGranularity + ? ((PeriodGranularity) granularity).getTimeZone() + : DateTimeZone.UTC; + } + @Override @JsonProperty public Map getContext() @@ -193,38 +227,19 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - - BaseQuery baseQuery = (BaseQuery) o; - - if (descending != baseQuery.descending) { - return false; - } - if (context != null ? !context.equals(baseQuery.context) : baseQuery.context != null) { - return false; - } - if (dataSource != null ? !dataSource.equals(baseQuery.dataSource) : baseQuery.dataSource != null) { - return false; - } - if (duration != null ? !duration.equals(baseQuery.duration) : baseQuery.duration != null) { - return false; - } - if (querySegmentSpec != null - ? !querySegmentSpec.equals(baseQuery.querySegmentSpec) - : baseQuery.querySegmentSpec != null) { - return false; - } - - return true; + BaseQuery baseQuery = (BaseQuery) o; + return descending == baseQuery.descending && + Objects.equals(dataSource, baseQuery.dataSource) && + Objects.equals(context, baseQuery.context) && + Objects.equals(querySegmentSpec, baseQuery.querySegmentSpec) && + Objects.equals(duration, baseQuery.duration) && + Objects.equals(granularity, baseQuery.granularity); } @Override public int hashCode() { - int result = dataSource != null ? dataSource.hashCode() : 0; - result = 31 * result + (descending ? 1 : 0); - result = 31 * result + (context != null ? context.hashCode() : 0); - result = 31 * result + (querySegmentSpec != null ? querySegmentSpec.hashCode() : 0); - result = 31 * result + (duration != null ? duration.hashCode() : 0); - return result; + + return Objects.hash(dataSource, descending, context, querySegmentSpec, duration, granularity); } } diff --git a/processing/src/main/java/io/druid/query/Query.java b/processing/src/main/java/io/druid/query/Query.java index efb9fd500e09..06ff069b205d 100644 --- a/processing/src/main/java/io/druid/query/Query.java +++ b/processing/src/main/java/io/druid/query/Query.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.collect.Ordering; import io.druid.guice.annotations.ExtensionPoint; +import io.druid.java.util.common.granularity.Granularity; import io.druid.query.datasourcemetadata.DataSourceMetadataQuery; import io.druid.query.filter.DimFilter; import io.druid.query.groupby.GroupByQuery; @@ -34,6 +35,7 @@ import io.druid.query.timeboundary.TimeBoundaryQuery; import io.druid.query.timeseries.TimeseriesQuery; import io.druid.query.topn.TopNQuery; +import org.joda.time.DateTimeZone; import org.joda.time.Duration; import org.joda.time.Interval; @@ -80,6 +82,12 @@ public interface Query Duration getDuration(); + // currently unused, but helping enforce the idea that all queries have a Granularity + @SuppressWarnings("unused") + Granularity getGranularity(); + + DateTimeZone getTimezone(); + Map getContext(); ContextType getContextValue(String key); diff --git a/processing/src/main/java/io/druid/query/TimewarpOperator.java b/processing/src/main/java/io/druid/query/TimewarpOperator.java index 64ed0cdd9af2..cee9047e5bf2 100644 --- a/processing/src/main/java/io/druid/query/TimewarpOperator.java +++ b/processing/src/main/java/io/druid/query/TimewarpOperator.java @@ -30,6 +30,7 @@ import io.druid.query.timeboundary.TimeBoundaryQuery; import io.druid.query.timeboundary.TimeBoundaryResultValue; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.joda.time.Interval; import org.joda.time.Period; @@ -80,7 +81,8 @@ public QueryRunner postProcess(final QueryRunner baseRunner, final long no @Override public Sequence run(final QueryPlus queryPlus, final Map responseContext) { - final long offset = computeOffset(now); + final DateTimeZone tz = queryPlus.getQuery().getTimezone(); + final long offset = computeOffset(now, tz); final Interval interval = queryPlus.getQuery().getIntervals().get(0); final Interval modifiedInterval = new Interval( @@ -142,7 +144,7 @@ public T apply(T input) * * @return the offset between the mapped time and time t */ - protected long computeOffset(final long t) + protected long computeOffset(final long t, final DateTimeZone tz) { // start is the beginning of the last period ending within dataInterval long start = dataInterval.getEndMillis() - periodMillis; @@ -159,6 +161,6 @@ protected long computeOffset(final long t) tOffset += periodMillis; } tOffset += start; - return tOffset - t; + return tOffset - t - (tz.getOffset(tOffset) - tz.getOffset(t)); } } diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java index 72fd236402be..9043ad6a1e12 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java @@ -96,7 +96,6 @@ public static Builder builder() private final LimitSpec limitSpec; private final HavingSpec havingSpec; private final DimFilter dimFilter; - private final Granularity granularity; private final List dimensions; private final List aggregatorSpecs; private final List postAggregatorSpecs; @@ -171,15 +170,15 @@ private GroupByQuery( final Map context ) { - super(dataSource, querySegmentSpec, false, context); + super(dataSource, querySegmentSpec, false, context, granularity); this.virtualColumns = VirtualColumns.nullToEmpty(virtualColumns); this.dimFilter = dimFilter; - this.granularity = granularity; this.dimensions = dimensions == null ? ImmutableList.of() : dimensions; for (DimensionSpec spec : this.dimensions) { Preconditions.checkArgument(spec != null, "dimensions has null DimensionSpec"); } + this.aggregatorSpecs = aggregatorSpecs == null ? ImmutableList.of() : aggregatorSpecs; this.postAggregatorSpecs = Queries.prepareAggregations( this.dimensions.stream().map(DimensionSpec::getOutputName).collect(Collectors.toList()), @@ -189,7 +188,6 @@ private GroupByQuery( this.havingSpec = havingSpec; this.limitSpec = LimitSpec.nullToNoopLimitSpec(limitSpec); - Preconditions.checkNotNull(this.granularity, "Must specify a granularity"); // Verify no duplicate names between dimensions, aggregators, and postAggregators. // They will all end up in the same namespace in the returned Rows and we can't have them clobbering each other. @@ -214,12 +212,6 @@ public DimFilter getDimFilter() return dimFilter; } - @JsonProperty - public Granularity getGranularity() - { - return granularity; - } - @JsonProperty public List getDimensions() { @@ -518,12 +510,12 @@ public Ordering getRowOrdering(final boolean granular) private Comparator getTimeComparator(boolean granular) { - if (Granularities.ALL.equals(granularity)) { + if (Granularities.ALL.equals(getGranularity())) { return null; } else if (granular) { return (lhs, rhs) -> Longs.compare( - granularity.bucketStart(lhs.getTimestamp()).getMillis(), - granularity.bucketStart(rhs.getTimestamp()).getMillis() + getGranularity().bucketStart(lhs.getTimestamp()).getMillis(), + getGranularity().bucketStart(rhs.getTimestamp()).getMillis() ); } else { return NON_GRANULAR_TIME_COMP; @@ -990,7 +982,7 @@ public String toString() ", virtualColumns=" + virtualColumns + ", limitSpec=" + limitSpec + ", dimFilter=" + dimFilter + - ", granularity=" + granularity + + ", granularity=" + getGranularity() + ", dimensions=" + dimensions + ", aggregatorSpecs=" + aggregatorSpecs + ", postAggregatorSpecs=" + postAggregatorSpecs + @@ -1015,7 +1007,6 @@ public boolean equals(final Object o) Objects.equals(limitSpec, that.limitSpec) && Objects.equals(havingSpec, that.havingSpec) && Objects.equals(dimFilter, that.dimFilter) && - Objects.equals(granularity, that.granularity) && Objects.equals(dimensions, that.dimensions) && Objects.equals(aggregatorSpecs, that.aggregatorSpecs) && Objects.equals(postAggregatorSpecs, that.postAggregatorSpecs); @@ -1030,7 +1021,6 @@ public int hashCode() limitSpec, havingSpec, dimFilter, - granularity, dimensions, aggregatorSpecs, postAggregatorSpecs diff --git a/processing/src/main/java/io/druid/query/search/SearchQuery.java b/processing/src/main/java/io/druid/query/search/SearchQuery.java index f9cd866e1f7f..df298d55c5ba 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQuery.java +++ b/processing/src/main/java/io/druid/query/search/SearchQuery.java @@ -45,7 +45,6 @@ public class SearchQuery extends BaseQuery> private final DimFilter dimFilter; private final SearchSortSpec sortSpec; - private final Granularity granularity; private final List dimensions; private final SearchQuerySpec querySpec; private final int limit; @@ -63,12 +62,11 @@ public SearchQuery( @JsonProperty("context") Map context ) { - super(dataSource, querySegmentSpec, false, context); + super(dataSource, querySegmentSpec, false, context, Granularities.nullToAll(granularity)); Preconditions.checkNotNull(querySegmentSpec, "Must specify an interval"); this.dimFilter = dimFilter; this.sortSpec = sortSpec == null ? DEFAULT_SORT_SPEC : sortSpec; - this.granularity = granularity == null ? Granularities.ALL : granularity; this.limit = (limit == 0) ? 1000 : limit; this.dimensions = dimensions; this.querySpec = querySpec == null ? new AllSearchQuerySpec() : querySpec; @@ -122,12 +120,6 @@ public DimFilter getDimensionsFilter() return dimFilter; } - @JsonProperty - public Granularity getGranularity() - { - return granularity; - } - @JsonProperty public int getLimit() { @@ -161,14 +153,14 @@ public SearchQuery withLimit(int newLimit) public String toString() { return "SearchQuery{" + - "dataSource='" + getDataSource() + '\'' + - ", dimFilter=" + dimFilter + - ", granularity='" + granularity + '\'' + - ", dimensions=" + dimensions + - ", querySpec=" + querySpec + - ", querySegmentSpec=" + getQuerySegmentSpec() + - ", limit=" + limit + - '}'; + "dataSource='" + getDataSource() + '\'' + + ", dimFilter=" + dimFilter + + ", granularity='" + getGranularity() + '\'' + + ", dimensions=" + dimensions + + ", querySpec=" + querySpec + + ", querySegmentSpec=" + getQuerySegmentSpec() + + ", limit=" + limit + + '}'; } @Override @@ -195,9 +187,6 @@ public boolean equals(Object o) if (dimensions != null ? !dimensions.equals(that.dimensions) : that.dimensions != null) { return false; } - if (granularity != null ? !granularity.equals(that.granularity) : that.granularity != null) { - return false; - } if (querySpec != null ? !querySpec.equals(that.querySpec) : that.querySpec != null) { return false; } @@ -214,7 +203,6 @@ public int hashCode() int result = super.hashCode(); result = 31 * result + (dimFilter != null ? dimFilter.hashCode() : 0); result = 31 * result + (sortSpec != null ? sortSpec.hashCode() : 0); - result = 31 * result + (granularity != null ? granularity.hashCode() : 0); result = 31 * result + (dimensions != null ? dimensions.hashCode() : 0); result = 31 * result + (querySpec != null ? querySpec.hashCode() : 0); result = 31 * result + limit; diff --git a/processing/src/main/java/io/druid/query/select/SelectQuery.java b/processing/src/main/java/io/druid/query/select/SelectQuery.java index 6676777ba3d1..2bc972b9fac7 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQuery.java +++ b/processing/src/main/java/io/druid/query/select/SelectQuery.java @@ -45,7 +45,6 @@ public class SelectQuery extends BaseQuery> { private final DimFilter dimFilter; - private final Granularity granularity; private final List dimensions; private final List metrics; private final VirtualColumns virtualColumns; @@ -65,9 +64,8 @@ public SelectQuery( @JsonProperty("context") Map context ) { - super(dataSource, querySegmentSpec, descending, context); + super(dataSource, querySegmentSpec, descending, context, Granularities.nullToAll(granularity)); this.dimFilter = dimFilter; - this.granularity = granularity == null ? Granularities.ALL : granularity; this.dimensions = dimensions; this.virtualColumns = VirtualColumns.nullToEmpty(virtualColumns); this.metrics = metrics; @@ -111,12 +109,6 @@ public DimFilter getDimensionsFilter() return dimFilter; } - @JsonProperty - public Granularity getGranularity() - { - return granularity; - } - @JsonProperty public List getDimensions() { @@ -179,16 +171,16 @@ public SelectQuery withDimFilter(DimFilter dimFilter) public String toString() { return "SelectQuery{" + - "dataSource='" + getDataSource() + '\'' + - ", querySegmentSpec=" + getQuerySegmentSpec() + - ", descending=" + isDescending() + - ", dimFilter=" + dimFilter + - ", granularity=" + granularity + - ", dimensions=" + dimensions + - ", metrics=" + metrics + - ", virtualColumns=" + virtualColumns + - ", pagingSpec=" + pagingSpec + - '}'; + "dataSource='" + getDataSource() + '\'' + + ", querySegmentSpec=" + getQuerySegmentSpec() + + ", descending=" + isDescending() + + ", dimFilter=" + dimFilter + + ", granularity=" + getGranularity() + + ", dimensions=" + dimensions + + ", metrics=" + metrics + + ", virtualColumns=" + virtualColumns + + ", pagingSpec=" + pagingSpec + + '}'; } @Override @@ -209,9 +201,6 @@ public boolean equals(Object o) if (!Objects.equals(dimFilter, that.dimFilter)) { return false; } - if (!Objects.equals(granularity, that.granularity)) { - return false; - } if (!Objects.equals(dimensions, that.dimensions)) { return false; } @@ -233,7 +222,6 @@ public int hashCode() { int result = super.hashCode(); result = 31 * result + (dimFilter != null ? dimFilter.hashCode() : 0); - result = 31 * result + (granularity != null ? granularity.hashCode() : 0); result = 31 * result + (dimensions != null ? dimensions.hashCode() : 0); result = 31 * result + (metrics != null ? metrics.hashCode() : 0); result = 31 * result + (virtualColumns != null ? virtualColumns.hashCode() : 0); diff --git a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java index 2cfa00b6235e..d1ac95cdadce 100644 --- a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java @@ -47,7 +47,6 @@ public class TimeseriesQuery extends BaseQuery> { private final VirtualColumns virtualColumns; private final DimFilter dimFilter; - private final Granularity granularity; private final List aggregatorSpecs; private final List postAggregatorSpecs; @@ -64,11 +63,10 @@ public TimeseriesQuery( @JsonProperty("context") Map context ) { - super(dataSource, querySegmentSpec, descending, context); + super(dataSource, querySegmentSpec, descending, context, granularity); this.virtualColumns = VirtualColumns.nullToEmpty(virtualColumns); this.dimFilter = dimFilter; - this.granularity = granularity; this.aggregatorSpecs = aggregatorSpecs == null ? ImmutableList.of() : aggregatorSpecs; this.postAggregatorSpecs = Queries.prepareAggregations( ImmutableList.of(), @@ -107,12 +105,6 @@ public DimFilter getDimensionsFilter() return dimFilter; } - @JsonProperty - public Granularity getGranularity() - { - return granularity; - } - @JsonProperty("aggregations") public List getAggregatorSpecs() { @@ -168,7 +160,7 @@ public String toString() ", descending=" + isDescending() + ", virtualColumns=" + virtualColumns + ", dimFilter=" + dimFilter + - ", granularity='" + granularity + '\'' + + ", granularity='" + getGranularity() + '\'' + ", aggregatorSpecs=" + aggregatorSpecs + ", postAggregatorSpecs=" + postAggregatorSpecs + ", context=" + getContext() + @@ -190,7 +182,6 @@ public boolean equals(final Object o) final TimeseriesQuery that = (TimeseriesQuery) o; return Objects.equals(virtualColumns, that.virtualColumns) && Objects.equals(dimFilter, that.dimFilter) && - Objects.equals(granularity, that.granularity) && Objects.equals(aggregatorSpecs, that.aggregatorSpecs) && Objects.equals(postAggregatorSpecs, that.postAggregatorSpecs); } @@ -198,6 +189,6 @@ public boolean equals(final Object o) @Override public int hashCode() { - return Objects.hash(super.hashCode(), virtualColumns, dimFilter, granularity, aggregatorSpecs, postAggregatorSpecs); + return Objects.hash(super.hashCode(), virtualColumns, dimFilter, aggregatorSpecs, postAggregatorSpecs); } } diff --git a/processing/src/main/java/io/druid/query/topn/TopNQuery.java b/processing/src/main/java/io/druid/query/topn/TopNQuery.java index 764990b063ed..844142d01c50 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNQuery.java +++ b/processing/src/main/java/io/druid/query/topn/TopNQuery.java @@ -51,7 +51,6 @@ public class TopNQuery extends BaseQuery> private final TopNMetricSpec topNMetricSpec; private final int threshold; private final DimFilter dimFilter; - private final Granularity granularity; private final List aggregatorSpecs; private final List postAggregatorSpecs; @@ -70,7 +69,7 @@ public TopNQuery( @JsonProperty("context") Map context ) { - super(dataSource, querySegmentSpec, false, context); + super(dataSource, querySegmentSpec, false, context, granularity); this.virtualColumns = VirtualColumns.nullToEmpty(virtualColumns); this.dimensionSpec = dimensionSpec; @@ -78,7 +77,6 @@ public TopNQuery( this.threshold = threshold; this.dimFilter = dimFilter; - this.granularity = granularity; this.aggregatorSpecs = aggregatorSpecs == null ? ImmutableList.of() : aggregatorSpecs; this.postAggregatorSpecs = Queries.prepareAggregations( ImmutableList.of(dimensionSpec.getOutputName()), @@ -143,12 +141,6 @@ public DimFilter getDimensionsFilter() return dimFilter; } - @JsonProperty - public Granularity getGranularity() - { - return granularity; - } - @JsonProperty("aggregations") public List getAggregatorSpecs() { @@ -218,7 +210,7 @@ public String toString() ", querySegmentSpec=" + getQuerySegmentSpec() + ", virtualColumns=" + virtualColumns + ", dimFilter=" + dimFilter + - ", granularity='" + granularity + '\'' + + ", granularity='" + getGranularity() + '\'' + ", aggregatorSpecs=" + aggregatorSpecs + ", postAggregatorSpecs=" + postAggregatorSpecs + '}'; @@ -242,7 +234,6 @@ public boolean equals(final Object o) Objects.equals(dimensionSpec, topNQuery.dimensionSpec) && Objects.equals(topNMetricSpec, topNQuery.topNMetricSpec) && Objects.equals(dimFilter, topNQuery.dimFilter) && - Objects.equals(granularity, topNQuery.granularity) && Objects.equals(aggregatorSpecs, topNQuery.aggregatorSpecs) && Objects.equals(postAggregatorSpecs, topNQuery.postAggregatorSpecs); } @@ -257,7 +248,6 @@ public int hashCode() topNMetricSpec, threshold, dimFilter, - granularity, aggregatorSpecs, postAggregatorSpecs ); diff --git a/processing/src/test/java/io/druid/query/TimewarpOperatorTest.java b/processing/src/test/java/io/druid/query/TimewarpOperatorTest.java index 8b5675b646fe..d8a859d298c7 100644 --- a/processing/src/test/java/io/druid/query/TimewarpOperatorTest.java +++ b/processing/src/test/java/io/druid/query/TimewarpOperatorTest.java @@ -24,6 +24,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import io.druid.java.util.common.DateTimes; +import io.druid.java.util.common.granularity.PeriodGranularity; +import io.druid.java.util.common.guava.Accumulators; import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; import io.druid.query.aggregation.AggregatorFactory; @@ -31,11 +33,13 @@ import io.druid.query.timeboundary.TimeBoundaryResultValue; import io.druid.query.timeseries.TimeseriesResultValue; import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import org.joda.time.Interval; import org.joda.time.Period; import org.junit.Assert; import org.junit.Test; +import java.util.ArrayList; import java.util.Arrays; import java.util.Map; @@ -57,14 +61,24 @@ public void testComputeOffset() throws Exception final DateTime t = DateTimes.of("2014-01-23"); final DateTime tOffset = DateTimes.of("2014-01-09"); - Assert.assertEquals(tOffset, t.plus(testOperator.computeOffset(t.getMillis()))); + Assert.assertEquals(tOffset, t.plus(testOperator.computeOffset(t.getMillis(), DateTimeZone.UTC))); } { final DateTime t = DateTimes.of("2014-08-02"); final DateTime tOffset = DateTimes.of("2014-01-11"); - Assert.assertEquals(tOffset, t.plus(testOperator.computeOffset(t.getMillis()))); + Assert.assertEquals(tOffset, t.plus(testOperator.computeOffset(t.getMillis(), DateTimeZone.UTC))); + } + + { + final DateTime t = DateTimes.of("2014-08-02T-07"); + final DateTime tOffset = DateTimes.of("2014-01-11T-08"); + + Assert.assertEquals( + tOffset, + t.plus(testOperator.computeOffset(t.getMillis(), DateTimeZone.forID("America/Los_Angeles"))) + ); } } @@ -183,6 +197,126 @@ public Sequence> run( } + @Test + public void testPostProcessWithTimezonesAndDstShift() throws Exception + { + QueryRunner> queryRunner = testOperator.postProcess( + new QueryRunner>() + { + @Override + public Sequence> run( + QueryPlus> queryPlus, + Map responseContext + ) + { + return Sequences.simple( + ImmutableList.of( + new Result<>( + DateTimes.of("2014-01-09T-08"), + new TimeseriesResultValue(ImmutableMap.of("metric", 2)) + ), + new Result<>( + DateTimes.of("2014-01-11T-08"), + new TimeseriesResultValue(ImmutableMap.of("metric", 3)) + ), + new Result<>( + queryPlus.getQuery().getIntervals().get(0).getEnd(), + new TimeseriesResultValue(ImmutableMap.of("metric", 5)) + ) + ) + ); + } + }, + DateTimes.of("2014-08-02T-07").getMillis() + ); + + final Query> query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2014-07-31T-07/2014-08-05T-07") + .granularity(new PeriodGranularity(new Period("P1D"), null, DateTimeZone.forID("America/Los_Angeles"))) + .aggregators(Arrays.asList(new CountAggregatorFactory("count"))) + .build(); + + Assert.assertEquals( + Lists.newArrayList( + new Result<>( + DateTimes.of("2014-07-31T-07"), + new TimeseriesResultValue(ImmutableMap.of("metric", 2)) + ), + new Result<>( + DateTimes.of("2014-08-02T-07"), + new TimeseriesResultValue(ImmutableMap.of("metric", 3)) + ), + new Result<>( + DateTimes.of("2014-08-02T-07"), + new TimeseriesResultValue(ImmutableMap.of("metric", 5)) + ) + ), + queryRunner.run(QueryPlus.wrap(query), CONTEXT).accumulate(new ArrayList<>(), Accumulators.list()) + ); + } + + @Test + public void testPostProcessWithTimezonesAndNoDstShift() throws Exception + { + QueryRunner> queryRunner = testOperator.postProcess( + new QueryRunner>() + { + @Override + public Sequence> run( + QueryPlus> queryPlus, + Map responseContext + ) + { + return Sequences.simple( + ImmutableList.of( + new Result<>( + DateTimes.of("2014-01-09T-07"), + new TimeseriesResultValue(ImmutableMap.of("metric", 2)) + ), + new Result<>( + DateTimes.of("2014-01-11T-07"), + new TimeseriesResultValue(ImmutableMap.of("metric", 3)) + ), + new Result<>( + queryPlus.getQuery().getIntervals().get(0).getEnd(), + new TimeseriesResultValue(ImmutableMap.of("metric", 5)) + ) + ) + ); + } + }, + DateTimes.of("2014-08-02T-07").getMillis() + ); + + final Query> query = + Druids.newTimeseriesQueryBuilder() + .dataSource("dummy") + .intervals("2014-07-31T-07/2014-08-05T-07") + .granularity(new PeriodGranularity(new Period("P1D"), null, DateTimeZone.forID("America/Phoenix"))) + .aggregators(Arrays.asList(new CountAggregatorFactory("count"))) + .build(); + + Assert.assertEquals( + Lists.newArrayList( + new Result<>( + DateTimes.of("2014-07-31T-07"), + new TimeseriesResultValue(ImmutableMap.of("metric", 2)) + ), + new Result<>( + DateTimes.of("2014-08-02T-07"), + new TimeseriesResultValue(ImmutableMap.of("metric", 3)) + ), + new Result<>( + DateTimes.of("2014-08-02T-07"), + new TimeseriesResultValue(ImmutableMap.of("metric", 5)) + ) + ), + queryRunner.run(QueryPlus.wrap(query), CONTEXT).accumulate(new ArrayList<>(), Accumulators.list()) + ); + } + @Test public void testEmptyFutureInterval() throws Exception { diff --git a/processing/src/test/java/io/druid/query/scan/ScanQuerySpecTest.java b/processing/src/test/java/io/druid/query/scan/ScanQuerySpecTest.java index 774a3b798a6e..8b903f7b72f5 100644 --- a/processing/src/test/java/io/druid/query/scan/ScanQuerySpecTest.java +++ b/processing/src/test/java/io/druid/query/scan/ScanQuerySpecTest.java @@ -57,7 +57,8 @@ public void testSerializationLegacyString() throws Exception + "\"columns\":[\"market\",\"quality\",\"index\"]," + "\"legacy\":null," + "\"context\":null," - + "\"descending\":false}"; + + "\"descending\":false," + + "\"granularity\":{\"type\":\"all\"}}"; ScanQuery query = new ScanQuery( new TableDataSource(QueryRunnerTestHelper.dataSource), diff --git a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java index 7939ebec1acc..a1e3f5f9bace 100644 --- a/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/CalciteQueryTest.java @@ -555,7 +555,7 @@ public void testExplainSelectStar() throws Exception ImmutableList.of(), ImmutableList.of( new Object[]{ - "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n" + "DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":null,\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"m1\",\"m2\",\"unique_dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{__time:LONG, cnt:LONG, dim1:STRING, dim2:STRING, m1:FLOAT, m2:DOUBLE, unique_dim1:COMPLEX}])\n" } ) ); @@ -801,8 +801,8 @@ public void testExplainSelfJoinWithFallback() throws Exception { final String explanation = "BindableJoin(condition=[=($0, $2)], joinType=[inner])\n" - + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"\",\"extractionFn\":null}},\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false}], signature=[{dim1:STRING}])\n" - + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":null,\"columns\":[\"dim1\",\"dim2\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false}], signature=[{dim1:STRING, dim2:STRING}])\n"; + + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"\",\"extractionFn\":null}},\"columns\":[\"dim1\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{dim1:STRING}])\n" + + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":null,\"columns\":[\"dim1\",\"dim2\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{dim1:STRING, dim2:STRING}])\n"; testQuery( PLANNER_CONFIG_FALLBACK, @@ -6175,7 +6175,7 @@ public void testUsingSubqueryAsPartOfOrFilter() throws Exception + " BindableFilter(condition=[OR(=($0, 'xxx'), CAST(AND(IS NOT NULL($4), <>($2, 0))):BOOLEAN)])\n" + " BindableJoin(condition=[=($1, $3)], joinType=[left])\n" + " BindableJoin(condition=[true], joinType=[inner])\n" - + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":null,\"columns\":[\"dim1\",\"dim2\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false}], signature=[{dim1:STRING, dim2:STRING}])\n" + + " DruidQueryRel(query=[{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[],\"resultFormat\":\"compactedList\",\"batchSize\":20480,\"limit\":9223372036854775807,\"filter\":null,\"columns\":[\"dim1\",\"dim2\"],\"legacy\":false,\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false,\"granularity\":{\"type\":\"all\"}}], signature=[{dim1:STRING, dim2:STRING}])\n" + " DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"descending\":false,\"virtualColumns\":[],\"filter\":{\"type\":\"like\",\"dimension\":\"dim1\",\"pattern\":\"%bc\",\"escape\":null,\"extractionFn\":null},\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"postAggregations\":[],\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"skipEmptyBuckets\":true,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"}}], signature=[{a0:LONG}])\n" + " DruidQueryRel(query=[{\"queryType\":\"groupBy\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"virtualColumns\":[{\"type\":\"expression\",\"name\":\"d1:v\",\"expression\":\"1\",\"outputType\":\"LONG\"}],\"filter\":{\"type\":\"like\",\"dimension\":\"dim1\",\"pattern\":\"%bc\",\"escape\":null,\"extractionFn\":null},\"granularity\":{\"type\":\"all\"},\"dimensions\":[{\"type\":\"default\",\"dimension\":\"dim1\",\"outputName\":\"d0\",\"outputType\":\"STRING\"},{\"type\":\"default\",\"dimension\":\"d1:v\",\"outputName\":\"d1\",\"outputType\":\"LONG\"}],\"aggregations\":[],\"postAggregations\":[],\"having\":null,\"limitSpec\":{\"type\":\"NoopLimitSpec\"},\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\"},\"descending\":false}], signature=[{d0:STRING, d1:LONG}])\n"; From 08bd954095ea37a9edac1055ad1097d7fec8083b Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Fri, 12 Jan 2018 14:58:15 -0800 Subject: [PATCH 11/82] Fix APPROX_QUANTILE on outer groupBys. (#5253) (#5254) --- .../histogram/sql/QuantileSqlAggregator.java | 14 +++- .../sql/QuantileSqlAggregatorTest.java | 70 +++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregator.java b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregator.java index 6711c36f4b1b..b9c8d3d3d46b 100644 --- a/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregator.java +++ b/extensions-core/histogram/src/main/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregator.java @@ -93,15 +93,27 @@ public Aggregation toDruidAggregation( project, aggregateCall.getArgList().get(1) ); - final float probability = ((Number) RexLiteral.value(probabilityArg)).floatValue(); + if (!probabilityArg.isA(SqlKind.LITERAL)) { + // Probability must be a literal in order to plan. + return null; + } + + final float probability = ((Number) RexLiteral.value(probabilityArg)).floatValue(); final int resolution; + if (aggregateCall.getArgList().size() >= 3) { final RexNode resolutionArg = Expressions.fromFieldAccess( rowSignature, project, aggregateCall.getArgList().get(2) ); + + if (!resolutionArg.isA(SqlKind.LITERAL)) { + // Resolution must be a literal in order to plan. + return null; + } + resolution = ((Number) RexLiteral.value(resolutionArg)).intValue(); } else { resolution = ApproximateHistogram.DEFAULT_HISTOGRAM_SIZE; diff --git a/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java index 45197c2c7e0e..77f629e5a1a1 100644 --- a/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/io/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java @@ -27,6 +27,7 @@ import io.druid.java.util.common.guava.Sequences; import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.Druids; +import io.druid.query.QueryDataSource; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.DoubleSumAggregatorFactory; import io.druid.query.aggregation.FilteredAggregatorFactory; @@ -34,9 +35,13 @@ import io.druid.query.aggregation.histogram.ApproximateHistogramDruidModule; import io.druid.query.aggregation.histogram.ApproximateHistogramFoldingAggregatorFactory; import io.druid.query.aggregation.histogram.QuantilePostAggregator; +import io.druid.query.aggregation.post.ArithmeticPostAggregator; +import io.druid.query.aggregation.post.FieldAccessPostAggregator; +import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.expression.TestExprMacroTable; import io.druid.query.filter.NotDimFilter; import io.druid.query.filter.SelectorDimFilter; +import io.druid.query.groupby.GroupByQuery; import io.druid.query.spec.MultipleIntervalSegmentSpec; import io.druid.segment.IndexBuilder; import io.druid.segment.QueryableIndex; @@ -293,4 +298,69 @@ public void testQuantileOnComplexColumn() throws Exception ); } } + + @Test + public void testQuantileOnInnerQuery() throws Exception + { + try (final DruidPlanner planner = plannerFactory.createPlanner(null)) { + final String sql = "SELECT AVG(x), APPROX_QUANTILE(x, 0.98)\n" + + "FROM (SELECT dim2, SUM(m1) AS x FROM foo GROUP BY dim2)"; + + final PlannerResult plannerResult = planner.plan(sql); + + // Verify results + final List results = Sequences.toList(plannerResult.run(), new ArrayList()); + final List expectedResults = ImmutableList.of( + new Object[]{7.0, 8.26386833190918} + ); + Assert.assertEquals(expectedResults.size(), results.size()); + for (int i = 0; i < expectedResults.size(); i++) { + Assert.assertArrayEquals(expectedResults.get(i), results.get(i)); + } + + // Verify query + Assert.assertEquals( + GroupByQuery.builder() + .setDataSource( + new QueryDataSource( + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE1) + .setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))) + .setGranularity(Granularities.ALL) + .setDimensions(ImmutableList.of(new DefaultDimensionSpec("dim2", "d0"))) + .setAggregatorSpecs( + ImmutableList.of( + new DoubleSumAggregatorFactory("a0", "m1") + ) + ) + .setContext(ImmutableMap.of()) + .build() + ) + ) + .setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))) + .setGranularity(Granularities.ALL) + .setAggregatorSpecs(ImmutableList.of( + new DoubleSumAggregatorFactory("_a0:sum", "a0"), + new CountAggregatorFactory("_a0:count"), + new ApproximateHistogramAggregatorFactory("_a1:agg", "a0", null, null, null, null) + )) + .setPostAggregatorSpecs( + ImmutableList.of( + new ArithmeticPostAggregator( + "_a0", + "quotient", + ImmutableList.of( + new FieldAccessPostAggregator(null, "_a0:sum"), + new FieldAccessPostAggregator(null, "_a0:count") + ) + ), + new QuantilePostAggregator("_a1", "_a1:agg", 0.98f) + ) + ) + .setContext(ImmutableMap.of()) + .build(), + Iterables.getOnlyElement(queryLogHook.getRecordedQueries()) + ); + } + } } From d7fe7923f89fb6a8124b8de01ecb1bb544ca7db3 Mon Sep 17 00:00:00 2001 From: jon-wei Date: Thu, 18 Jan 2018 12:28:53 -0800 Subject: [PATCH 12/82] [maven-release-plugin] prepare release druid-0.12.0-rc1 --- api/pom.xml | 2 +- aws-common/pom.xml | 2 +- benchmarks/pom.xml | 2 +- bytebuffer-collections/pom.xml | 5 ++--- common/pom.xml | 2 +- distribution/pom.xml | 9 ++++----- examples/pom.xml | 2 +- extendedset/pom.xml | 5 ++--- extensions-contrib/ambari-metrics-emitter/pom.xml | 5 ++--- extensions-contrib/azure-extensions/pom.xml | 5 ++--- extensions-contrib/cassandra-storage/pom.xml | 2 +- extensions-contrib/cloudfiles-extensions/pom.xml | 5 ++--- extensions-contrib/distinctcount/pom.xml | 5 ++--- extensions-contrib/druid-rocketmq/pom.xml | 6 ++---- extensions-contrib/google-extensions/pom.xml | 2 +- extensions-contrib/graphite-emitter/pom.xml | 5 ++--- extensions-contrib/kafka-eight-simpleConsumer/pom.xml | 5 ++--- extensions-contrib/kafka-emitter/pom.xml | 5 ++--- extensions-contrib/orc-extensions/pom.xml | 6 ++---- extensions-contrib/parquet-extensions/pom.xml | 6 ++---- extensions-contrib/rabbitmq/pom.xml | 2 +- extensions-contrib/redis-cache/pom.xml | 5 ++--- extensions-contrib/sqlserver-metadata-storage/pom.xml | 2 +- extensions-contrib/statsd-emitter/pom.xml | 6 ++---- extensions-contrib/thrift-extensions/pom.xml | 6 ++---- extensions-contrib/time-min-max/pom.xml | 6 ++---- extensions-contrib/virtual-columns/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 5 ++--- extensions-core/datasketches/pom.xml | 5 ++--- extensions-core/druid-basic-security/pom.xml | 6 ++---- extensions-core/druid-kerberos/pom.xml | 5 ++--- extensions-core/hdfs-storage/pom.xml | 2 +- extensions-core/histogram/pom.xml | 2 +- extensions-core/kafka-eight/pom.xml | 2 +- extensions-core/kafka-extraction-namespace/pom.xml | 5 ++--- extensions-core/kafka-indexing-service/pom.xml | 2 +- extensions-core/lookups-cached-global/pom.xml | 5 ++--- extensions-core/lookups-cached-single/pom.xml | 5 ++--- extensions-core/mysql-metadata-storage/pom.xml | 2 +- extensions-core/postgresql-metadata-storage/pom.xml | 2 +- extensions-core/protobuf-extensions/pom.xml | 6 ++---- extensions-core/s3-extensions/pom.xml | 5 ++--- extensions-core/simple-client-sslcontext/pom.xml | 6 ++---- extensions-core/stats/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- integration-tests/pom.xml | 2 +- java-util/pom.xml | 2 +- pom.xml | 7 +++---- processing/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 4 ++-- sql/pom.xml | 5 ++--- 54 files changed, 87 insertions(+), 125 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index d4bcddd43704..c9a5fd4eef7d 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 diff --git a/aws-common/pom.xml b/aws-common/pom.xml index 608552f98f54..fd0132b0e245 100644 --- a/aws-common/pom.xml +++ b/aws-common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 596931c14391..0732d6bca394 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 diff --git a/bytebuffer-collections/pom.xml b/bytebuffer-collections/pom.xml index 4b8b231d5ea0..246f26f663dd 100755 --- a/bytebuffer-collections/pom.xml +++ b/bytebuffer-collections/pom.xml @@ -18,14 +18,13 @@ ~ under the License. --> - + 4.0.0 io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 bytebuffer-collections diff --git a/common/pom.xml b/common/pom.xml index 58379d338640..c09746fc0cb6 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 diff --git a/distribution/pom.xml b/distribution/pom.xml index 365ee356c46c..061c9c2c588b 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -16,8 +16,7 @@ ~ limitations under the License. --> - + 4.0.0 pom @@ -29,7 +28,7 @@ druid io.druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 @@ -60,7 +59,7 @@ java -classpath - + -Ddruid.extensions.loadList=[] -Ddruid.extensions.directory=${project.build.directory}/extensions @@ -190,7 +189,7 @@ java -classpath - + -Ddruid.extensions.loadList=[] -Ddruid.extensions.directory=${project.build.directory}/extensions diff --git a/examples/pom.xml b/examples/pom.xml index 231469f4b06e..d628cec75935 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 diff --git a/extendedset/pom.xml b/extendedset/pom.xml index 91f19144c8ec..ebbe6fc75365 100755 --- a/extendedset/pom.xml +++ b/extendedset/pom.xml @@ -18,8 +18,7 @@ ~ under the License. --> - + 4.0.0 extendedset @@ -32,7 +31,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index 14fe7cd44bb9..8d0ba1abd1a9 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -18,14 +18,13 @@ ~ under the License. --> - + 4.0.0 io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-contrib/azure-extensions/pom.xml b/extensions-contrib/azure-extensions/pom.xml index 1c647b0e91d4..66cfb74ec0aa 100644 --- a/extensions-contrib/azure-extensions/pom.xml +++ b/extensions-contrib/azure-extensions/pom.xml @@ -18,8 +18,7 @@ ~ under the License. --> - + 4.0.0 io.druid.extensions.contrib @@ -30,7 +29,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-contrib/cassandra-storage/pom.xml b/extensions-contrib/cassandra-storage/pom.xml index ea32b084b02f..93b36d558993 100644 --- a/extensions-contrib/cassandra-storage/pom.xml +++ b/extensions-contrib/cassandra-storage/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index da06a9e9e51a..a4a3efd175c2 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -18,8 +18,7 @@ ~ under the License. --> - + 4.0.0 io.druid.extensions.contrib @@ -30,7 +29,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-contrib/distinctcount/pom.xml b/extensions-contrib/distinctcount/pom.xml index 86c3fcc09c9b..86d6378daa63 100644 --- a/extensions-contrib/distinctcount/pom.xml +++ b/extensions-contrib/distinctcount/pom.xml @@ -18,8 +18,7 @@ ~ under the License. --> - + 4.0.0 io.druid.extensions.contrib @@ -30,7 +29,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-contrib/druid-rocketmq/pom.xml b/extensions-contrib/druid-rocketmq/pom.xml index 924a07cdd424..e10159dbcda4 100644 --- a/extensions-contrib/druid-rocketmq/pom.xml +++ b/extensions-contrib/druid-rocketmq/pom.xml @@ -17,14 +17,12 @@ ~ specific language governing permissions and limitations ~ under the License. --> - + 4.0.0 druid io.druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-contrib/google-extensions/pom.xml b/extensions-contrib/google-extensions/pom.xml index 20a3dd7ba3d4..28978ae0f366 100644 --- a/extensions-contrib/google-extensions/pom.xml +++ b/extensions-contrib/google-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index 8569823ea94b..dc1152f05007 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -18,14 +18,13 @@ ~ under the License. --> - + 4.0.0 io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml index 1c9f5c17ffe2..fba9bb699901 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml +++ b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml @@ -16,8 +16,7 @@ ~ limitations under the License. --> - + 4.0.0 io.druid.extensions.contrib druid-kafka-eight-simple-consumer @@ -27,7 +26,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index 9130834d243f..d1cd65802bee 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -18,14 +18,13 @@ ~ under the License. --> - + 4.0.0 io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-contrib/orc-extensions/pom.xml b/extensions-contrib/orc-extensions/pom.xml index 4eb48e949ebf..b91dc3128dfc 100644 --- a/extensions-contrib/orc-extensions/pom.xml +++ b/extensions-contrib/orc-extensions/pom.xml @@ -15,9 +15,7 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + io.druid.extensions.contrib druid-orc-extensions druid-orc-extensions @@ -26,7 +24,7 @@ druid io.druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/parquet-extensions/pom.xml b/extensions-contrib/parquet-extensions/pom.xml index b2a460fc2891..956f593f05b4 100644 --- a/extensions-contrib/parquet-extensions/pom.xml +++ b/extensions-contrib/parquet-extensions/pom.xml @@ -1,7 +1,5 @@ - + io.druid.extensions.contrib druid-parquet-extensions druid-parquet-extensions @@ -10,7 +8,7 @@ druid io.druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/rabbitmq/pom.xml b/extensions-contrib/rabbitmq/pom.xml index 8f1ebc47f199..09cce1954a30 100644 --- a/extensions-contrib/rabbitmq/pom.xml +++ b/extensions-contrib/rabbitmq/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-contrib/redis-cache/pom.xml b/extensions-contrib/redis-cache/pom.xml index 9d0c578bcd0d..02e8241f58d0 100644 --- a/extensions-contrib/redis-cache/pom.xml +++ b/extensions-contrib/redis-cache/pom.xml @@ -19,8 +19,7 @@ ~ under the License. --> - + 4.0.0 io.druid.extensions.contrib @@ -30,7 +29,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-contrib/sqlserver-metadata-storage/pom.xml b/extensions-contrib/sqlserver-metadata-storage/pom.xml index 0f3970cdf204..0d32ae602bd7 100644 --- a/extensions-contrib/sqlserver-metadata-storage/pom.xml +++ b/extensions-contrib/sqlserver-metadata-storage/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index 061b2a961a8d..03361e403ddf 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -17,13 +17,11 @@ ~ specific language governing permissions and limitations ~ under the License. --> - + druid io.druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index 18c51677a615..43c092dd8345 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -1,7 +1,5 @@ - + io.druid.extensions.contrib druid-thrift-extensions @@ -11,7 +9,7 @@ druid io.druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/time-min-max/pom.xml b/extensions-contrib/time-min-max/pom.xml index 59bc7ee6b1a7..a18e5cf7a03a 100644 --- a/extensions-contrib/time-min-max/pom.xml +++ b/extensions-contrib/time-min-max/pom.xml @@ -17,13 +17,11 @@ ~ specific language governing permissions and limitations ~ under the License. --> - + druid io.druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/virtual-columns/pom.xml b/extensions-contrib/virtual-columns/pom.xml index 3102a75d1d98..0d544cb2e3a8 100644 --- a/extensions-contrib/virtual-columns/pom.xml +++ b/extensions-contrib/virtual-columns/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index 9102202fac31..c2f441da1f06 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -18,8 +18,7 @@ ~ under the License. --> - + 4.0.0 io.druid.extensions @@ -30,7 +29,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index f7cf74e7458e..d345622c0bcb 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -18,8 +18,7 @@ ~ under the License. --> - + 4.0.0 io.druid.extensions @@ -30,7 +29,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-core/druid-basic-security/pom.xml b/extensions-core/druid-basic-security/pom.xml index 1de016381863..7e82757ade57 100644 --- a/extensions-core/druid-basic-security/pom.xml +++ b/extensions-core/druid-basic-security/pom.xml @@ -17,9 +17,7 @@ ~ limitations under the License. --> - + 4.0.0 io.druid.extensions @@ -30,7 +28,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index 08696bed8122..d8efbe83811d 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -18,8 +18,7 @@ ~ under the License. --> - + 4.0.0 io.druid.extensions @@ -30,7 +29,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index 5b6bb1cd4320..88e968f8fb62 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-core/histogram/pom.xml b/extensions-core/histogram/pom.xml index 5f0a3765d649..63f64ad5d4be 100644 --- a/extensions-core/histogram/pom.xml +++ b/extensions-core/histogram/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-core/kafka-eight/pom.xml b/extensions-core/kafka-eight/pom.xml index 2b94032c5878..acb1ccd8234b 100644 --- a/extensions-core/kafka-eight/pom.xml +++ b/extensions-core/kafka-eight/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index 0edc753e9bc0..1e4e69704e6e 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -18,8 +18,7 @@ ~ under the License. --> - + 4.0.0 io.druid.extensions druid-kafka-extraction-namespace @@ -29,7 +28,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index 007336e282f8..588092fd6e99 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-core/lookups-cached-global/pom.xml b/extensions-core/lookups-cached-global/pom.xml index 4ce26c7a803c..cdbafb77ed36 100644 --- a/extensions-core/lookups-cached-global/pom.xml +++ b/extensions-core/lookups-cached-global/pom.xml @@ -18,8 +18,7 @@ ~ under the License. --> - + 4.0.0 io.druid.extensions druid-lookups-cached-global @@ -29,7 +28,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-core/lookups-cached-single/pom.xml b/extensions-core/lookups-cached-single/pom.xml index 9b4ea5a5ea36..d5c5c99a8a72 100644 --- a/extensions-core/lookups-cached-single/pom.xml +++ b/extensions-core/lookups-cached-single/pom.xml @@ -18,8 +18,7 @@ ~ under the License. --> - + 4.0.0 io.druid.extensions druid-lookups-cached-single @@ -29,7 +28,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index 950a985ce8ff..cc5bab8a86e3 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-core/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml index d4625a6499ce..641806831381 100644 --- a/extensions-core/postgresql-metadata-storage/pom.xml +++ b/extensions-core/postgresql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-core/protobuf-extensions/pom.xml b/extensions-core/protobuf-extensions/pom.xml index e1a14d412b6e..30e248ff47c6 100644 --- a/extensions-core/protobuf-extensions/pom.xml +++ b/extensions-core/protobuf-extensions/pom.xml @@ -16,9 +16,7 @@ ~ limitations under the License. --> - + 4.0.0 io.druid.extensions @@ -29,7 +27,7 @@ druid io.druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index c08208c81e1e..f9cfbd6b011d 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -16,8 +16,7 @@ ~ limitations under the License. --> - + 4.0.0 io.druid.extensions @@ -28,7 +27,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/extensions-core/simple-client-sslcontext/pom.xml b/extensions-core/simple-client-sslcontext/pom.xml index 9277db3a5526..02f0ca8a7d92 100644 --- a/extensions-core/simple-client-sslcontext/pom.xml +++ b/extensions-core/simple-client-sslcontext/pom.xml @@ -1,11 +1,9 @@ - + druid io.druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml 4.0.0 diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml index e6dd75e687d5..d46b7c09abe7 100644 --- a/extensions-core/stats/pom.xml +++ b/extensions-core/stats/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 ../../pom.xml diff --git a/hll/pom.xml b/hll/pom.xml index 653b2ca63b32..f43b7a454394 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 druid-hll diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 9ab3867d3bf9..a8c62254594f 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index eedb98c94875..e30732b384b4 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 0c17f2fa2038..5607b21914c5 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 diff --git a/java-util/pom.xml b/java-util/pom.xml index 84f093753e44..381a25d14e8a 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 java-util diff --git a/pom.xml b/pom.xml index fe9546068c3d..47f8a3ed0f7b 100644 --- a/pom.xml +++ b/pom.xml @@ -16,8 +16,7 @@ ~ limitations under the License. --> - + 4.0.0 @@ -27,7 +26,7 @@ druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 pom ${project.groupId}:${project.artifactId} @@ -54,7 +53,7 @@ scm:git:ssh://git@github.com/druid-io/druid.git scm:git:ssh://git@github.com/druid-io/druid.git https://github.com/druid-io/druid.git - 0.12.0-SNAPSHOT + druid-0.12.0-rc1 diff --git a/processing/pom.xml b/processing/pom.xml index c5c74ab39d1f..6a407109c5e3 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 diff --git a/server/pom.xml b/server/pom.xml index f1aa77c1f7ee..78cd667120c7 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 diff --git a/services/pom.xml b/services/pom.xml index 9d164c847048..fd37f592e28f 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -25,7 +25,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 @@ -116,7 +116,7 @@ - + diff --git a/sql/pom.xml b/sql/pom.xml index da49d38fc62b..921196083d04 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -18,8 +18,7 @@ ~ under the License. --> - + 4.0.0 druid-sql @@ -29,7 +28,7 @@ io.druid druid - 0.12.0-SNAPSHOT + 0.12.0-rc1 From b287f8f36386ff22e1309933da6381a8d578e274 Mon Sep 17 00:00:00 2001 From: jon-wei Date: Thu, 18 Jan 2018 12:29:05 -0800 Subject: [PATCH 13/82] [maven-release-plugin] prepare for next development iteration --- api/pom.xml | 2 +- aws-common/pom.xml | 2 +- benchmarks/pom.xml | 2 +- bytebuffer-collections/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- extendedset/pom.xml | 2 +- extensions-contrib/ambari-metrics-emitter/pom.xml | 2 +- extensions-contrib/azure-extensions/pom.xml | 2 +- extensions-contrib/cassandra-storage/pom.xml | 2 +- extensions-contrib/cloudfiles-extensions/pom.xml | 2 +- extensions-contrib/distinctcount/pom.xml | 2 +- extensions-contrib/druid-rocketmq/pom.xml | 2 +- extensions-contrib/google-extensions/pom.xml | 2 +- extensions-contrib/graphite-emitter/pom.xml | 2 +- extensions-contrib/kafka-eight-simpleConsumer/pom.xml | 2 +- extensions-contrib/kafka-emitter/pom.xml | 2 +- extensions-contrib/orc-extensions/pom.xml | 2 +- extensions-contrib/parquet-extensions/pom.xml | 2 +- extensions-contrib/rabbitmq/pom.xml | 2 +- extensions-contrib/redis-cache/pom.xml | 2 +- extensions-contrib/sqlserver-metadata-storage/pom.xml | 2 +- extensions-contrib/statsd-emitter/pom.xml | 2 +- extensions-contrib/thrift-extensions/pom.xml | 2 +- extensions-contrib/time-min-max/pom.xml | 2 +- extensions-contrib/virtual-columns/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 2 +- extensions-core/datasketches/pom.xml | 2 +- extensions-core/druid-basic-security/pom.xml | 2 +- extensions-core/druid-kerberos/pom.xml | 2 +- extensions-core/hdfs-storage/pom.xml | 2 +- extensions-core/histogram/pom.xml | 2 +- extensions-core/kafka-eight/pom.xml | 2 +- extensions-core/kafka-extraction-namespace/pom.xml | 2 +- extensions-core/kafka-indexing-service/pom.xml | 2 +- extensions-core/lookups-cached-global/pom.xml | 2 +- extensions-core/lookups-cached-single/pom.xml | 2 +- extensions-core/mysql-metadata-storage/pom.xml | 2 +- extensions-core/postgresql-metadata-storage/pom.xml | 2 +- extensions-core/protobuf-extensions/pom.xml | 2 +- extensions-core/s3-extensions/pom.xml | 2 +- extensions-core/simple-client-sslcontext/pom.xml | 2 +- extensions-core/stats/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- integration-tests/pom.xml | 2 +- java-util/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- sql/pom.xml | 2 +- 54 files changed, 55 insertions(+), 55 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index c9a5fd4eef7d..1c682410c835 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT diff --git a/aws-common/pom.xml b/aws-common/pom.xml index fd0132b0e245..93461b80a807 100644 --- a/aws-common/pom.xml +++ b/aws-common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 0732d6bca394..2c008c6cb96b 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT diff --git a/bytebuffer-collections/pom.xml b/bytebuffer-collections/pom.xml index 246f26f663dd..435e82b9759f 100755 --- a/bytebuffer-collections/pom.xml +++ b/bytebuffer-collections/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT bytebuffer-collections diff --git a/common/pom.xml b/common/pom.xml index c09746fc0cb6..fbe104a2f7e4 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT diff --git a/distribution/pom.xml b/distribution/pom.xml index 061c9c2c588b..a4e07724ab93 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -28,7 +28,7 @@ druid io.druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index d628cec75935..15a23eb80697 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT diff --git a/extendedset/pom.xml b/extendedset/pom.xml index ebbe6fc75365..f055966c5380 100755 --- a/extendedset/pom.xml +++ b/extendedset/pom.xml @@ -31,7 +31,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index 8d0ba1abd1a9..a61468d2f5bd 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/azure-extensions/pom.xml b/extensions-contrib/azure-extensions/pom.xml index 66cfb74ec0aa..d45421bca1a2 100644 --- a/extensions-contrib/azure-extensions/pom.xml +++ b/extensions-contrib/azure-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/cassandra-storage/pom.xml b/extensions-contrib/cassandra-storage/pom.xml index 93b36d558993..8f984ab912e7 100644 --- a/extensions-contrib/cassandra-storage/pom.xml +++ b/extensions-contrib/cassandra-storage/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index a4a3efd175c2..e12f3a6625fb 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/distinctcount/pom.xml b/extensions-contrib/distinctcount/pom.xml index 86d6378daa63..04c15bc285b4 100644 --- a/extensions-contrib/distinctcount/pom.xml +++ b/extensions-contrib/distinctcount/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/druid-rocketmq/pom.xml b/extensions-contrib/druid-rocketmq/pom.xml index e10159dbcda4..faf5ca728bc6 100644 --- a/extensions-contrib/druid-rocketmq/pom.xml +++ b/extensions-contrib/druid-rocketmq/pom.xml @@ -22,7 +22,7 @@ druid io.druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/google-extensions/pom.xml b/extensions-contrib/google-extensions/pom.xml index 28978ae0f366..86fc2a529aea 100644 --- a/extensions-contrib/google-extensions/pom.xml +++ b/extensions-contrib/google-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index dc1152f05007..70ac15a00fff 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml index fba9bb699901..4e7ce512421b 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml +++ b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index d1cd65802bee..cec80c6dcd5a 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/orc-extensions/pom.xml b/extensions-contrib/orc-extensions/pom.xml index b91dc3128dfc..cce83a1f00db 100644 --- a/extensions-contrib/orc-extensions/pom.xml +++ b/extensions-contrib/orc-extensions/pom.xml @@ -24,7 +24,7 @@ druid io.druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/parquet-extensions/pom.xml b/extensions-contrib/parquet-extensions/pom.xml index 956f593f05b4..22ae9f2fc121 100644 --- a/extensions-contrib/parquet-extensions/pom.xml +++ b/extensions-contrib/parquet-extensions/pom.xml @@ -8,7 +8,7 @@ druid io.druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/rabbitmq/pom.xml b/extensions-contrib/rabbitmq/pom.xml index 09cce1954a30..74980e39ce24 100644 --- a/extensions-contrib/rabbitmq/pom.xml +++ b/extensions-contrib/rabbitmq/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/redis-cache/pom.xml b/extensions-contrib/redis-cache/pom.xml index 02e8241f58d0..2875c9b79d6f 100644 --- a/extensions-contrib/redis-cache/pom.xml +++ b/extensions-contrib/redis-cache/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/sqlserver-metadata-storage/pom.xml b/extensions-contrib/sqlserver-metadata-storage/pom.xml index 0d32ae602bd7..e6abb94e2f9a 100644 --- a/extensions-contrib/sqlserver-metadata-storage/pom.xml +++ b/extensions-contrib/sqlserver-metadata-storage/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index 03361e403ddf..542972d2bd6f 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index 43c092dd8345..d76e4f1568bc 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -9,7 +9,7 @@ druid io.druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/time-min-max/pom.xml b/extensions-contrib/time-min-max/pom.xml index a18e5cf7a03a..a91bc4b97b8f 100644 --- a/extensions-contrib/time-min-max/pom.xml +++ b/extensions-contrib/time-min-max/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/virtual-columns/pom.xml b/extensions-contrib/virtual-columns/pom.xml index 0d544cb2e3a8..d2aa4cfe127e 100644 --- a/extensions-contrib/virtual-columns/pom.xml +++ b/extensions-contrib/virtual-columns/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index c2f441da1f06..20499946fe6e 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index d345622c0bcb..6b3916845b3e 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-basic-security/pom.xml b/extensions-core/druid-basic-security/pom.xml index 7e82757ade57..7e4b8b4faf99 100644 --- a/extensions-core/druid-basic-security/pom.xml +++ b/extensions-core/druid-basic-security/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index d8efbe83811d..acb82457ef0f 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index 88e968f8fb62..4d461c297102 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/histogram/pom.xml b/extensions-core/histogram/pom.xml index 63f64ad5d4be..e08cb5d29d11 100644 --- a/extensions-core/histogram/pom.xml +++ b/extensions-core/histogram/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-eight/pom.xml b/extensions-core/kafka-eight/pom.xml index acb1ccd8234b..47ccdd340f13 100644 --- a/extensions-core/kafka-eight/pom.xml +++ b/extensions-core/kafka-eight/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index 1e4e69704e6e..1ae74d56f254 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index 588092fd6e99..9c0ba11c0a88 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/lookups-cached-global/pom.xml b/extensions-core/lookups-cached-global/pom.xml index cdbafb77ed36..7c75dee17106 100644 --- a/extensions-core/lookups-cached-global/pom.xml +++ b/extensions-core/lookups-cached-global/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/lookups-cached-single/pom.xml b/extensions-core/lookups-cached-single/pom.xml index d5c5c99a8a72..28614129f85d 100644 --- a/extensions-core/lookups-cached-single/pom.xml +++ b/extensions-core/lookups-cached-single/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index cc5bab8a86e3..33ec83e9adfd 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml index 641806831381..a8fd60ce036d 100644 --- a/extensions-core/postgresql-metadata-storage/pom.xml +++ b/extensions-core/postgresql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/protobuf-extensions/pom.xml b/extensions-core/protobuf-extensions/pom.xml index 30e248ff47c6..85fc52cd6342 100644 --- a/extensions-core/protobuf-extensions/pom.xml +++ b/extensions-core/protobuf-extensions/pom.xml @@ -27,7 +27,7 @@ druid io.druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index f9cfbd6b011d..207369684c12 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/simple-client-sslcontext/pom.xml b/extensions-core/simple-client-sslcontext/pom.xml index 02f0ca8a7d92..3f2defdd9da3 100644 --- a/extensions-core/simple-client-sslcontext/pom.xml +++ b/extensions-core/simple-client-sslcontext/pom.xml @@ -3,7 +3,7 @@ druid io.druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml index d46b7c09abe7..d666981362f2 100644 --- a/extensions-core/stats/pom.xml +++ b/extensions-core/stats/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT ../../pom.xml diff --git a/hll/pom.xml b/hll/pom.xml index f43b7a454394..700775d48295 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT druid-hll diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index a8c62254594f..e947275bb56f 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index e30732b384b4..e5161b7963f3 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 5607b21914c5..a69881914c1a 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT diff --git a/java-util/pom.xml b/java-util/pom.xml index 381a25d14e8a..27fa96e77fa5 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT java-util diff --git a/pom.xml b/pom.xml index 47f8a3ed0f7b..a2cf140d708a 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT pom ${project.groupId}:${project.artifactId} @@ -53,7 +53,7 @@ scm:git:ssh://git@github.com/druid-io/druid.git scm:git:ssh://git@github.com/druid-io/druid.git https://github.com/druid-io/druid.git - druid-0.12.0-rc1 + 0.12.0-SNAPSHOT diff --git a/processing/pom.xml b/processing/pom.xml index 6a407109c5e3..b2ff197662bd 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 78cd667120c7..7c7aa8fe6df4 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index fd37f592e28f..62ac7d2e17ec 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -25,7 +25,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT diff --git a/sql/pom.xml b/sql/pom.xml index 921196083d04..f1776868e1d6 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc1 + 0.12.0-rc2-SNAPSHOT From 5a8b106990e6bc6af4e78ab6937108e1675a03dd Mon Sep 17 00:00:00 2001 From: Roman Leventov Date: Tue, 23 Jan 2018 18:51:03 +0100 Subject: [PATCH 14/82] Fix bugs in ImmutableRTree; Merge bytebuffer-collections module into druid-processing (#5275) (#5283) * Fix bugs in ImmutableRTree; optimize ImmmutableRTreeObjectStrategy.writeTo(); Merge bytebuffer-collections module into druid-processing * Remove unused declaration * Fix another bug --- bytebuffer-collections/pom.xml | 137 ------------------ pom.xml | 1 - ...tions.bitmap.RangeBitmapBenchmarkTest.html | 0 ...ions.bitmap.RangeBitmapBenchmarkTest.jsonp | 0 ...ons.bitmap.UniformBitmapBenchmarkTest.html | 0 ...ns.bitmap.UniformBitmapBenchmarkTest.jsonp | 0 processing/pom.xml | 54 ++++++- .../java/io/druid/collections/IntegerSet.java | 0 .../bitmap/BitSetBitmapFactory.java | 0 .../collections/bitmap/BitmapFactory.java | 0 .../bitmap/ConciseBitmapFactory.java | 0 .../collections/bitmap/ImmutableBitmap.java | 0 .../collections/bitmap/MutableBitmap.java | 0 .../bitmap/RoaringBitmapFactory.java | 0 .../bitmap/WrappedBitSetBitmap.java | 0 .../bitmap/WrappedConciseBitmap.java | 0 .../bitmap/WrappedImmutableBitSetBitmap.java | 0 .../bitmap/WrappedImmutableConciseBitmap.java | 5 - .../bitmap/WrappedImmutableRoaringBitmap.java | 0 .../bitmap/WrappedRoaringBitmap.java | 0 .../collections/spatial/ImmutableNode.java | 0 .../collections/spatial/ImmutablePoint.java | 0 .../collections/spatial/ImmutableRTree.java | 42 ++++-- .../io/druid/collections/spatial/Node.java | 0 .../io/druid/collections/spatial/Point.java | 0 .../io/druid/collections/spatial/RTree.java | 0 .../druid/collections/spatial/RTreeUtils.java | 0 .../collections/spatial/search/Bound.java | 0 .../spatial/search/GutmanSearchStrategy.java | 0 .../spatial/search/PolygonBound.java | 0 .../spatial/search/RadiusBound.java | 0 .../spatial/search/RectangularBound.java | 0 .../spatial/search/SearchStrategy.java | 0 .../spatial/split/GutmanSplitStrategy.java | 0 .../split/LinearGutmanSplitStrategy.java | 0 .../spatial/split/SplitStrategy.java | 0 .../ColumnSelectorBitmapIndexSelector.java | 4 +- .../io/druid/segment/data/GenericIndexed.java | 4 +- .../segment/data/GenericIndexedWriter.java | 4 +- .../data/ImmutableRTreeObjectStrategy.java | 12 +- .../segment/serde/ComplexMetricSerde.java | 6 +- .../druid/collections/IntSetTestUtility.java | 0 .../io/druid/collections/TestIntegerSet.java | 0 .../collections/bitmap/BitmapBenchmark.java | 4 +- .../bitmap/BitmapIterationTest.java | 0 .../bitmap/ConciseBitmapFactoryTest.java | 0 .../bitmap/RangeBitmapBenchmarkTest.java | 5 +- .../bitmap/RoaringBitmapFactoryTest.java | 0 .../bitmap/UniformBitmapBenchmarkTest.java | 5 +- .../bitmap/WrappedBitSetBitmapBitSetTest.java | 0 .../bitmap/WrappedRoaringBitmapTest.java | 0 .../spatial/ImmutableRTreeTest.java | 24 +++ .../druid/collections/spatial/RTreeTest.java | 0 .../spatial/search/PolygonBoundTest.java | 0 .../spatial/search/RadiusBoundTest.java | 0 .../spatial/search/RectangularBoundTest.java | 0 .../split/LinearGutmanSplitStrategyTest.java | 0 .../test/annotation/Benchmark.java | 2 +- .../collections}/test/annotation/Dummy.java | 2 +- .../druid/client/cache/CacheConfigTest.java | 2 +- 60 files changed, 143 insertions(+), 170 deletions(-) delete mode 100755 bytebuffer-collections/pom.xml rename {bytebuffer-collections => processing}/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.html (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.jsonp (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.html (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.jsonp (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/IntegerSet.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/bitmap/BitSetBitmapFactory.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/bitmap/BitmapFactory.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/bitmap/ImmutableBitmap.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/bitmap/MutableBitmap.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/bitmap/RoaringBitmapFactory.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/bitmap/WrappedBitSetBitmap.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/bitmap/WrappedConciseBitmap.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/bitmap/WrappedImmutableBitSetBitmap.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java (94%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/bitmap/WrappedImmutableRoaringBitmap.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/bitmap/WrappedRoaringBitmap.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/spatial/ImmutableNode.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/spatial/ImmutablePoint.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/spatial/ImmutableRTree.java (79%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/spatial/Node.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/spatial/Point.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/spatial/RTree.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/spatial/RTreeUtils.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/spatial/search/Bound.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/spatial/search/GutmanSearchStrategy.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/spatial/search/PolygonBound.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/spatial/search/RadiusBound.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/spatial/search/RectangularBound.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/spatial/search/SearchStrategy.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/spatial/split/GutmanSplitStrategy.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategy.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/main/java/io/druid/collections/spatial/split/SplitStrategy.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/test/java/io/druid/collections/IntSetTestUtility.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/test/java/io/druid/collections/TestIntegerSet.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java (99%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/test/java/io/druid/collections/bitmap/BitmapIterationTest.java (100%) rename {bytebuffer-collections => processing}/src/test/java/io/druid/collections/bitmap/ConciseBitmapFactoryTest.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/test/java/io/druid/collections/bitmap/RangeBitmapBenchmarkTest.java (96%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/test/java/io/druid/collections/bitmap/RoaringBitmapFactoryTest.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/test/java/io/druid/collections/bitmap/UniformBitmapBenchmarkTest.java (96%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/test/java/io/druid/collections/bitmap/WrappedBitSetBitmapBitSetTest.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/test/java/io/druid/collections/bitmap/WrappedRoaringBitmapTest.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java (95%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/test/java/io/druid/collections/spatial/RTreeTest.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/test/java/io/druid/collections/spatial/search/PolygonBoundTest.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/test/java/io/druid/collections/spatial/search/RadiusBoundTest.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/test/java/io/druid/collections/spatial/search/RectangularBoundTest.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections => processing}/src/test/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategyTest.java (100%) mode change 100755 => 100644 rename {bytebuffer-collections/src/test/java/io/druid => processing/src/test/java/io/druid/collections}/test/annotation/Benchmark.java (94%) mode change 100755 => 100644 rename {bytebuffer-collections/src/test/java/io/druid => processing/src/test/java/io/druid/collections}/test/annotation/Dummy.java (94%) mode change 100755 => 100644 diff --git a/bytebuffer-collections/pom.xml b/bytebuffer-collections/pom.xml deleted file mode 100755 index 435e82b9759f..000000000000 --- a/bytebuffer-collections/pom.xml +++ /dev/null @@ -1,137 +0,0 @@ - - - - - 4.0.0 - - - io.druid - druid - 0.12.0-rc2-SNAPSHOT - - - bytebuffer-collections - bytebuffer-collections - ByteBuffer Collections - - - - io.druid - extendedset - ${project.parent.version} - - - com.google.guava - guava - 16.0.1 - - - com.fasterxml.jackson.core - jackson-annotations - 2.4.6 - - - com.fasterxml.jackson.core - jackson-core - 2.4.6 - - - com.fasterxml.jackson.core - jackson-databind - 2.4.6 - - - org.roaringbitmap - RoaringBitmap - - - - - junit - junit - test - - - org.easymock - easymock - 3.0 - test - - - com.carrotsearch - junit-benchmarks - 0.7.2 - test - - - com.h2database - h2 - 1.4.182 - test - - - com.google.guava - guava-testlib - test - - - - - - - org.apache.maven.plugins - maven-jar-plugin - 2.4 - - - - test-jar - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - io.druid.test.annotation.Benchmark - - - - - - - - benchmark - - - - maven-surefire-plugin - - -server -Xms3G -Xmx3G -Djub.consumers=CONSOLE,H2 -Djub.db.file=benchmarks/benchmarks - io.druid.test.annotation.Benchmark - io.druid.test.annotation.Dummy - - - - - - - diff --git a/pom.xml b/pom.xml index a2cf140d708a..5d187eb938ef 100644 --- a/pom.xml +++ b/pom.xml @@ -99,7 +99,6 @@ benchmarks aws-common java-util - bytebuffer-collections extendedset hll diff --git a/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.html b/processing/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.html old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.html rename to processing/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.html diff --git a/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.jsonp b/processing/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.jsonp old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.jsonp rename to processing/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.jsonp diff --git a/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.html b/processing/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.html old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.html rename to processing/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.html diff --git a/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.jsonp b/processing/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.jsonp old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.jsonp rename to processing/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.jsonp diff --git a/processing/pom.xml b/processing/pom.xml index b2ff197662bd..d964346cce94 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -47,9 +47,25 @@ io.druid - bytebuffer-collections + extendedset ${project.parent.version} + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + org.roaringbitmap + RoaringBitmap + it.unimi.dsi fastutil @@ -125,6 +141,17 @@ JUnitParams test + + com.h2database + h2 + 1.4.182 + test + + + com.google.guava + guava-testlib + test + @@ -156,7 +183,32 @@ + + org.apache.maven.plugins + maven-surefire-plugin + + io.druid.collections.test.annotation.Benchmark + + + + + benchmark + + + + maven-surefire-plugin + + -server -Xms3G -Xmx3G -Djub.consumers=CONSOLE,H2 -Djub.db.file=benchmarks/benchmarks + io.druid.collections.test.annotation.Benchmark + io.druid.collections.test.annotation.Dummy + + + + + + + diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/IntegerSet.java b/processing/src/main/java/io/druid/collections/IntegerSet.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/IntegerSet.java rename to processing/src/main/java/io/druid/collections/IntegerSet.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/BitSetBitmapFactory.java b/processing/src/main/java/io/druid/collections/bitmap/BitSetBitmapFactory.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/BitSetBitmapFactory.java rename to processing/src/main/java/io/druid/collections/bitmap/BitSetBitmapFactory.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/BitmapFactory.java b/processing/src/main/java/io/druid/collections/bitmap/BitmapFactory.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/BitmapFactory.java rename to processing/src/main/java/io/druid/collections/bitmap/BitmapFactory.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java b/processing/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java rename to processing/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ImmutableBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/ImmutableBitmap.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ImmutableBitmap.java rename to processing/src/main/java/io/druid/collections/bitmap/ImmutableBitmap.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/MutableBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/MutableBitmap.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/MutableBitmap.java rename to processing/src/main/java/io/druid/collections/bitmap/MutableBitmap.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/RoaringBitmapFactory.java b/processing/src/main/java/io/druid/collections/bitmap/RoaringBitmapFactory.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/RoaringBitmapFactory.java rename to processing/src/main/java/io/druid/collections/bitmap/RoaringBitmapFactory.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedBitSetBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/WrappedBitSetBitmap.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedBitSetBitmap.java rename to processing/src/main/java/io/druid/collections/bitmap/WrappedBitSetBitmap.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedConciseBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/WrappedConciseBitmap.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedConciseBitmap.java rename to processing/src/main/java/io/druid/collections/bitmap/WrappedConciseBitmap.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableBitSetBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableBitSetBitmap.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableBitSetBitmap.java rename to processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableBitSetBitmap.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java old mode 100755 new mode 100644 similarity index 94% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java rename to processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java index 01768cfecf97..349fb493743e --- a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java +++ b/processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java @@ -64,11 +64,6 @@ public byte[] toBytes() return bitmap.toBytes(); } - public int compareTo(ImmutableBitmap other) - { - return bitmap.compareTo(((WrappedImmutableConciseBitmap) other).getBitmap()); - } - @Override public String toString() { diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableRoaringBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableRoaringBitmap.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableRoaringBitmap.java rename to processing/src/main/java/io/druid/collections/bitmap/WrappedImmutableRoaringBitmap.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedRoaringBitmap.java b/processing/src/main/java/io/druid/collections/bitmap/WrappedRoaringBitmap.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedRoaringBitmap.java rename to processing/src/main/java/io/druid/collections/bitmap/WrappedRoaringBitmap.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableNode.java b/processing/src/main/java/io/druid/collections/spatial/ImmutableNode.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableNode.java rename to processing/src/main/java/io/druid/collections/spatial/ImmutableNode.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutablePoint.java b/processing/src/main/java/io/druid/collections/spatial/ImmutablePoint.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutablePoint.java rename to processing/src/main/java/io/druid/collections/spatial/ImmutablePoint.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableRTree.java b/processing/src/main/java/io/druid/collections/spatial/ImmutableRTree.java old mode 100755 new mode 100644 similarity index 79% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableRTree.java rename to processing/src/main/java/io/druid/collections/spatial/ImmutableRTree.java index 2cb3e3d4acf0..9b442d3c5bc8 --- a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableRTree.java +++ b/processing/src/main/java/io/druid/collections/spatial/ImmutableRTree.java @@ -27,24 +27,36 @@ import io.druid.collections.spatial.search.Bound; import io.druid.collections.spatial.search.GutmanSearchStrategy; import io.druid.collections.spatial.search.SearchStrategy; +import io.druid.io.Channels; +import io.druid.segment.writeout.WriteOutBytes; +import it.unimi.dsi.fastutil.bytes.ByteArrays; +import java.io.IOException; import java.nio.ByteBuffer; /** * An immutable representation of an {@link RTree} for spatial indexing. */ -public class ImmutableRTree +public final class ImmutableRTree { - private static byte VERSION = 0x0; + private static final byte VERSION = 0x0; + + private static final ImmutableRTree EMPTY = new ImmutableRTree(); + + public static ImmutableRTree empty() + { + return EMPTY; + } + private final int numDims; private final ImmutableNode root; private final ByteBuffer data; private final SearchStrategy defaultSearchStrategy = new GutmanSearchStrategy(); - public ImmutableRTree() + private ImmutableRTree() { this.numDims = 0; - this.data = ByteBuffer.wrap(new byte[]{}); + this.data = ByteBuffer.wrap(ByteArrays.EMPTY_ARRAY); this.root = null; } @@ -52,7 +64,7 @@ public ImmutableRTree(ByteBuffer data, BitmapFactory bitmapFactory) { data = data.asReadOnlyBuffer(); final int initPosition = data.position(); - Preconditions.checkArgument(data.get(0) == VERSION, "Mismatching versions"); + Preconditions.checkArgument(data.get(initPosition) == VERSION, "Mismatching versions"); this.numDims = data.getInt(1 + initPosition) & 0x7FFF; this.data = data; this.root = new ImmutableNode(numDims, initPosition, 1 + Ints.BYTES, data, bitmapFactory); @@ -61,7 +73,7 @@ public ImmutableRTree(ByteBuffer data, BitmapFactory bitmapFactory) public static ImmutableRTree newImmutableFromMutable(RTree rTree) { if (rTree.getSize() == 0) { - return new ImmutableRTree(); + return empty(); } ByteBuffer buffer = ByteBuffer.wrap(new byte[calcNumBytes(rTree)]); @@ -103,7 +115,7 @@ private static int calcNodeBytes(Node node) public int size() { - return data.capacity(); + return data.remaining(); } public Iterable search(Bound bound) @@ -123,9 +135,19 @@ public Iterable search(SearchStrategy strategy, Bound bound) public byte[] toBytes() { - ByteBuffer buf = ByteBuffer.allocate(data.capacity()); - buf.put(data.asReadOnlyBuffer()); - return buf.array(); + if (size() == 0) { + return ByteArrays.EMPTY_ARRAY; + } + byte[] res = new byte[data.remaining()]; + data.duplicate().get(res); + return res; + } + + public void writeTo(WriteOutBytes out) throws IOException + { + if (size() != 0) { + Channels.writeFully(out, data.duplicate()); + } } public int compareTo(ImmutableRTree other) diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/Node.java b/processing/src/main/java/io/druid/collections/spatial/Node.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/Node.java rename to processing/src/main/java/io/druid/collections/spatial/Node.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/Point.java b/processing/src/main/java/io/druid/collections/spatial/Point.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/Point.java rename to processing/src/main/java/io/druid/collections/spatial/Point.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/RTree.java b/processing/src/main/java/io/druid/collections/spatial/RTree.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/RTree.java rename to processing/src/main/java/io/druid/collections/spatial/RTree.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/RTreeUtils.java b/processing/src/main/java/io/druid/collections/spatial/RTreeUtils.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/RTreeUtils.java rename to processing/src/main/java/io/druid/collections/spatial/RTreeUtils.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/Bound.java b/processing/src/main/java/io/druid/collections/spatial/search/Bound.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/Bound.java rename to processing/src/main/java/io/druid/collections/spatial/search/Bound.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/GutmanSearchStrategy.java b/processing/src/main/java/io/druid/collections/spatial/search/GutmanSearchStrategy.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/GutmanSearchStrategy.java rename to processing/src/main/java/io/druid/collections/spatial/search/GutmanSearchStrategy.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/PolygonBound.java b/processing/src/main/java/io/druid/collections/spatial/search/PolygonBound.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/PolygonBound.java rename to processing/src/main/java/io/druid/collections/spatial/search/PolygonBound.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/RadiusBound.java b/processing/src/main/java/io/druid/collections/spatial/search/RadiusBound.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/RadiusBound.java rename to processing/src/main/java/io/druid/collections/spatial/search/RadiusBound.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/RectangularBound.java b/processing/src/main/java/io/druid/collections/spatial/search/RectangularBound.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/RectangularBound.java rename to processing/src/main/java/io/druid/collections/spatial/search/RectangularBound.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/SearchStrategy.java b/processing/src/main/java/io/druid/collections/spatial/search/SearchStrategy.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/SearchStrategy.java rename to processing/src/main/java/io/druid/collections/spatial/search/SearchStrategy.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/GutmanSplitStrategy.java b/processing/src/main/java/io/druid/collections/spatial/split/GutmanSplitStrategy.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/GutmanSplitStrategy.java rename to processing/src/main/java/io/druid/collections/spatial/split/GutmanSplitStrategy.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategy.java b/processing/src/main/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategy.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategy.java rename to processing/src/main/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategy.java diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/SplitStrategy.java b/processing/src/main/java/io/druid/collections/spatial/split/SplitStrategy.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/SplitStrategy.java rename to processing/src/main/java/io/druid/collections/spatial/split/SplitStrategy.java diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java index ce9dbbf2727b..3ad6574a08dc 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -229,12 +229,12 @@ public ImmutableBitmap getBitmapIndex(String dimension, String value) public ImmutableRTree getSpatialIndex(String dimension) { if (isVirtualColumn(dimension)) { - return new ImmutableRTree(); + return ImmutableRTree.empty(); } final Column column = index.getColumn(dimension); if (column == null || !column.getCapabilities().hasSpatialIndexes()) { - return new ImmutableRTree(); + return ImmutableRTree.empty(); } return column.getSpatialIndex().getRTree(); diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java index 2bbfa9ef6568..136658bf2a62 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -498,7 +498,9 @@ private static GenericIndexed fromIterableVersionOne( // for compatibility with the format, but this field is unused valuesOut.writeInt(0); - strategy.writeTo(next, valuesOut); + if (next != null) { + strategy.writeTo(next, valuesOut); + } headerOut.writeInt(Ints.checkedCast(valuesOut.size())); if (prevVal instanceof Closeable) { diff --git a/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java b/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java index 31822e13cc51..341026a1c579 100644 --- a/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java @@ -217,7 +217,9 @@ public void write(T objectToWrite) throws IOException // for compatibility with the format (see GenericIndexed javadoc for description of the format), but this field is // unused. valuesOut.writeInt(0); - strategy.writeTo(objectToWrite, valuesOut); + if (objectToWrite != null) { + strategy.writeTo(objectToWrite, valuesOut); + } if (!requireMultipleFiles) { headerOut.writeInt(Ints.checkedCast(valuesOut.size())); diff --git a/processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java index 14f38f4ef49e..6e49c8626b7a 100644 --- a/processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/ImmutableRTreeObjectStrategy.java @@ -22,8 +22,9 @@ import com.google.common.collect.Ordering; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.spatial.ImmutableRTree; -import it.unimi.dsi.fastutil.bytes.ByteArrays; +import io.druid.segment.writeout.WriteOutBytes; +import java.io.IOException; import java.nio.ByteBuffer; public class ImmutableRTreeObjectStrategy implements ObjectStrategy @@ -71,12 +72,15 @@ public ImmutableRTree fromByteBuffer(ByteBuffer buffer, int numBytes) @Override public byte[] toBytes(ImmutableRTree val) { - if (val == null || val.size() == 0) { - return ByteArrays.EMPTY_ARRAY; - } return val.toBytes(); } + @Override + public void writeTo(ImmutableRTree val, WriteOutBytes out) throws IOException + { + val.writeTo(out); + } + @Override public int compare(ImmutableRTree o1, ImmutableRTree o2) { diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java index dc3c6647f3a1..8698b36e1f53 100644 --- a/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java @@ -25,7 +25,9 @@ import io.druid.segment.GenericColumnSerializer; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.data.ObjectStrategy; +import it.unimi.dsi.fastutil.bytes.ByteArrays; +import javax.annotation.Nullable; import java.nio.ByteBuffer; /** @@ -80,9 +82,9 @@ public Function inputSizeFn() * * @return serialized intermediate representation of aggregate in byte[] */ - public byte[] toBytes(Object val) + public byte[] toBytes(@Nullable Object val) { - return getObjectStrategy().toBytes(val); + return val != null ? getObjectStrategy().toBytes(val) : ByteArrays.EMPTY_ARRAY; } /** diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/IntSetTestUtility.java b/processing/src/test/java/io/druid/collections/IntSetTestUtility.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/IntSetTestUtility.java rename to processing/src/test/java/io/druid/collections/IntSetTestUtility.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/TestIntegerSet.java b/processing/src/test/java/io/druid/collections/TestIntegerSet.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/TestIntegerSet.java rename to processing/src/test/java/io/druid/collections/TestIntegerSet.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java b/processing/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java old mode 100755 new mode 100644 similarity index 99% rename from bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java rename to processing/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java index 93ee6eb9c909..26a19fe8e230 --- a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java +++ b/processing/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java @@ -39,7 +39,9 @@ import java.util.Locale; import java.util.Random; - +/** + * TODO rewrite this benchmark to JMH + */ @BenchmarkOptions(clock = Clock.NANO_TIME, benchmarkRounds = 50) public class BitmapBenchmark { diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapIterationTest.java b/processing/src/test/java/io/druid/collections/bitmap/BitmapIterationTest.java similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapIterationTest.java rename to processing/src/test/java/io/druid/collections/bitmap/BitmapIterationTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/ConciseBitmapFactoryTest.java b/processing/src/test/java/io/druid/collections/bitmap/ConciseBitmapFactoryTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/bitmap/ConciseBitmapFactoryTest.java rename to processing/src/test/java/io/druid/collections/bitmap/ConciseBitmapFactoryTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/RangeBitmapBenchmarkTest.java b/processing/src/test/java/io/druid/collections/bitmap/RangeBitmapBenchmarkTest.java old mode 100755 new mode 100644 similarity index 96% rename from bytebuffer-collections/src/test/java/io/druid/collections/bitmap/RangeBitmapBenchmarkTest.java rename to processing/src/test/java/io/druid/collections/bitmap/RangeBitmapBenchmarkTest.java index 4c5d3769c9ab..74999ef80469 --- a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/RangeBitmapBenchmarkTest.java +++ b/processing/src/test/java/io/druid/collections/bitmap/RangeBitmapBenchmarkTest.java @@ -24,13 +24,16 @@ import io.druid.extendedset.intset.ConciseSet; import io.druid.extendedset.intset.ImmutableConciseSet; import io.druid.java.util.common.StringUtils; -import io.druid.test.annotation.Benchmark; +import io.druid.collections.test.annotation.Benchmark; import org.junit.BeforeClass; import org.junit.experimental.categories.Category; import org.roaringbitmap.buffer.MutableRoaringBitmap; import java.util.BitSet; +/** + * TODO rewrite this benchmark to JMH + */ @Category({Benchmark.class}) @BenchmarkHistoryChart(labelWith = LabelType.CUSTOM_KEY, maxRuns = 20) public class RangeBitmapBenchmarkTest extends BitmapBenchmark diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/RoaringBitmapFactoryTest.java b/processing/src/test/java/io/druid/collections/bitmap/RoaringBitmapFactoryTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/bitmap/RoaringBitmapFactoryTest.java rename to processing/src/test/java/io/druid/collections/bitmap/RoaringBitmapFactoryTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/UniformBitmapBenchmarkTest.java b/processing/src/test/java/io/druid/collections/bitmap/UniformBitmapBenchmarkTest.java old mode 100755 new mode 100644 similarity index 96% rename from bytebuffer-collections/src/test/java/io/druid/collections/bitmap/UniformBitmapBenchmarkTest.java rename to processing/src/test/java/io/druid/collections/bitmap/UniformBitmapBenchmarkTest.java index 6a0438178891..eb6039528d92 --- a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/UniformBitmapBenchmarkTest.java +++ b/processing/src/test/java/io/druid/collections/bitmap/UniformBitmapBenchmarkTest.java @@ -24,13 +24,16 @@ import io.druid.extendedset.intset.ConciseSet; import io.druid.extendedset.intset.ImmutableConciseSet; import io.druid.java.util.common.StringUtils; -import io.druid.test.annotation.Benchmark; +import io.druid.collections.test.annotation.Benchmark; import org.junit.BeforeClass; import org.junit.experimental.categories.Category; import org.roaringbitmap.buffer.MutableRoaringBitmap; import java.util.BitSet; +/** + * TODO rewrite this benchmark to JMH + */ @Category({Benchmark.class}) @BenchmarkHistoryChart(labelWith = LabelType.CUSTOM_KEY, maxRuns = 20) public class UniformBitmapBenchmarkTest extends BitmapBenchmark diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/WrappedBitSetBitmapBitSetTest.java b/processing/src/test/java/io/druid/collections/bitmap/WrappedBitSetBitmapBitSetTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/bitmap/WrappedBitSetBitmapBitSetTest.java rename to processing/src/test/java/io/druid/collections/bitmap/WrappedBitSetBitmapBitSetTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/WrappedRoaringBitmapTest.java b/processing/src/test/java/io/druid/collections/bitmap/WrappedRoaringBitmapTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/bitmap/WrappedRoaringBitmapTest.java rename to processing/src/test/java/io/druid/collections/bitmap/WrappedRoaringBitmapTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java b/processing/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java old mode 100755 new mode 100644 similarity index 95% rename from bytebuffer-collections/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java rename to processing/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java index 3c3bb9c9eba4..20e9d4d76be1 --- a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java +++ b/processing/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java @@ -23,6 +23,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; +import com.google.common.primitives.Bytes; import io.druid.collections.bitmap.BitmapFactory; import io.druid.collections.bitmap.ConciseBitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; @@ -31,11 +32,14 @@ import io.druid.collections.spatial.search.RadiusBound; import io.druid.collections.spatial.search.RectangularBound; import io.druid.collections.spatial.split.LinearGutmanSplitStrategy; +import io.druid.segment.data.GenericIndexed; +import io.druid.segment.data.ImmutableRTreeObjectStrategy; import junit.framework.Assert; import org.junit.Test; import org.roaringbitmap.IntIterator; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Locale; import java.util.Random; import java.util.Set; @@ -648,4 +652,24 @@ public void showBenchmarksBoundWithLimits() } } } + + @Test + public void testToBytes() + { + BitmapFactory bf = new RoaringBitmapFactory(); + ImmutableRTreeObjectStrategy rTreeObjectStrategy = new ImmutableRTreeObjectStrategy(bf); + RTree rTree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + rTree.insert(new float[]{0, 0}, 1); + ImmutableRTree immutableRTree = ImmutableRTree.newImmutableFromMutable(rTree); + byte[] bytes1 = immutableRTree.toBytes(); + + GenericIndexed genericIndexed = GenericIndexed.fromIterable( + Arrays.asList(immutableRTree, immutableRTree), + rTreeObjectStrategy + ); + + ImmutableRTree deserializedTree = genericIndexed.get(0); + byte[] bytes2 = deserializedTree.toBytes(); + org.junit.Assert.assertEquals(Bytes.asList(bytes1), Bytes.asList(bytes2)); + } } diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/RTreeTest.java b/processing/src/test/java/io/druid/collections/spatial/RTreeTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/spatial/RTreeTest.java rename to processing/src/test/java/io/druid/collections/spatial/RTreeTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/PolygonBoundTest.java b/processing/src/test/java/io/druid/collections/spatial/search/PolygonBoundTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/PolygonBoundTest.java rename to processing/src/test/java/io/druid/collections/spatial/search/PolygonBoundTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/RadiusBoundTest.java b/processing/src/test/java/io/druid/collections/spatial/search/RadiusBoundTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/RadiusBoundTest.java rename to processing/src/test/java/io/druid/collections/spatial/search/RadiusBoundTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/RectangularBoundTest.java b/processing/src/test/java/io/druid/collections/spatial/search/RectangularBoundTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/RectangularBoundTest.java rename to processing/src/test/java/io/druid/collections/spatial/search/RectangularBoundTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategyTest.java b/processing/src/test/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategyTest.java old mode 100755 new mode 100644 similarity index 100% rename from bytebuffer-collections/src/test/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategyTest.java rename to processing/src/test/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategyTest.java diff --git a/bytebuffer-collections/src/test/java/io/druid/test/annotation/Benchmark.java b/processing/src/test/java/io/druid/collections/test/annotation/Benchmark.java old mode 100755 new mode 100644 similarity index 94% rename from bytebuffer-collections/src/test/java/io/druid/test/annotation/Benchmark.java rename to processing/src/test/java/io/druid/collections/test/annotation/Benchmark.java index 2fa616eef401..99aa5517a48c --- a/bytebuffer-collections/src/test/java/io/druid/test/annotation/Benchmark.java +++ b/processing/src/test/java/io/druid/collections/test/annotation/Benchmark.java @@ -17,7 +17,7 @@ * under the License. */ -package io.druid.test.annotation; +package io.druid.collections.test.annotation; public interface Benchmark { diff --git a/bytebuffer-collections/src/test/java/io/druid/test/annotation/Dummy.java b/processing/src/test/java/io/druid/collections/test/annotation/Dummy.java old mode 100755 new mode 100644 similarity index 94% rename from bytebuffer-collections/src/test/java/io/druid/test/annotation/Dummy.java rename to processing/src/test/java/io/druid/collections/test/annotation/Dummy.java index ae1d36869320..75b0fa2925c8 --- a/bytebuffer-collections/src/test/java/io/druid/test/annotation/Dummy.java +++ b/processing/src/test/java/io/druid/collections/test/annotation/Dummy.java @@ -17,7 +17,7 @@ * under the License. */ -package io.druid.test.annotation; +package io.druid.collections.test.annotation; public interface Dummy { diff --git a/server/src/test/java/io/druid/client/cache/CacheConfigTest.java b/server/src/test/java/io/druid/client/cache/CacheConfigTest.java index 096887d81b52..7643a618405a 100644 --- a/server/src/test/java/io/druid/client/cache/CacheConfigTest.java +++ b/server/src/test/java/io/druid/client/cache/CacheConfigTest.java @@ -45,7 +45,7 @@ public class CacheConfigTest static Injector injector; static JsonConfigurator configurator; JsonConfigProvider configProvider; - private static final String propertyPrefix = "io.druid.test.cache"; + private static final String propertyPrefix = "io.druid.collections.test.cache"; @BeforeClass public static void populateStatics() From 3f53820fe06808e33cd19bf944b18d2ef7807dad Mon Sep 17 00:00:00 2001 From: Nishant Bangarwa Date: Wed, 24 Jan 2018 04:29:39 +0530 Subject: [PATCH 15/82] Fix rewrite of queryPath for encoded joda intervals as query param on druid router (#5274) (#5285) * Fix rewrite of queryPath for encoded joda intervals as query param on druid router * fix checkstyle * fix comment --- .../server/AsyncQueryForwardingServlet.java | 19 +++++++++---------- .../AsyncQueryForwardingServletTest.java | 15 ++++++++++++++- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java index b6c5f287235f..f864a062740b 100644 --- a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java +++ b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java @@ -45,6 +45,7 @@ import io.druid.server.router.Router; import io.druid.server.security.AuthConfig; import io.druid.server.security.Escalator; +import org.apache.http.client.utils.URIBuilder; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.api.Request; import org.eclipse.jetty.client.api.Response; @@ -58,10 +59,8 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.core.MediaType; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URISyntaxException; -import java.net.URLDecoder; import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -331,15 +330,15 @@ protected URI rewriteURI(HttpServletRequest request, String scheme, String host) protected static URI makeURI(String scheme, String host, String requestURI, String rawQueryString) { try { - return new URI( - scheme, - host, - requestURI, - rawQueryString == null ? null : URLDecoder.decode(rawQueryString, "UTF-8"), - null - ); + return new URIBuilder() + .setScheme(scheme) + .setHost(host) + .setPath(requestURI) + // No need to encode-decode queryString, it is already encoded + .setQuery(rawQueryString) + .build(); } - catch (UnsupportedEncodingException | URISyntaxException e) { + catch (URISyntaxException e) { log.error(e, "Unable to rewrite URI [%s]", e.getMessage()); throw Throwables.propagate(e); } diff --git a/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java b/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java index c1c0d3b2a4ed..9fcf4736fed1 100644 --- a/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java +++ b/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java @@ -52,9 +52,9 @@ import io.druid.server.router.QueryHostFinder; import io.druid.server.router.RendezvousHashAvaticaConnectionBalancer; import io.druid.server.security.AllowAllAuthorizer; -import io.druid.server.security.NoopEscalator; import io.druid.server.security.Authorizer; import io.druid.server.security.AuthorizerMapper; +import io.druid.server.security.NoopEscalator; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; @@ -308,6 +308,19 @@ public void testRewriteURI() throws Exception new URI("http://localhost/"), AsyncQueryForwardingServlet.makeURI("http", "localhost", "/", null) ); + + // Test reWrite Encoded interval with timezone info + // decoded parameters 1900-01-01T00:00:00.000+01.00 -> 1900-01-01T00:00:00.000+01:00 + Assert.assertEquals( + new URI( + "http://localhost:1234/some/path?intervals=1900-01-01T00%3A00%3A00.000%2B01%3A00%2F3000-01-01T00%3A00%3A00.000%2B01%3A00"), + AsyncQueryForwardingServlet.makeURI( + "http", + "localhost:1234", + "/some/path", + "intervals=1900-01-01T00%3A00%3A00.000%2B01%3A00%2F3000-01-01T00%3A00%3A00.000%2B01%3A00" + ) + ); } private static class TestServer implements io.druid.client.selector.Server From cdcd7e31dbd8efc88ce3bea75fa6da4018e17f58 Mon Sep 17 00:00:00 2001 From: David Lim Date: Tue, 30 Jan 2018 13:30:33 -0700 Subject: [PATCH 16/82] clean up intermediate_pushes directory for LocalDataSegmentPusher (#5306) (#5307) --- .../io/druid/segment/loading/LocalDataSegmentPusher.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java index 0c00e0aeb334..f60228dfeb3e 100644 --- a/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java +++ b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java @@ -73,9 +73,8 @@ public String getPathForHadoop(String dataSource) @Override public DataSegment push(File dataSegmentFile, DataSegment segment, boolean replaceExisting) throws IOException { - final String storageDir = this.getStorageDir(segment); final File baseStorageDir = config.getStorageDirectory(); - final File outDir = new File(baseStorageDir, storageDir); + final File outDir = new File(baseStorageDir, this.getStorageDir(segment)); log.info("Copying segment[%s] to local filesystem at location[%s]", segment.getIdentifier(), outDir.toString()); @@ -93,7 +92,7 @@ public DataSegment push(File dataSegmentFile, DataSegment segment, boolean repla ); } - final File tmpOutDir = new File(baseStorageDir, intermediateDirFor(storageDir)); + final File tmpOutDir = new File(baseStorageDir, makeIntermediateDir()); log.info("Creating intermediate directory[%s] for segment[%s]", tmpOutDir.toString(), segment.getIdentifier()); FileUtils.forceMkdir(tmpOutDir); @@ -150,9 +149,9 @@ public Map makeLoadSpec(URI finalIndexZipFilePath) return ImmutableMap.of("type", "local", "path", finalIndexZipFilePath.getPath()); } - private String intermediateDirFor(String storageDir) + private String makeIntermediateDir() { - return "intermediate_pushes/" + storageDir + "." + UUID.randomUUID().toString(); + return "intermediate_pushes/" + UUID.randomUUID().toString(); } private long compressSegment(File dataSegmentFile, File dest) throws IOException From 00f78ee1d3256719851939dd4bff5f59629b7a34 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Fri, 2 Feb 2018 06:01:40 -0800 Subject: [PATCH 17/82] [Backport] fix RemoteTaskRunner terminating lazy workers below autoscaler minNumWorkers value (#5324) * fix RemoteTaskRunner terminating lazy workers below autoscaler minNumWorkers value * add comment --- .../indexing/overlord/RemoteTaskRunner.java | 5 +++++ .../overlord/RemoteTaskRunnerTest.java | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java b/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java index c3e26bd5fac4..c04300571d49 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java @@ -1229,6 +1229,11 @@ private void taskComplete( @Override public Collection markWorkersLazy(Predicate isLazyWorker, int maxWorkers) { + // skip the lock and bail early if we should not mark any workers lazy (e.g. number + // of current workers is at or below the minNumWorkers of autoscaler config) + if (maxWorkers < 1) { + return Collections.emptyList(); + } // status lock is used to prevent any tasks being assigned to the worker while we mark it lazy synchronized (statusLock) { Iterator iterator = zkWorkers.keySet().iterator(); diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java index c2a199f872d3..996023f36a47 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java @@ -550,6 +550,24 @@ public boolean apply(ImmutableWorkerInfo input) Assert.assertEquals(1, remoteTaskRunner.getLazyWorkers().size()); } + @Test + public void testFindLazyWorkerNotRunningAnyTaskButWithZeroMaxWorkers() throws Exception + { + doSetup(); + Collection lazyworkers = remoteTaskRunner.markWorkersLazy( + new Predicate() + { + @Override + public boolean apply(ImmutableWorkerInfo input) + { + return true; + } + }, 0 + ); + Assert.assertEquals(0, lazyworkers.size()); + Assert.assertEquals(0, remoteTaskRunner.getLazyWorkers().size()); + } + @Test public void testWorkerZKReconnect() throws Exception { From b5e11087793d90ac37dc2563178eae0ab93ddfea Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Mon, 5 Feb 2018 15:01:38 -0800 Subject: [PATCH 18/82] Remove Escalator jetty http client escalation method (#5322) (#5348) --- .../authentication/BasicHTTPEscalator.java | 52 -------------- .../security/kerberos/KerberosEscalator.java | 72 ------------------- .../server/AsyncQueryForwardingServlet.java | 17 +++-- .../AuthorizerMapperModule.java | 11 ++- .../io/druid/server/security/Escalator.java | 12 ---- .../druid/server/security/NoopEscalator.java | 6 -- .../AsyncQueryForwardingServletTest.java | 4 +- 7 files changed, 19 insertions(+), 155 deletions(-) diff --git a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BasicHTTPEscalator.java b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BasicHTTPEscalator.java index 2eb73a23bf20..4029537f5499 100644 --- a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BasicHTTPEscalator.java +++ b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BasicHTTPEscalator.java @@ -22,21 +22,11 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.base.Throwables; import com.metamx.http.client.CredentialedHttpClient; import com.metamx.http.client.HttpClient; import com.metamx.http.client.auth.BasicCredentials; -import io.druid.java.util.common.StringUtils; -import io.druid.security.basic.BasicAuthUtils; import io.druid.server.security.AuthenticationResult; import io.druid.server.security.Escalator; -import org.eclipse.jetty.client.api.Authentication; -import org.eclipse.jetty.client.api.ContentResponse; -import org.eclipse.jetty.client.api.Request; -import org.eclipse.jetty.util.Attributes; -import org.jboss.netty.handler.codec.http.HttpHeaders; - -import java.net.URI; @JsonTypeName("basic") public class BasicHTTPEscalator implements Escalator @@ -66,48 +56,6 @@ public HttpClient createEscalatedClient(HttpClient baseClient) ); } - @Override - public org.eclipse.jetty.client.HttpClient createEscalatedJettyClient(org.eclipse.jetty.client.HttpClient baseClient) - { - baseClient.getAuthenticationStore().addAuthentication(new Authentication() - { - @Override - public boolean matches(String type, URI uri, String realm) - { - return true; - } - - @Override - public Result authenticate( - final Request request, ContentResponse response, Authentication.HeaderInfo headerInfo, Attributes context - ) - { - return new Result() - { - @Override - public URI getURI() - { - return request.getURI(); - } - - @Override - public void apply(Request request) - { - try { - final String unencodedCreds = StringUtils.format("%s:%s", internalClientUsername, internalClientPassword); - final String base64Creds = BasicAuthUtils.getEncodedCredentials(unencodedCreds); - request.getHeaders().add(HttpHeaders.Names.AUTHORIZATION, "Basic " + base64Creds); - } - catch (Throwable e) { - Throwables.propagate(e); - } - } - }; - } - }); - return baseClient; - } - @Override public AuthenticationResult createEscalatedAuthenticationResult() { diff --git a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosEscalator.java b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosEscalator.java index e995233a23c2..887b5296d04e 100644 --- a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosEscalator.java +++ b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosEscalator.java @@ -22,20 +22,10 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.base.Throwables; import com.metamx.http.client.HttpClient; import io.druid.java.util.common.logger.Logger; import io.druid.server.security.AuthenticationResult; import io.druid.server.security.Escalator; -import org.apache.hadoop.security.UserGroupInformation; -import org.eclipse.jetty.client.api.Authentication; -import org.eclipse.jetty.client.api.ContentResponse; -import org.eclipse.jetty.client.api.Request; -import org.eclipse.jetty.util.Attributes; -import org.jboss.netty.handler.codec.http.HttpHeaders; - -import java.net.URI; -import java.security.PrivilegedExceptionAction; @JsonTypeName("kerberos") public class KerberosEscalator implements Escalator @@ -64,68 +54,6 @@ public HttpClient createEscalatedClient(HttpClient baseClient) return new KerberosHttpClient(baseClient, internalClientPrincipal, internalClientKeytab); } - @Override - public org.eclipse.jetty.client.HttpClient createEscalatedJettyClient(org.eclipse.jetty.client.HttpClient baseClient) - { - baseClient.getAuthenticationStore().addAuthentication(new Authentication() - { - @Override - public boolean matches(String type, URI uri, String realm) - { - return true; - } - - @Override - public Result authenticate( - final Request request, ContentResponse response, Authentication.HeaderInfo headerInfo, Attributes context - ) - { - return new Result() - { - @Override - public URI getURI() - { - return request.getURI(); - } - - @Override - public void apply(Request request) - { - try { - // No need to set cookies as they are handled by Jetty Http Client itself. - URI uri = request.getURI(); - if (DruidKerberosUtil.needToSendCredentials(baseClient.getCookieStore(), uri)) { - log.debug( - "No Auth Cookie found for URI[%s]. Existing Cookies[%s] Authenticating... ", - uri, - baseClient.getCookieStore().getCookies() - ); - final String host = request.getHost(); - DruidKerberosUtil.authenticateIfRequired(internalClientPrincipal, internalClientKeytab); - UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); - String challenge = currentUser.doAs(new PrivilegedExceptionAction() - { - @Override - public String run() throws Exception - { - return DruidKerberosUtil.kerberosChallenge(host); - } - }); - request.getHeaders().add(HttpHeaders.Names.AUTHORIZATION, "Negotiate " + challenge); - } else { - log.debug("Found Auth Cookie found for URI[%s].", uri); - } - } - catch (Throwable e) { - Throwables.propagate(e); - } - } - }; - } - }); - return baseClient; - } - @Override public AuthenticationResult createEscalatedAuthenticationResult() { diff --git a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java index f864a062740b..ebe1b13c67f3 100644 --- a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java +++ b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java @@ -44,7 +44,6 @@ import io.druid.server.router.QueryHostFinder; import io.druid.server.router.Router; import io.druid.server.security.AuthConfig; -import io.druid.server.security.Escalator; import org.apache.http.client.utils.URIBuilder; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.api.Request; @@ -112,7 +111,6 @@ private static void handleException(HttpServletResponse response, ObjectMapper o private final ServiceEmitter emitter; private final RequestLogger requestLogger; private final GenericQueryMetricsFactory queryMetricsFactory; - private final Escalator escalator; private HttpClient broadcastClient; @@ -126,8 +124,7 @@ public AsyncQueryForwardingServlet( @Router DruidHttpClientConfig httpClientConfig, ServiceEmitter emitter, RequestLogger requestLogger, - GenericQueryMetricsFactory queryMetricsFactory, - Escalator escalator + GenericQueryMetricsFactory queryMetricsFactory ) { this.warehouse = warehouse; @@ -139,7 +136,6 @@ public AsyncQueryForwardingServlet( this.emitter = emitter; this.requestLogger = requestLogger; this.queryMetricsFactory = queryMetricsFactory; - this.escalator = escalator; } @Override @@ -213,11 +209,14 @@ protected void service(HttpServletRequest request, HttpServletResponse response) ); } }; - broadcastClient + + Request broadcastReq = broadcastClient .newRequest(rewriteURI(request, server.getScheme(), server.getHost())) .method(HttpMethod.DELETE) - .timeout(CANCELLATION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS) - .send(completeListener); + .timeout(CANCELLATION_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + + copyRequestHeaders(request, broadcastReq); + broadcastReq.send(completeListener); } interruptedQueryCount.incrementAndGet(); } @@ -347,7 +346,7 @@ protected static URI makeURI(String scheme, String host, String requestURI, Stri @Override protected HttpClient newHttpClient() { - return escalator.createEscalatedJettyClient(httpClientProvider.get()); + return httpClientProvider.get(); } @Override diff --git a/server/src/main/java/io/druid/server/initialization/AuthorizerMapperModule.java b/server/src/main/java/io/druid/server/initialization/AuthorizerMapperModule.java index 4ba019550833..7a5b17a484b2 100644 --- a/server/src/main/java/io/druid/server/initialization/AuthorizerMapperModule.java +++ b/server/src/main/java/io/druid/server/initialization/AuthorizerMapperModule.java @@ -95,11 +95,20 @@ public AuthorizerMapper get() // Default is allow all if (authorizers == null) { + AllowAllAuthorizer allowAllAuthorizer = new AllowAllAuthorizer(); + authorizerMap.put(AuthConfig.ALLOW_ALL_NAME, allowAllAuthorizer); + return new AuthorizerMapper(null) { @Override public Authorizer getAuthorizer(String name) { - return new AllowAllAuthorizer(); + return allowAllAuthorizer; + } + + @Override + public Map getAuthorizerMap() + { + return authorizerMap; } }; } diff --git a/server/src/main/java/io/druid/server/security/Escalator.java b/server/src/main/java/io/druid/server/security/Escalator.java index c3a9c19aa7d5..67b64363cf38 100644 --- a/server/src/main/java/io/druid/server/security/Escalator.java +++ b/server/src/main/java/io/druid/server/security/Escalator.java @@ -46,18 +46,6 @@ public interface Escalator */ HttpClient createEscalatedClient(HttpClient baseClient); - /** - * Return a client that sends requests with the format/information necessary to authenticate successfully - * against this Authenticator's authentication scheme using the identity of the internal system user. - *

- * This HTTP client is used by the Druid Router node. - * - * @param baseClient Base Jetty HttpClient - * - * @return Jetty HttpClient that sends requests with the credentials of the internal system user - */ - org.eclipse.jetty.client.HttpClient createEscalatedJettyClient(org.eclipse.jetty.client.HttpClient baseClient); - /** * @return an AuthenticationResult representing the identity of the internal system user. */ diff --git a/server/src/main/java/io/druid/server/security/NoopEscalator.java b/server/src/main/java/io/druid/server/security/NoopEscalator.java index 583d12dc249a..970c8eb07d70 100644 --- a/server/src/main/java/io/druid/server/security/NoopEscalator.java +++ b/server/src/main/java/io/druid/server/security/NoopEscalator.java @@ -29,12 +29,6 @@ public HttpClient createEscalatedClient(HttpClient baseClient) return baseClient; } - @Override - public org.eclipse.jetty.client.HttpClient createEscalatedJettyClient(org.eclipse.jetty.client.HttpClient baseClient) - { - return baseClient; - } - @Override public AuthenticationResult createEscalatedAuthenticationResult() { diff --git a/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java b/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java index 9fcf4736fed1..4118e7ae4f98 100644 --- a/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java +++ b/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java @@ -54,7 +54,6 @@ import io.druid.server.security.AllowAllAuthorizer; import io.druid.server.security.Authorizer; import io.druid.server.security.AuthorizerMapper; -import io.druid.server.security.NoopEscalator; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; @@ -252,8 +251,7 @@ public void log(RequestLogLine requestLogLine) throws IOException // noop } }, - new DefaultGenericQueryMetricsFactory(jsonMapper), - new NoopEscalator() + new DefaultGenericQueryMetricsFactory(jsonMapper) ) { @Override From 9a2a182588d328c958972d3adf15dff74d217fca Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 6 Feb 2018 12:08:11 -0800 Subject: [PATCH 19/82] Fix CompactionTask doc (#5354) --- docs/content/ingestion/tasks.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/content/ingestion/tasks.md b/docs/content/ingestion/tasks.md index b5b1bc1c543b..4d424efcadb4 100644 --- a/docs/content/ingestion/tasks.md +++ b/docs/content/ingestion/tasks.md @@ -276,8 +276,10 @@ An example of compaction task is } ``` -This compaction task merges _all segments_ of the interval `2017-01-01/2018-01-01` into a _single segment_. -To merge each day's worth of data into a separate segment, you can submit multiple `compact` tasks, one for each day. They will run in parallel. +This compaction task reads _all segments_ of the interval `2017-01-01/2018-01-01` and results in new segments. +Note that intervals of the input segments are merged into a single interval of `2017-01-01/2018-01-01` no matter what the segmentGranularity was. +To controll the number of result segments, you can set `targetPartitionSize` or `numShards`. See [indexTuningConfig](#tuningconfig) for more details. +To merge each day's worth of data into separate segments, you can submit multiple `compact` tasks, one for each day. They will run in parallel. A compaction task internally generates an `index` task spec for performing compaction work with some fixed parameters. For example, its `firehose` is always the [ingestSegmentSpec](./firehose.html), and `dimensionsSpec` and `metricsSpec` From 1d251d47f1bddb43d77b4178978811fc2d1fb527 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 6 Feb 2018 16:09:10 -0800 Subject: [PATCH 20/82] More ParseException handling for numeric dimensions (#5312) (#5356) * Discard rows with unparseable numeric dimensions * PR comments * Don't throw away entire row on parse exception * PR comments * Fix import --- .../io/druid/indexer/IndexGeneratorJob.java | 48 ++++- .../java/io/druid/indexer/InputRowSerde.java | 203 +++++++++++++++++- .../indexer/IndexGeneratorCombinerTest.java | 38 +++- .../io/druid/indexer/InputRowSerdeTest.java | 113 ++++++++-- .../druid/segment/DimensionHandlerUtils.java | 36 +++- .../io/druid/segment/DimensionIndexer.java | 3 +- .../druid/segment/DoubleDimensionIndexer.java | 6 +- .../druid/segment/FloatDimensionIndexer.java | 6 +- .../druid/segment/LongDimensionIndexer.java | 6 +- .../druid/segment/StringDimensionIndexer.java | 2 +- .../segment/incremental/IncrementalIndex.java | 5 +- .../incremental/IncrementalIndexAdapter.java | 2 +- .../incremental/IncrementalIndexTest.java | 58 +++-- 13 files changed, 461 insertions(+), 65 deletions(-) diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java index c9111d00cd35..a9726bb2db16 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/IndexGeneratorJob.java @@ -64,6 +64,7 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.InvalidJobConfException; import org.apache.hadoop.mapreduce.Counter; +import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.Partitioner; @@ -210,9 +211,17 @@ public boolean run() boolean success = job.waitForCompletion(true); - Counter invalidRowCount = job.getCounters() - .findCounter(HadoopDruidIndexerConfig.IndexJobCounters.INVALID_ROW_COUNTER); - jobStats.setInvalidRowCount(invalidRowCount.getValue()); + Counters counters = job.getCounters(); + if (counters == null) { + log.info("No counters found for job [%s]", job.getJobName()); + } else { + Counter invalidRowCount = counters.findCounter(HadoopDruidIndexerConfig.IndexJobCounters.INVALID_ROW_COUNTER); + if (invalidRowCount != null) { + jobStats.setInvalidRowCount(invalidRowCount.getValue()); + } else { + log.info("No invalid row counter found for job [%s]", job.getJobName()); + } + } return success; } @@ -258,6 +267,7 @@ public static class IndexGeneratorMapper extends HadoopDruidIndexerMapper typeHelperMap; @Override protected void setup(Context context) @@ -269,6 +279,11 @@ protected void setup(Context context) for (int i = 0; i < aggregators.length; ++i) { combiningAggs[i] = aggregators[i].getCombiningFactory(); } + typeHelperMap = InputRowSerde.getTypeHelperMap(config.getSchema() + .getDataSchema() + .getParser() + .getParseSpec() + .getDimensionsSpec()); } @Override @@ -299,9 +314,9 @@ protected void innerMap( // and they contain the columns as they show up in the segment after ingestion, not what you would see in raw // data byte[] serializedInputRow = inputRow instanceof SegmentInputRow ? - InputRowSerde.toBytes(inputRow, combiningAggs, reportParseExceptions) + InputRowSerde.toBytes(typeHelperMap, inputRow, combiningAggs, reportParseExceptions) : - InputRowSerde.toBytes(inputRow, aggregators, reportParseExceptions); + InputRowSerde.toBytes(typeHelperMap, inputRow, aggregators, reportParseExceptions); context.write( new SortableBytes( @@ -322,6 +337,7 @@ public static class IndexGeneratorCombiner extends Reducer typeHelperMap; @Override protected void setup(Context context) @@ -334,6 +350,11 @@ protected void setup(Context context) for (int i = 0; i < aggregators.length; ++i) { combiningAggs[i] = aggregators[i].getCombiningFactory(); } + typeHelperMap = InputRowSerde.getTypeHelperMap(config.getSchema() + .getDataSchema() + .getParser() + .getParseSpec() + .getDimensionsSpec()); } @Override @@ -350,11 +371,11 @@ protected void reduce( SortableBytes keyBytes = SortableBytes.fromBytesWritable(key); Bucket bucket = Bucket.fromGroupKey(keyBytes.getGroupKey()).lhs; IncrementalIndex index = makeIncrementalIndex(bucket, combiningAggs, config, null, null); - index.add(InputRowSerde.fromBytes(first.getBytes(), aggregators)); + index.add(InputRowSerde.fromBytes(typeHelperMap, first.getBytes(), aggregators)); while (iter.hasNext()) { context.progress(); - InputRow value = InputRowSerde.fromBytes(iter.next().getBytes(), aggregators); + InputRow value = InputRowSerde.fromBytes(typeHelperMap, iter.next().getBytes(), aggregators); if (!index.canAppendRow()) { dimOrder.addAll(index.getDimensionOrder()); @@ -381,10 +402,13 @@ private void flushIndexToContextAndClose(BytesWritable key, IncrementalIndex ind context.progress(); Row row = rows.next(); InputRow inputRow = getInputRowFromRow(row, dimensions); + // reportParseExceptions is true as any unparseable data is already handled by the mapper. + byte[] serializedRow = InputRowSerde.toBytes(typeHelperMap, inputRow, combiningAggs, true); + context.write( key, - new BytesWritable(InputRowSerde.toBytes(inputRow, combiningAggs, true)) + new BytesWritable(serializedRow) ); } index.close(); @@ -479,6 +503,7 @@ public static class IndexGeneratorReducer extends Reducer metricNames = Lists.newArrayList(); private AggregatorFactory[] aggregators; private AggregatorFactory[] combiningAggs; + private Map typeHelperMap; protected ProgressIndicator makeProgressIndicator(final Context context) { @@ -530,6 +555,11 @@ protected void setup(Context context) metricNames.add(aggregators[i].getName()); combiningAggs[i] = aggregators[i].getCombiningFactory(); } + typeHelperMap = InputRowSerde.getTypeHelperMap(config.getSchema() + .getDataSchema() + .getParser() + .getParseSpec() + .getDimensionsSpec()); } @Override @@ -597,7 +627,7 @@ public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) for (final BytesWritable bw : values) { context.progress(); - final InputRow inputRow = index.formatRow(InputRowSerde.fromBytes(bw.getBytes(), aggregators)); + final InputRow inputRow = index.formatRow(InputRowSerde.fromBytes(typeHelperMap, bw.getBytes(), aggregators)); int numRows = index.add(inputRow); ++lineCount; diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java index 672f77133746..cd1dd531604a 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/InputRowSerde.java @@ -22,17 +22,24 @@ import com.google.common.base.Supplier; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.io.ByteArrayDataInput; import com.google.common.io.ByteArrayDataOutput; import com.google.common.io.ByteStreams; + import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; +import io.druid.data.input.Rows; +import io.druid.data.input.impl.DimensionSchema; +import io.druid.data.input.impl.DimensionsSpec; import io.druid.java.util.common.IAE; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; import io.druid.java.util.common.parsers.ParseException; import io.druid.query.aggregation.Aggregator; import io.druid.query.aggregation.AggregatorFactory; +import io.druid.segment.DimensionHandlerUtils; import io.druid.segment.VirtualColumns; +import io.druid.segment.column.ValueType; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.serde.ComplexMetricSerde; import io.druid.segment.serde.ComplexMetrics; @@ -49,7 +56,165 @@ public class InputRowSerde { private static final Logger log = new Logger(InputRowSerde.class); - public static final byte[] toBytes(final InputRow row, AggregatorFactory[] aggs, boolean reportParseExceptions) + private static final IndexSerdeTypeHelper STRING_HELPER = new StringIndexSerdeTypeHelper(); + private static final IndexSerdeTypeHelper LONG_HELPER = new LongIndexSerdeTypeHelper(); + private static final IndexSerdeTypeHelper FLOAT_HELPER = new FloatIndexSerdeTypeHelper(); + private static final IndexSerdeTypeHelper DOUBLE_HELPER = new DoubleIndexSerdeTypeHelper(); + + public interface IndexSerdeTypeHelper + { + ValueType getType(); + + void serialize(ByteArrayDataOutput out, Object value, boolean reportParseExceptions); + + T deserialize(ByteArrayDataInput in); + } + + public static Map getTypeHelperMap(DimensionsSpec dimensionsSpec) + { + Map typeHelperMap = Maps.newHashMap(); + for (DimensionSchema dimensionSchema : dimensionsSpec.getDimensions()) { + IndexSerdeTypeHelper typeHelper; + switch (dimensionSchema.getValueType()) { + case STRING: + typeHelper = STRING_HELPER; + break; + case LONG: + typeHelper = LONG_HELPER; + break; + case FLOAT: + typeHelper = FLOAT_HELPER; + break; + case DOUBLE: + typeHelper = DOUBLE_HELPER; + break; + default: + throw new IAE("Invalid type: [%s]", dimensionSchema.getValueType()); + } + typeHelperMap.put(dimensionSchema.getName(), typeHelper); + } + return typeHelperMap; + } + + public static class StringIndexSerdeTypeHelper implements IndexSerdeTypeHelper> + { + @Override + public ValueType getType() + { + return ValueType.STRING; + } + + @Override + public void serialize(ByteArrayDataOutput out, Object value, boolean reportParseExceptions) + { + List values = Rows.objectToStrings(value); + try { + writeStringArray(values, out); + } + catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + + @Override + public List deserialize(ByteArrayDataInput in) + { + try { + return readStringArray(in); + } + catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + } + + public static class LongIndexSerdeTypeHelper implements IndexSerdeTypeHelper + { + @Override + public ValueType getType() + { + return ValueType.LONG; + } + + @Override + public void serialize(ByteArrayDataOutput out, Object value, boolean reportParseExceptions) + { + Long ret = DimensionHandlerUtils.convertObjectToLong(value, reportParseExceptions); + if (ret == null) { + // remove null -> zero conversion when https://github.com/druid-io/druid/pull/5278 series of patches is merged + // we'll also need to change the serialized encoding so that it can represent numeric nulls + ret = DimensionHandlerUtils.ZERO_LONG; + } + out.writeLong(ret); + } + + @Override + public Long deserialize(ByteArrayDataInput in) + { + return in.readLong(); + } + } + + public static class FloatIndexSerdeTypeHelper implements IndexSerdeTypeHelper + { + @Override + public ValueType getType() + { + return ValueType.FLOAT; + } + + @Override + public void serialize(ByteArrayDataOutput out, Object value, boolean reportParseExceptions) + { + Float ret = DimensionHandlerUtils.convertObjectToFloat(value, reportParseExceptions); + if (ret == null) { + // remove null -> zero conversion when https://github.com/druid-io/druid/pull/5278 series of patches is merged + // we'll also need to change the serialized encoding so that it can represent numeric nulls + ret = DimensionHandlerUtils.ZERO_FLOAT; + } + out.writeFloat(ret); + } + + @Override + public Float deserialize(ByteArrayDataInput in) + { + return in.readFloat(); + } + } + + public static class DoubleIndexSerdeTypeHelper implements IndexSerdeTypeHelper + { + @Override + public ValueType getType() + { + return ValueType.DOUBLE; + } + + @Override + public void serialize(ByteArrayDataOutput out, Object value, boolean reportParseExceptions) + { + Double ret = DimensionHandlerUtils.convertObjectToDouble(value, reportParseExceptions); + if (ret == null) { + // remove null -> zero conversion when https://github.com/druid-io/druid/pull/5278 series of patches is merged + // we'll also need to change the serialized encoding so that it can represent numeric nulls + ret = DimensionHandlerUtils.ZERO_DOUBLE; + } + out.writeDouble(ret); + } + + @Override + public Double deserialize(ByteArrayDataInput in) + { + return in.readDouble(); + } + } + + public static final byte[] toBytes( + final Map typeHelperMap, + final InputRow row, + AggregatorFactory[] aggs, + boolean reportParseExceptions + ) { try { ByteArrayDataOutput out = ByteStreams.newDataOutput(); @@ -63,9 +228,12 @@ public static final byte[] toBytes(final InputRow row, AggregatorFactory[] aggs, WritableUtils.writeVInt(out, dimList.size()); if (dimList != null) { for (String dim : dimList) { - List dimValues = row.getDimension(dim); + IndexSerdeTypeHelper typeHelper = typeHelperMap.get(dim); + if (typeHelper == null) { + typeHelper = STRING_HELPER; + } writeString(dim, out); - writeStringArray(dimValues, out); + typeHelper.serialize(out, row.getRaw(dim), reportParseExceptions); } } @@ -176,10 +344,14 @@ private static List readStringArray(DataInput in) throws IOException return values; } - public static final InputRow fromBytes(byte[] data, AggregatorFactory[] aggs) + public static final InputRow fromBytes( + final Map typeHelperMap, + byte[] data, + AggregatorFactory[] aggs + ) { try { - DataInput in = ByteStreams.newDataInput(data); + ByteArrayDataInput in = ByteStreams.newDataInput(data); //Read timestamp long timestamp = in.readLong(); @@ -192,14 +364,25 @@ public static final InputRow fromBytes(byte[] data, AggregatorFactory[] aggs) for (int i = 0; i < dimNum; i++) { String dimension = readString(in); dimensions.add(dimension); - List dimensionValues = readStringArray(in); - if (dimensionValues == null) { + + IndexSerdeTypeHelper typeHelper = typeHelperMap.get(dimension); + if (typeHelper == null) { + typeHelper = STRING_HELPER; + } + Object dimValues = typeHelper.deserialize(in); + if (dimValues == null) { continue; } - if (dimensionValues.size() == 1) { - event.put(dimension, dimensionValues.get(0)); + + if (typeHelper.getType() == ValueType.STRING) { + List dimensionValues = (List) dimValues; + if (dimensionValues.size() == 1) { + event.put(dimension, dimensionValues.get(0)); + } else { + event.put(dimension, dimensionValues); + } } else { - event.put(dimension, dimensionValues); + event.put(dimension, dimValues); } } diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/IndexGeneratorCombinerTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/IndexGeneratorCombinerTest.java index 9eaf75349ed9..9eb75e27b779 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/IndexGeneratorCombinerTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/IndexGeneratorCombinerTest.java @@ -25,6 +25,7 @@ import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.impl.DimensionsSpec; +import io.druid.data.input.impl.StringDimensionSchema; import io.druid.data.input.impl.StringInputRowParser; import io.druid.data.input.impl.TimeAndDimsParseSpec; import io.druid.data.input.impl.TimestampSpec; @@ -144,6 +145,17 @@ public void testMultipleRowsMerged() throws Exception ); BytesWritable key = keySortableBytes.toBytesWritable(); + DimensionsSpec dimensionsSpec = new DimensionsSpec( + Arrays.asList( + new StringDimensionSchema("host"), + new StringDimensionSchema("keywords") + ), + null, + null + ); + + Map typeHelperMap = InputRowSerde.getTypeHelperMap(dimensionsSpec); + InputRow row1 = new MapBasedInputRow( timestamp, ImmutableList.of("keywords"), @@ -163,8 +175,8 @@ public void testMultipleRowsMerged() throws Exception ) ); List rows = Lists.newArrayList( - new BytesWritable(InputRowSerde.toBytes(row1, aggregators, true)), - new BytesWritable(InputRowSerde.toBytes(row2, aggregators, true)) + new BytesWritable(InputRowSerde.toBytes(typeHelperMap, row1, aggregators, true)), + new BytesWritable(InputRowSerde.toBytes(typeHelperMap, row2, aggregators, true)) ); Reducer.Context context = EasyMock.createNiceMock(Reducer.Context.class); @@ -183,7 +195,7 @@ public void testMultipleRowsMerged() throws Exception Assert.assertTrue(captureKey.getValue() == key); - InputRow capturedRow = InputRowSerde.fromBytes(captureVal.getValue().getBytes(), aggregators); + InputRow capturedRow = InputRowSerde.fromBytes(typeHelperMap, captureVal.getValue().getBytes(), aggregators); Assert.assertEquals(Arrays.asList("host", "keywords"), capturedRow.getDimensions()); Assert.assertEquals(ImmutableList.of(), capturedRow.getDimension("host")); Assert.assertEquals(Arrays.asList("bar", "foo"), capturedRow.getDimension("keywords")); @@ -228,9 +240,21 @@ public void testMultipleRowsNotMerged() throws Exception "visited", 5 ) ); + + DimensionsSpec dimensionsSpec = new DimensionsSpec( + Arrays.asList( + new StringDimensionSchema("host"), + new StringDimensionSchema("keywords") + ), + null, + null + ); + + Map typeHelperMap = InputRowSerde.getTypeHelperMap(dimensionsSpec); + List rows = Lists.newArrayList( - new BytesWritable(InputRowSerde.toBytes(row1, aggregators, true)), - new BytesWritable(InputRowSerde.toBytes(row2, aggregators, true)) + new BytesWritable(InputRowSerde.toBytes(typeHelperMap, row1, aggregators, true)), + new BytesWritable(InputRowSerde.toBytes(typeHelperMap, row2, aggregators, true)) ); Reducer.Context context = EasyMock.createNiceMock(Reducer.Context.class); @@ -253,7 +277,7 @@ public void testMultipleRowsNotMerged() throws Exception Assert.assertTrue(captureKey1.getValue() == key); Assert.assertTrue(captureKey2.getValue() == key); - InputRow capturedRow1 = InputRowSerde.fromBytes(captureVal1.getValue().getBytes(), aggregators); + InputRow capturedRow1 = InputRowSerde.fromBytes(typeHelperMap, captureVal1.getValue().getBytes(), aggregators); Assert.assertEquals(Arrays.asList("host", "keywords"), capturedRow1.getDimensions()); Assert.assertEquals(Collections.singletonList("host1"), capturedRow1.getDimension("host")); Assert.assertEquals(Arrays.asList("bar", "foo"), capturedRow1.getDimension("keywords")); @@ -264,7 +288,7 @@ public void testMultipleRowsNotMerged() throws Exception 0.001 ); - InputRow capturedRow2 = InputRowSerde.fromBytes(captureVal2.getValue().getBytes(), aggregators); + InputRow capturedRow2 = InputRowSerde.fromBytes(typeHelperMap, captureVal2.getValue().getBytes(), aggregators); Assert.assertEquals(Arrays.asList("host", "keywords"), capturedRow2.getDimensions()); Assert.assertEquals(Collections.singletonList("host2"), capturedRow2.getDimension("host")); Assert.assertEquals(Arrays.asList("bar", "foo"), capturedRow2.getDimension("keywords")); diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java index 67a9b5abf8a3..71609e42dd32 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/InputRowSerdeTest.java @@ -20,9 +20,14 @@ package io.druid.indexer; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; +import io.druid.data.input.impl.DimensionsSpec; +import io.druid.data.input.impl.DoubleDimensionSchema; +import io.druid.data.input.impl.FloatDimensionSchema; +import io.druid.data.input.impl.LongDimensionSchema; +import io.druid.data.input.impl.StringDimensionSchema; import io.druid.hll.HyperLogLogCollector; import io.druid.jackson.AggregatorsModule; import io.druid.java.util.common.parsers.ParseException; @@ -35,8 +40,11 @@ import io.druid.segment.ColumnSelectorFactory; import org.easymock.EasyMock; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -53,17 +61,22 @@ public class InputRowSerdeTest new AggregatorsModule(); //registers ComplexMetric serde for hyperUnique } + @Rule + public ExpectedException expectedException = ExpectedException.none(); + public InputRowSerdeTest() { this.timestamp = System.currentTimeMillis(); - this.dims = ImmutableList.of("dim_non_existing", "d1", "d2"); - this.event = ImmutableMap.of( - "d1", "d1v", - "d2", ImmutableList.of("d2v1", "d2v2"), - "m1", 5.0f, - "m2", 100L, - "m3", "m3v" - ); + this.dims = ImmutableList.of("dim_non_existing", "d1", "d2", "d3", "d4", "d5"); + this.event = Maps.newHashMap(); + event.put("d1", "d1v"); + event.put("d2", ImmutableList.of("d2v1", "d2v2")); + event.put("d3", 200L); + event.put("d4", 300.1f); + event.put("d5", 400.5d); + event.put("m1", 5.0f); + event.put("m2", 100L); + event.put("m3", "m3v"); } @Test @@ -99,14 +112,29 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) } }; - byte[] data = InputRowSerde.toBytes(in, aggregatorFactories, false); // Ignore Unparseable aggregator - InputRow out = InputRowSerde.fromBytes(data, aggregatorFactories); + DimensionsSpec dimensionsSpec = new DimensionsSpec( + Arrays.asList( + new StringDimensionSchema("d1"), + new StringDimensionSchema("d2"), + new LongDimensionSchema("d3"), + new FloatDimensionSchema("d4"), + new DoubleDimensionSchema("d5") + ), + null, + null + ); + + byte[] data = InputRowSerde.toBytes(InputRowSerde.getTypeHelperMap(dimensionsSpec), in, aggregatorFactories, false); // Ignore Unparseable aggregator + InputRow out = InputRowSerde.fromBytes(InputRowSerde.getTypeHelperMap(dimensionsSpec), data, aggregatorFactories); Assert.assertEquals(timestamp, out.getTimestampFromEpoch()); Assert.assertEquals(dims, out.getDimensions()); Assert.assertEquals(Collections.EMPTY_LIST, out.getDimension("dim_non_existing")); Assert.assertEquals(ImmutableList.of("d1v"), out.getDimension("d1")); Assert.assertEquals(ImmutableList.of("d2v1", "d2v2"), out.getDimension("d2")); + Assert.assertEquals(200L, out.getRaw("d3")); + Assert.assertEquals(300.1f, out.getRaw("d4")); + Assert.assertEquals(400.5d, out.getRaw("d5")); Assert.assertEquals(0.0f, out.getMetric("agg_non_existing").floatValue(), 0.00001); Assert.assertEquals(5.0f, out.getMetric("m1out").floatValue(), 0.00001); @@ -117,7 +145,7 @@ public Aggregator factorize(ColumnSelectorFactory metricFactory) EasyMock.verify(mockedAggregator); } - @Test(expected = ParseException.class) + @Test public void testThrowParseExceptions() { InputRow in = new MapBasedInputRow( @@ -133,7 +161,66 @@ public void testThrowParseExceptions() new LongSumAggregatorFactory("unparseable", "m3") // Unparseable from String to Long }; - InputRowSerde.toBytes(in, aggregatorFactories, true); + DimensionsSpec dimensionsSpec = new DimensionsSpec( + Arrays.asList( + new StringDimensionSchema("d1"), + new StringDimensionSchema("d2"), + new LongDimensionSchema("d3"), + new FloatDimensionSchema("d4"), + new DoubleDimensionSchema("d5") + ), + null, + null + ); + expectedException.expect(ParseException.class); + expectedException.expectMessage("Encountered parse error for aggregator[unparseable]"); + InputRowSerde.toBytes(InputRowSerde.getTypeHelperMap(dimensionsSpec), in, aggregatorFactories, true); + } + + @Test + public void testDimensionParseExceptions() + { + InputRow in = new MapBasedInputRow( + timestamp, + dims, + event + ); + AggregatorFactory[] aggregatorFactories = new AggregatorFactory[]{ + new LongSumAggregatorFactory("m2out", "m2") + }; + + expectedException.expect(ParseException.class); + expectedException.expectMessage("could not convert value [d1v] to long"); + DimensionsSpec dimensionsSpec = new DimensionsSpec( + Arrays.asList( + new LongDimensionSchema("d1") + ), + null, + null + ); + InputRowSerde.toBytes(InputRowSerde.getTypeHelperMap(dimensionsSpec), in, aggregatorFactories, true); + + expectedException.expect(ParseException.class); + expectedException.expectMessage("could not convert value [d1v] to float"); + dimensionsSpec = new DimensionsSpec( + Arrays.asList( + new FloatDimensionSchema("d1") + ), + null, + null + ); + InputRowSerde.toBytes(InputRowSerde.getTypeHelperMap(dimensionsSpec), in, aggregatorFactories, true); + + expectedException.expect(ParseException.class); + expectedException.expectMessage("could not convert value [d1v] to double"); + dimensionsSpec = new DimensionsSpec( + Arrays.asList( + new DoubleDimensionSchema("d1") + ), + null, + null + ); + InputRowSerde.toBytes(InputRowSerde.getTypeHelperMap(dimensionsSpec), in, aggregatorFactories, true); } } diff --git a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java index c10ed586325f..58f175adbcef 100644 --- a/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java +++ b/processing/src/main/java/io/druid/segment/DimensionHandlerUtils.java @@ -239,6 +239,12 @@ private static Colu @Nullable public static Long convertObjectToLong(@Nullable Object valObj) + { + return convertObjectToLong(valObj, false); + } + + @Nullable + public static Long convertObjectToLong(@Nullable Object valObj, boolean reportParseExceptions) { if (valObj == null) { return ZERO_LONG; @@ -249,7 +255,11 @@ public static Long convertObjectToLong(@Nullable Object valObj) } else if (valObj instanceof Number) { return ((Number) valObj).longValue(); } else if (valObj instanceof String) { - return DimensionHandlerUtils.getExactLongFromDecimalString((String) valObj); + Long ret = DimensionHandlerUtils.getExactLongFromDecimalString((String) valObj); + if (reportParseExceptions && ret == null) { + throw new ParseException("could not convert value [%s] to long", valObj); + } + return ret; } else { throw new ParseException("Unknown type[%s]", valObj.getClass()); } @@ -257,6 +267,12 @@ public static Long convertObjectToLong(@Nullable Object valObj) @Nullable public static Float convertObjectToFloat(@Nullable Object valObj) + { + return convertObjectToFloat(valObj, false); + } + + @Nullable + public static Float convertObjectToFloat(@Nullable Object valObj, boolean reportParseExceptions) { if (valObj == null) { return ZERO_FLOAT; @@ -267,7 +283,11 @@ public static Float convertObjectToFloat(@Nullable Object valObj) } else if (valObj instanceof Number) { return ((Number) valObj).floatValue(); } else if (valObj instanceof String) { - return Floats.tryParse((String) valObj); + Float ret = Floats.tryParse((String) valObj); + if (reportParseExceptions && ret == null) { + throw new ParseException("could not convert value [%s] to float", valObj); + } + return ret; } else { throw new ParseException("Unknown type[%s]", valObj.getClass()); } @@ -275,6 +295,12 @@ public static Float convertObjectToFloat(@Nullable Object valObj) @Nullable public static Double convertObjectToDouble(@Nullable Object valObj) + { + return convertObjectToDouble(valObj, false); + } + + @Nullable + public static Double convertObjectToDouble(@Nullable Object valObj, boolean reportParseExceptions) { if (valObj == null) { return ZERO_DOUBLE; @@ -285,7 +311,11 @@ public static Double convertObjectToDouble(@Nullable Object valObj) } else if (valObj instanceof Number) { return ((Number) valObj).doubleValue(); } else if (valObj instanceof String) { - return Doubles.tryParse((String) valObj); + Double ret = Doubles.tryParse((String) valObj); + if (reportParseExceptions && ret == null) { + throw new ParseException("could not convert value [%s] to double", valObj); + } + return ret; } else { throw new ParseException("Unknown type[%s]", valObj.getClass()); } diff --git a/processing/src/main/java/io/druid/segment/DimensionIndexer.java b/processing/src/main/java/io/druid/segment/DimensionIndexer.java index 390c1992ea63..cbc298c2f246 100644 --- a/processing/src/main/java/io/druid/segment/DimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/DimensionIndexer.java @@ -122,9 +122,10 @@ public interface DimensionIndexer * * @param dimValues Single row val to process * + * @param reportParseExceptions * @return An array containing an encoded representation of the input row value. */ - EncodedKeyComponentType processRowValsToUnsortedEncodedKeyComponent(Object dimValues); + EncodedKeyComponentType processRowValsToUnsortedEncodedKeyComponent(Object dimValues, boolean reportParseExceptions); /** diff --git a/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java b/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java index 26d71abc2933..2643ed6ff386 100644 --- a/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/DoubleDimensionIndexer.java @@ -34,12 +34,14 @@ public class DoubleDimensionIndexer implements DimensionIndexer zero conversion when https://github.com/druid-io/druid/pull/5278 series of patches is merged + return ret == null ? DimensionHandlerUtils.ZERO_DOUBLE : ret; } @Override diff --git a/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java b/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java index 49c93f576107..6ac82d6abc36 100644 --- a/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/FloatDimensionIndexer.java @@ -34,13 +34,15 @@ public class FloatDimensionIndexer implements DimensionIndexer zero conversion when https://github.com/druid-io/druid/pull/5278 series of patches is merged + return ret == null ? DimensionHandlerUtils.ZERO_FLOAT : ret; } @Override diff --git a/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java b/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java index 8aeeca25395d..29afdde65b9c 100644 --- a/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/LongDimensionIndexer.java @@ -34,13 +34,15 @@ public class LongDimensionIndexer implements DimensionIndexer { @Override - public Long processRowValsToUnsortedEncodedKeyComponent(Object dimValues) + public Long processRowValsToUnsortedEncodedKeyComponent(Object dimValues, boolean reportParseExceptions) { if (dimValues instanceof List) { throw new UnsupportedOperationException("Numeric columns do not support multivalue rows."); } - return DimensionHandlerUtils.convertObjectToLong(dimValues); + Long ret = DimensionHandlerUtils.convertObjectToLong(dimValues, reportParseExceptions); + // remove null -> zero conversion when https://github.com/druid-io/druid/pull/5278 series of patches is merged + return ret == null ? DimensionHandlerUtils.ZERO_LONG : ret; } @Override diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index 71e1c3443fe4..96fd4d8f74d1 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -188,7 +188,7 @@ public StringDimensionIndexer(MultiValueHandling multiValueHandling) } @Override - public int[] processRowValsToUnsortedEncodedKeyComponent(Object dimValues) + public int[] processRowValsToUnsortedEncodedKeyComponent(Object dimValues, boolean reportParseExceptions) { final int[] encodedDimensionValues; final int oldDictSize = dimLookup.size(); diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java index 4aa4a1b976b3..a752c5d4425f 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java @@ -552,7 +552,10 @@ TimeAndDims toTimeAndDims(InputRow row) } DimensionHandler handler = desc.getHandler(); DimensionIndexer indexer = desc.getIndexer(); - Object dimsKey = indexer.processRowValsToUnsortedEncodedKeyComponent(row.getRaw(dimension)); + Object dimsKey = indexer.processRowValsToUnsortedEncodedKeyComponent( + row.getRaw(dimension), + reportParseExceptions + ); // Set column capabilities as data is coming in if (!capabilities.hasMultipleValues() && dimsKey != null && handler.getLengthOfEncodedKeyComponent(dimsKey) > 1) { diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java index 8cb44ffa669b..8eb148d6bb4a 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java @@ -100,7 +100,7 @@ public IncrementalIndexAdapter( // Add 'null' to the dimension's dictionary. if (dimIndex >= dims.length || dims[dimIndex] == null) { - accessor.indexer.processRowValsToUnsortedEncodedKeyComponent(null); + accessor.indexer.processRowValsToUnsortedEncodedKeyComponent(null, true); continue; } final ColumnCapabilities capabilities = dimension.getCapabilities(); diff --git a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java index ee728cb51605..ca6ffe9fb66f 100644 --- a/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexTest.java @@ -25,13 +25,15 @@ import com.google.common.collect.Lists; import io.druid.collections.StupidPool; import io.druid.data.input.MapBasedInputRow; -import io.druid.data.input.Row; import io.druid.data.input.impl.DimensionSchema; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.DoubleDimensionSchema; +import io.druid.data.input.impl.FloatDimensionSchema; +import io.druid.data.input.impl.LongDimensionSchema; import io.druid.data.input.impl.StringDimensionSchema; import io.druid.java.util.common.ISE; import io.druid.java.util.common.granularity.Granularities; +import io.druid.java.util.common.parsers.ParseException; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.FilteredAggregatorFactory; @@ -40,6 +42,7 @@ import org.junit.Assert; import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -60,6 +63,9 @@ interface IndexCreator IncrementalIndex createIndex(); } + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Rule public final CloserRule closer = new CloserRule(false); @@ -76,8 +82,8 @@ public static Collection constructorFeeder() throws IOException DimensionsSpec dimensions = new DimensionsSpec( Arrays.asList( new StringDimensionSchema("string"), - new StringDimensionSchema("float"), - new StringDimensionSchema("long"), + new FloatDimensionSchema("float"), + new LongDimensionSchema("long"), new DoubleDimensionSchema("double") ), null, null ); @@ -206,28 +212,54 @@ public void controlTest() throws IndexSizeExceededException } @Test - public void testNullDimensionTransform() throws IndexSizeExceededException + public void testUnparseableNumerics() throws IndexSizeExceededException { IncrementalIndex index = closer.closeLater(indexCreator.createIndex()); + + expectedException.expect(ParseException.class); + expectedException.expectMessage("could not convert value [asdj] to long"); index.add( new MapBasedInputRow( System.currentTimeMillis() - 1, Lists.newArrayList("string", "float", "long", "double"), ImmutableMap.of( - "string", Arrays.asList("A", null, ""), - "float", Arrays.asList(Float.POSITIVE_INFINITY, null, ""), - "long", Arrays.asList(Long.MIN_VALUE, null, ""), - "double", "" + "string", "A", + "float", "19.0", + "long", "asdj", + "double", 21.0d ) ) ); - Row row = index.iterator().next(); + expectedException.expect(ParseException.class); + expectedException.expectMessage("could not convert value [aaa] to float"); + index.add( + new MapBasedInputRow( + System.currentTimeMillis() - 1, + Lists.newArrayList("string", "float", "long", "double"), + ImmutableMap.of( + "string", "A", + "float", "aaa", + "long", 20, + "double", 21.0d + ) + ) + ); - Assert.assertEquals(Arrays.asList(new String[]{"", "", "A"}), row.getRaw("string")); - Assert.assertEquals(Arrays.asList(new String[]{"", "", String.valueOf(Float.POSITIVE_INFINITY)}), row.getRaw("float")); - Assert.assertEquals(Arrays.asList(new String[]{"", "", String.valueOf(Long.MIN_VALUE)}), row.getRaw("long")); - Assert.assertEquals(0.0, row.getMetric("double").doubleValue(), 0.0); + expectedException.expect(ParseException.class); + expectedException.expectMessage("could not convert value [] to double"); + index.add( + new MapBasedInputRow( + System.currentTimeMillis() - 1, + Lists.newArrayList("string", "float", "long", "double"), + ImmutableMap.of( + "string", "A", + "float", 19.0, + "long", 20, + "double", "" + ) + ) + ); } @Test From 7f280902d380771b878c495e9ffdd5c2859bcf3c Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 6 Feb 2018 17:11:45 -0800 Subject: [PATCH 21/82] Fix two improper casts in HavingSpecMetricComparator. (#5352) (#5357) * Fix two improper casts in HavingSpecMetricComparator. Fixes two things: 1. An improper double-to-long cast when comparing double metrics to any kind of value, which was a regression from #4883. 2. An improper double-to-long cast when comparing a long/int metric to a double/float value: the value was cast to long/int, drawing strange conclusions like int 100 matching a havingSpec of equalTo(100.5). * Add comments. * Remove extraneous comment. * Simplify code a bit. --- .../having/HavingSpecMetricComparator.java | 47 ++++++++++----- .../query/groupby/having/HavingSpecTest.java | 58 ++++++++++++++++++- 2 files changed, 88 insertions(+), 17 deletions(-) diff --git a/processing/src/main/java/io/druid/query/groupby/having/HavingSpecMetricComparator.java b/processing/src/main/java/io/druid/query/groupby/having/HavingSpecMetricComparator.java index ff08202bf7cb..1b041dc1be5a 100644 --- a/processing/src/main/java/io/druid/query/groupby/having/HavingSpecMetricComparator.java +++ b/processing/src/main/java/io/druid/query/groupby/having/HavingSpecMetricComparator.java @@ -20,13 +20,12 @@ package io.druid.query.groupby.having; import com.google.common.primitives.Doubles; -import com.google.common.primitives.Floats; -import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import io.druid.data.input.Row; -import io.druid.query.aggregation.AggregatorFactory; import io.druid.java.util.common.ISE; +import io.druid.query.aggregation.AggregatorFactory; +import java.math.BigDecimal; import java.util.Map; import java.util.regex.Pattern; @@ -46,18 +45,29 @@ static int compare(Row row, String aggregationName, Number value, Map Date: Tue, 6 Feb 2018 17:51:17 -0800 Subject: [PATCH 22/82] Fix races in LookupSnapshotTaker, CoordinatorPollingBasicAuthenticatorCacheManager (#5344) (#5360) * Fix races in LookupSnapshotTaker, CoordinatorPollingBasicAuthenticatorCacheManager. Both were susceptible to the following conditions: 1. Two JVMs on the same machine (perhaps two peons) could conflict by one reading while the other was writing, or by writing to the file at the same time. 2. One JVM could partially write a file, then crash, leaving a truncated file. * Use StringUtils.format --- ...PollingBasicAuthenticatorCacheManager.java | 4 +- .../io/druid/java/util/common/FileUtils.java | 71 ++++++++++++++++++- .../druid/java/util/common/FileUtilsTest.java | 26 +++++++ .../query/lookup/LookupSnapshotTaker.java | 10 ++- .../query/lookup/LookupSnapshotTakerTest.java | 12 +++- 5 files changed, 114 insertions(+), 9 deletions(-) diff --git a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/db/cache/CoordinatorPollingBasicAuthenticatorCacheManager.java b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/db/cache/CoordinatorPollingBasicAuthenticatorCacheManager.java index 178232e13dfc..dd8bc10501a1 100644 --- a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/db/cache/CoordinatorPollingBasicAuthenticatorCacheManager.java +++ b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/db/cache/CoordinatorPollingBasicAuthenticatorCacheManager.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; -import com.google.common.io.Files; import com.google.inject.Inject; import com.google.inject.Injector; import com.metamx.emitter.EmittingLogger; @@ -31,6 +30,7 @@ import io.druid.discovery.DruidLeaderClient; import io.druid.guice.ManageLifecycle; import io.druid.guice.annotations.Smile; +import io.druid.java.util.common.FileUtils; import io.druid.java.util.common.ISE; import io.druid.java.util.common.RetryUtils; import io.druid.java.util.common.StringUtils; @@ -235,7 +235,7 @@ private void writeUserMapToDisk(String prefix, byte[] userMapBytes) throws IOExc File cacheDir = new File(commonCacheConfig.getCacheDirectory()); cacheDir.mkdirs(); File userMapFile = new File(commonCacheConfig.getCacheDirectory(), getUserMapFilename(prefix)); - Files.write(userMapBytes, userMapFile); + FileUtils.writeAtomically(userMapFile, out -> out.write(userMapBytes)); } private Map tryFetchUserMapFromCoordinator(String prefix) throws Exception diff --git a/java-util/src/main/java/io/druid/java/util/common/FileUtils.java b/java-util/src/main/java/io/druid/java/util/common/FileUtils.java index 72be3a57e18f..0a5d08063881 100644 --- a/java-util/src/main/java/io/druid/java/util/common/FileUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/FileUtils.java @@ -27,11 +27,16 @@ import java.io.File; import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FilterOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; +import java.nio.file.StandardCopyOption; import java.util.Arrays; import java.util.Collection; +import java.util.UUID; public class FileUtils { @@ -46,6 +51,7 @@ public boolean apply(Throwable input) return input instanceof Exception; } }; + /** * Copy input byte source to outFile. If outFile exists, it is attempted to be deleted. * @@ -150,10 +156,11 @@ public void addFile(File file) * }} * * @param file the file to map + * * @return a {@link MappedByteBufferHandler}, wrapping a read-only buffer reflecting {@code file} - * @throws FileNotFoundException if the {@code file} does not exist - * @throws IOException if an I/O error occurs * + * @throws FileNotFoundException if the {@code file} does not exist + * @throws IOException if an I/O error occurs * @see FileChannel#map(FileChannel.MapMode, long, long) */ public static MappedByteBufferHandler map(File file) throws IOException @@ -161,4 +168,64 @@ public static MappedByteBufferHandler map(File file) throws IOException MappedByteBuffer mappedByteBuffer = Files.map(file); return new MappedByteBufferHandler(mappedByteBuffer); } + + /** + * Write to a file atomically, by first writing to a temporary file in the same directory and then moving it to + * the target location. This function attempts to clean up its temporary files when possible, but they may stick + * around (for example, if the JVM crashes partway through executing the function). In any case, the target file + * should be unharmed. + * + * The OutputStream passed to the consumer is uncloseable; calling close on it will do nothing. This is to ensure + * that the stream stays open so we can fsync it here before closing. Hopefully, this doesn't cause any problems + * for callers. + * + * This method is not just thread-safe, but is also safe to use from multiple processes on the same machine. + */ + public static void writeAtomically(final File file, OutputStreamConsumer f) throws IOException + { + writeAtomically(file, file.getParentFile(), f); + } + + private static void writeAtomically(final File file, final File tmpDir, OutputStreamConsumer f) throws IOException + { + final File tmpFile = new File(tmpDir, StringUtils.format(".%s.%s", file.getName(), UUID.randomUUID())); + + try { + try (final FileOutputStream out = new FileOutputStream(tmpFile)) { + // Pass f an uncloseable stream so we can fsync before closing. + f.accept(uncloseable(out)); + + // fsync to avoid write-then-rename-then-crash causing empty files on some filesystems. + out.getChannel().force(true); + } + + // No exception thrown; do the move. + java.nio.file.Files.move( + tmpFile.toPath(), + file.toPath(), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING + ); + } + finally { + tmpFile.delete(); + } + } + + private static OutputStream uncloseable(final OutputStream out) throws IOException + { + return new FilterOutputStream(out) + { + @Override + public void close() throws IOException + { + // Do nothing. + } + }; + } + + public interface OutputStreamConsumer + { + void accept(OutputStream outputStream) throws IOException; + } } diff --git a/java-util/src/test/java/io/druid/java/util/common/FileUtilsTest.java b/java-util/src/test/java/io/druid/java/util/common/FileUtilsTest.java index a9594f7a02a1..3e4aea4eca8e 100644 --- a/java-util/src/test/java/io/druid/java/util/common/FileUtilsTest.java +++ b/java-util/src/test/java/io/druid/java/util/common/FileUtilsTest.java @@ -27,6 +27,7 @@ import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; +import java.nio.file.Files; public class FileUtilsTest { @@ -48,4 +49,29 @@ public void testMap() throws IOException long buffersMemoryAfter = BufferUtils.totalMemoryUsedByDirectAndMappedBuffers(); Assert.assertEquals(buffersMemoryBefore, buffersMemoryAfter); } + + @Test + public void testWriteAtomically() throws IOException + { + final File tmpDir = folder.newFolder(); + final File tmpFile = new File(tmpDir, "file1"); + FileUtils.writeAtomically(tmpFile, out -> out.write(StringUtils.toUtf8("foo"))); + Assert.assertEquals("foo", StringUtils.fromUtf8(Files.readAllBytes(tmpFile.toPath()))); + + // Try writing again, throw error partway through. + try { + FileUtils.writeAtomically(tmpFile, out -> { + out.write(StringUtils.toUtf8("bar")); + out.flush(); + throw new ISE("OMG!"); + }); + } + catch (IllegalStateException e) { + // Suppress + } + Assert.assertEquals("foo", StringUtils.fromUtf8(Files.readAllBytes(tmpFile.toPath()))); + + FileUtils.writeAtomically(tmpFile, out -> out.write(StringUtils.toUtf8("baz"))); + Assert.assertEquals("baz", StringUtils.fromUtf8(Files.readAllBytes(tmpFile.toPath()))); + } } diff --git a/processing/src/main/java/io/druid/query/lookup/LookupSnapshotTaker.java b/processing/src/main/java/io/druid/query/lookup/LookupSnapshotTaker.java index 3d67124610ff..9162bf4b2035 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupSnapshotTaker.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupSnapshotTaker.java @@ -25,6 +25,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import io.druid.guice.annotations.Json; +import io.druid.java.util.common.FileUtils; import io.druid.java.util.common.ISE; import io.druid.java.util.common.logger.Logger; @@ -50,7 +51,10 @@ public LookupSnapshotTaker( ) { this.objectMapper = jsonMapper; - Preconditions.checkArgument(!Strings.isNullOrEmpty(persistDirectory), "can not work without specifying persistDirectory"); + Preconditions.checkArgument( + !Strings.isNullOrEmpty(persistDirectory), + "can not work without specifying persistDirectory" + ); this.persistDirectory = new File(persistDirectory); if (!this.persistDirectory.exists()) { Preconditions.checkArgument(this.persistDirectory.mkdirs(), "Oups was not able to create persist directory"); @@ -72,7 +76,7 @@ public synchronized List pullExistingSnapshot() LOGGER.warn("found empty file no lookups to load from [%s]", persistFile.getAbsolutePath()); return Collections.emptyList(); } - lookupBeanList = objectMapper.readValue(persistFile, new TypeReference>(){}); + lookupBeanList = objectMapper.readValue(persistFile, new TypeReference>() {}); return lookupBeanList; } catch (IOException e) { @@ -83,7 +87,7 @@ public synchronized List pullExistingSnapshot() public synchronized void takeSnapshot(List lookups) { try { - objectMapper.writeValue(persistFile, lookups); + FileUtils.writeAtomically(persistFile, out -> objectMapper.writeValue(out, lookups)); } catch (IOException e) { throw new ISE(e, "Exception during serialization of lookups using file [%s]", persistFile.getAbsolutePath()); diff --git a/server/src/test/java/io/druid/query/lookup/LookupSnapshotTakerTest.java b/server/src/test/java/io/druid/query/lookup/LookupSnapshotTakerTest.java index 3fc085623ffa..ac63bbdf01e3 100644 --- a/server/src/test/java/io/druid/query/lookup/LookupSnapshotTakerTest.java +++ b/server/src/test/java/io/druid/query/lookup/LookupSnapshotTakerTest.java @@ -31,6 +31,7 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import java.io.File; @@ -43,6 +44,10 @@ public class LookupSnapshotTakerTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + private final ObjectMapper mapper = TestHelper.makeJsonMapper(); @@ -112,7 +117,7 @@ public void testBackwardCompatibility() throws IOException Assert.assertEquals(lookupBeanList, actualList); } - @Test(expected = ISE.class) + @Test public void testIOExceptionDuringLookupPersist() throws IOException { File directory = temporaryFolder.newFolder(); @@ -120,6 +125,7 @@ public void testIOExceptionDuringLookupPersist() throws IOException Assert.assertFalse(snapshotFile.exists()); Assert.assertTrue(snapshotFile.createNewFile()); Assert.assertTrue(snapshotFile.setReadOnly()); + Assert.assertTrue(snapshotFile.getParentFile().setReadOnly()); LookupSnapshotTaker lookupSnapshotTaker = new LookupSnapshotTaker(mapper, directory.getAbsolutePath()); LookupBean lookupBean = new LookupBean( "name", @@ -135,10 +141,12 @@ public void testIOExceptionDuringLookupPersist() throws IOException ) ); List lookupBeanList = Lists.newArrayList(lookupBean); + + expectedException.expect(ISE.class); + expectedException.expectMessage("Exception during serialization of lookups"); lookupSnapshotTaker.takeSnapshot(lookupBeanList); } - @Test public void tesLookupPullingFromEmptyFile() throws IOException { From 4ba1bb8e61ccb754c17b4b5605b58f3b5c8f5f4b Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Tue, 6 Feb 2018 19:02:03 -0800 Subject: [PATCH 23/82] Fix race in CoordinatorPollingBasicAuthorizerCacheManager. (#5359) (#5361) Similar to #5344 but for the authorizer instead of the authenticator. --- .../cache/CoordinatorPollingBasicAuthorizerCacheManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authorization/db/cache/CoordinatorPollingBasicAuthorizerCacheManager.java b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authorization/db/cache/CoordinatorPollingBasicAuthorizerCacheManager.java index a57d931c97f5..a8c4a1837aa5 100644 --- a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authorization/db/cache/CoordinatorPollingBasicAuthorizerCacheManager.java +++ b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authorization/db/cache/CoordinatorPollingBasicAuthorizerCacheManager.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; -import com.google.common.io.Files; import com.google.inject.Inject; import com.google.inject.Injector; import com.metamx.emitter.EmittingLogger; @@ -31,6 +30,7 @@ import io.druid.discovery.DruidLeaderClient; import io.druid.guice.ManageLifecycle; import io.druid.guice.annotations.Smile; +import io.druid.java.util.common.FileUtils; import io.druid.java.util.common.ISE; import io.druid.java.util.common.RetryUtils; import io.druid.java.util.common.StringUtils; @@ -212,7 +212,7 @@ private void writeMapToDisk(String prefix, byte[] userMapBytes) throws IOExcepti File cacheDir = new File(commonCacheConfig.getCacheDirectory()); cacheDir.mkdirs(); File userMapFile = new File(commonCacheConfig.getCacheDirectory(), getUserRoleMapFilename(prefix)); - Files.write(userMapBytes, userMapFile); + FileUtils.writeAtomically(userMapFile, out -> out.write(userMapBytes)); } @Nullable From 49e27d641f967006ab0a325ffe7f5a9fbe75ca99 Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Wed, 7 Feb 2018 15:24:19 -0800 Subject: [PATCH 24/82] [Backport] Use a separate snapshot file per lookup tier. (#5358) (#5367) Prevents conflicts if two processes on the same machine use the same lookup snapshot directory but are in different tiers. --- .../query/lookup/LookupSnapshotTaker.java | 22 +++--- .../query/lookup/LookupReferencesManager.java | 7 +- .../lookup/LookupReferencesManagerTest.java | 30 ++++---- .../query/lookup/LookupSnapshotTakerTest.java | 74 ++++++------------- 4 files changed, 54 insertions(+), 79 deletions(-) diff --git a/processing/src/main/java/io/druid/query/lookup/LookupSnapshotTaker.java b/processing/src/main/java/io/druid/query/lookup/LookupSnapshotTaker.java index 9162bf4b2035..52b69c03270c 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupSnapshotTaker.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupSnapshotTaker.java @@ -19,14 +19,15 @@ package io.druid.query.lookup; - import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import io.druid.guice.annotations.Json; import io.druid.java.util.common.FileUtils; import io.druid.java.util.common.ISE; +import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; import java.io.File; @@ -34,16 +35,13 @@ import java.util.Collections; import java.util.List; - public class LookupSnapshotTaker { private static final Logger LOGGER = new Logger(LookupSnapshotTaker.class); - protected static final String PERSIST_FILE_NAME = "lookupSnapshot.json"; + private static final String PERSIST_FILE_SUFFIX = "lookupSnapshot.json"; private final ObjectMapper objectMapper; private final File persistDirectory; - private final File persistFile; - public LookupSnapshotTaker( final @Json ObjectMapper jsonMapper, @@ -62,11 +60,12 @@ public LookupSnapshotTaker( if (!this.persistDirectory.isDirectory()) { throw new ISE("Can only persist to directories, [%s] wasn't a directory", persistDirectory); } - this.persistFile = new File(persistDirectory, PERSIST_FILE_NAME); } - public synchronized List pullExistingSnapshot() + public synchronized List pullExistingSnapshot(final String tier) { + final File persistFile = getPersistFile(tier); + List lookupBeanList; try { if (!persistFile.isFile()) { @@ -84,8 +83,10 @@ public synchronized List pullExistingSnapshot() } } - public synchronized void takeSnapshot(List lookups) + public synchronized void takeSnapshot(String tier, List lookups) { + final File persistFile = getPersistFile(tier); + try { FileUtils.writeAtomically(persistFile, out -> objectMapper.writeValue(out, lookups)); } @@ -94,8 +95,9 @@ public synchronized void takeSnapshot(List lookups) } } - public File getPersistFile() + @VisibleForTesting + File getPersistFile(final String tier) { - return persistFile; + return new File(persistDirectory, StringUtils.format("%s.%s", tier, PERSIST_FILE_SUFFIX)); } } diff --git a/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java b/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java index df2229f9229b..c9ab30c35bae 100644 --- a/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java +++ b/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java @@ -339,7 +339,7 @@ private void updateToLoadAndDrop( private void takeSnapshot(Map lookupMap) { if (lookupSnapshotTaker != null) { - lookupSnapshotTaker.takeSnapshot(getLookupBeanList(lookupMap)); + lookupSnapshotTaker.takeSnapshot(lookupListeningAnnouncerConfig.getLookupTier(), getLookupBeanList(lookupMap)); } } @@ -362,8 +362,7 @@ private List getLookupsList() { List lookupBeanList; if (lookupConfig.getEnableLookupSyncOnStartup()) { - String tier = lookupListeningAnnouncerConfig.getLookupTier(); - lookupBeanList = getLookupListFromCoordinator(tier); + lookupBeanList = getLookupListFromCoordinator(lookupListeningAnnouncerConfig.getLookupTier()); if (lookupBeanList == null) { LOG.info("Coordinator is unavailable. Loading saved snapshot instead"); lookupBeanList = getLookupListFromSnapshot(); @@ -455,7 +454,7 @@ private Map tryGetLookupListFromCoordin private List getLookupListFromSnapshot() { if (lookupSnapshotTaker != null) { - return lookupSnapshotTaker.pullExistingSnapshot(); + return lookupSnapshotTaker.pullExistingSnapshot(lookupListeningAnnouncerConfig.getLookupTier()); } return null; } diff --git a/server/src/test/java/io/druid/query/lookup/LookupReferencesManagerTest.java b/server/src/test/java/io/druid/query/lookup/LookupReferencesManagerTest.java index 62860d4d4d4f..8cc51b91f0bf 100644 --- a/server/src/test/java/io/druid/query/lookup/LookupReferencesManagerTest.java +++ b/server/src/test/java/io/druid/query/lookup/LookupReferencesManagerTest.java @@ -102,7 +102,7 @@ public void testStartStop() throws InterruptedException, IOException lookupMap.put("testMockForStartStop", container); String strResult = mapper.writeValueAsString(lookupMap); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); - expect(config.getLookupTier()).andReturn(LOOKUP_TIER); + expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes(); replay(config); expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true")) .andReturn(request); @@ -163,7 +163,7 @@ public void testAddGetRemove() throws Exception lookupMap.put("testMockForAddGetRemove", container); String strResult = mapper.writeValueAsString(lookupMap); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); - expect(config.getLookupTier()).andReturn(LOOKUP_TIER); + expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes(); replay(config); expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true")) .andReturn(request); @@ -201,7 +201,7 @@ public void testCloseIsCalledAfterStopping() throws Exception lookupMap.put("testMockForCloseIsCalledAfterStopping", container); String strResult = mapper.writeValueAsString(lookupMap); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); - expect(config.getLookupTier()).andReturn(LOOKUP_TIER); + expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes(); replay(config); expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true")) .andReturn(request); @@ -232,7 +232,7 @@ public void testCloseIsCalledAfterRemove() throws Exception lookupMap.put("testMockForCloseIsCalledAfterRemove", container); String strResult = mapper.writeValueAsString(lookupMap); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); - expect(config.getLookupTier()).andReturn(LOOKUP_TIER); + expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes(); replay(config); expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true")) .andReturn(request); @@ -260,7 +260,7 @@ public void testGetNotThere() throws Exception lookupMap.put("testMockForGetNotThere", container); String strResult = mapper.writeValueAsString(lookupMap); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); - expect(config.getLookupTier()).andReturn(LOOKUP_TIER); + expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes(); replay(config); expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true")) .andReturn(request); @@ -290,7 +290,7 @@ public void testUpdateWithHigherVersion() throws Exception lookupMap.put("testMockForUpdateWithHigherVersion", container); String strResult = mapper.writeValueAsString(lookupMap); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); - expect(config.getLookupTier()).andReturn(LOOKUP_TIER); + expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes(); replay(config); expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true")) .andReturn(request); @@ -324,7 +324,7 @@ public void testUpdateWithLowerVersion() throws Exception lookupMap.put("testMockForUpdateWithLowerVersion", container); String strResult = mapper.writeValueAsString(lookupMap); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); - expect(config.getLookupTier()).andReturn(LOOKUP_TIER); + expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes(); replay(config); expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true")) .andReturn(request); @@ -352,7 +352,7 @@ public void testRemoveNonExisting() throws Exception lookupMap.put("testMockForRemoveNonExisting", container); String strResult = mapper.writeValueAsString(lookupMap); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); - expect(config.getLookupTier()).andReturn(LOOKUP_TIER); + expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes(); replay(config); expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true")) .andReturn(request); @@ -403,7 +403,7 @@ public void testGetAllLookupsState() throws Exception Map lookupMap = new HashMap<>(); String strResult = mapper.writeValueAsString(lookupMap); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); - expect(config.getLookupTier()).andReturn(LOOKUP_TIER); + expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes(); replay(config); expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true")) .andReturn(request); @@ -445,7 +445,7 @@ public void testRealModeWithMainThread() throws Exception lookupMap.put("testMockForRealModeWithMainThread", container); String strResult = mapper.writeValueAsString(lookupMap); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); - expect(config.getLookupTier()).andReturn(LOOKUP_TIER); + expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes(); replay(config); expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true")) .andReturn(request); @@ -521,7 +521,7 @@ public void testCoordinatorLookupSync() throws Exception lookupMap.put("testLookup3", container3); String strResult = mapper.writeValueAsString(lookupMap); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); - expect(config.getLookupTier()).andReturn(LOOKUP_TIER); + expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes(); replay(config); expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true")) .andReturn(request); @@ -547,7 +547,7 @@ public void testLoadLookupOnCoordinatorFailure() throws Exception lookupMap.put("testMockForLoadLookupOnCoordinatorFailure", container); String strResult = mapper.writeValueAsString(lookupMap); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); - expect(config.getLookupTier()).andReturn(LOOKUP_TIER); + expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes(); replay(config); expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true")) .andReturn(request) @@ -565,13 +565,13 @@ public void testLoadLookupOnCoordinatorFailure() throws Exception lookupReferencesManager.handlePendingNotices(); lookupReferencesManager.stop(); lookupReferencesManager = new LookupReferencesManager( - new LookupConfig(lookupReferencesManager.lookupSnapshotTaker.getPersistFile().getParent()), + new LookupConfig(lookupReferencesManager.lookupSnapshotTaker.getPersistFile(LOOKUP_TIER).getParent()), mapper, druidLeaderClient, config, true ); reset(config); reset(druidLeaderClient); - expect(config.getLookupTier()).andReturn(LOOKUP_TIER); + expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes(); replay(config); expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true")) .andReturn(request) @@ -593,7 +593,7 @@ public void testDisableLookupSync() throws Exception lookupMap.put("testMockForDisableLookupSync", container); String strResult = mapper.writeValueAsString(lookupMap); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); - expect(config.getLookupTier()).andReturn(LOOKUP_TIER); + expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes(); replay(config); expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/lookupTier?detailed=true")) .andReturn(request); diff --git a/server/src/test/java/io/druid/query/lookup/LookupSnapshotTakerTest.java b/server/src/test/java/io/druid/query/lookup/LookupSnapshotTakerTest.java index ac63bbdf01e3..fed17880abba 100644 --- a/server/src/test/java/io/druid/query/lookup/LookupSnapshotTakerTest.java +++ b/server/src/test/java/io/druid/query/lookup/LookupSnapshotTakerTest.java @@ -26,7 +26,6 @@ import io.druid.java.util.common.ISE; import io.druid.java.util.common.StringUtils; import io.druid.segment.TestHelper; -import org.apache.commons.io.FileUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -42,6 +41,9 @@ public class LookupSnapshotTakerTest { + private static final String TIER1 = "tier1"; + private static final String TIER2 = "tier2"; + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -65,68 +67,40 @@ public void setUp() throws IOException @Test public void testTakeSnapshotAndPullExisting() throws IOException { - - LookupBean lookupBean = new LookupBean( - "name", + LookupBean lookupBean1 = new LookupBean( + "name1", null, new LookupExtractorFactoryContainer( "v1", - new MapLookupExtractorFactory( - ImmutableMap.of( - "key", - "value" - ), true - ) + new MapLookupExtractorFactory(ImmutableMap.of("key", "value"), true) ) ); - List lookupBeanList = Lists.newArrayList(lookupBean); - lookupSnapshotTaker.takeSnapshot(lookupBeanList); - List actualList = lookupSnapshotTaker.pullExistingSnapshot(); - Assert.assertEquals(lookupBeanList, actualList); - } - - //test backward compatibility with snapshots stored using 0.9.x code - @Test - public void testBackwardCompatibility() throws IOException - { - File directory = temporaryFolder.newFolder(); - File snapshotFile = new File(directory, LookupSnapshotTaker.PERSIST_FILE_NAME); - Assert.assertFalse(snapshotFile.exists()); - FileUtils.write( - snapshotFile, - "[{\"factory\":{\"type\":\"map\",\"map\":{\"key\":\"value\"},\"isOneToOne\":true},\"name\":\"name\"}]" - ); - Assert.assertTrue(snapshotFile.exists()); - List actualList = new LookupSnapshotTaker(mapper, directory.getAbsolutePath()).pullExistingSnapshot(); - - LookupBean lookupBean = new LookupBean( - "name", + LookupBean lookupBean2 = new LookupBean( + "name2", null, new LookupExtractorFactoryContainer( - null, - new MapLookupExtractorFactory( - ImmutableMap.of( - "key", - "value" - ), true - ) + "v1", + new MapLookupExtractorFactory(ImmutableMap.of("key", "value"), true) ) ); - List lookupBeanList = Lists.newArrayList(lookupBean); - - Assert.assertEquals(lookupBeanList, actualList); + List lookupBeanList1 = Lists.newArrayList(lookupBean1); + lookupSnapshotTaker.takeSnapshot(TIER1, lookupBeanList1); + List lookupBeanList2 = Lists.newArrayList(lookupBean2); + lookupSnapshotTaker.takeSnapshot(TIER2, lookupBeanList2); + Assert.assertEquals(lookupBeanList1, lookupSnapshotTaker.pullExistingSnapshot(TIER1)); + Assert.assertEquals(lookupBeanList2, lookupSnapshotTaker.pullExistingSnapshot(TIER2)); } @Test public void testIOExceptionDuringLookupPersist() throws IOException { File directory = temporaryFolder.newFolder(); - File snapshotFile = new File(directory, LookupSnapshotTaker.PERSIST_FILE_NAME); + LookupSnapshotTaker lookupSnapshotTaker = new LookupSnapshotTaker(mapper, directory.getAbsolutePath()); + File snapshotFile = lookupSnapshotTaker.getPersistFile(TIER1); Assert.assertFalse(snapshotFile.exists()); Assert.assertTrue(snapshotFile.createNewFile()); Assert.assertTrue(snapshotFile.setReadOnly()); Assert.assertTrue(snapshotFile.getParentFile().setReadOnly()); - LookupSnapshotTaker lookupSnapshotTaker = new LookupSnapshotTaker(mapper, directory.getAbsolutePath()); LookupBean lookupBean = new LookupBean( "name", null, @@ -144,25 +118,25 @@ public void testIOExceptionDuringLookupPersist() throws IOException expectedException.expect(ISE.class); expectedException.expectMessage("Exception during serialization of lookups"); - lookupSnapshotTaker.takeSnapshot(lookupBeanList); + lookupSnapshotTaker.takeSnapshot(TIER1, lookupBeanList); } @Test public void tesLookupPullingFromEmptyFile() throws IOException { - File snapshotFile = lookupSnapshotTaker.getPersistFile(); + File snapshotFile = lookupSnapshotTaker.getPersistFile(TIER1); Assert.assertTrue(snapshotFile.createNewFile()); - Assert.assertEquals(Collections.EMPTY_LIST, lookupSnapshotTaker.pullExistingSnapshot()); + Assert.assertEquals(Collections.EMPTY_LIST, lookupSnapshotTaker.pullExistingSnapshot(TIER1)); } @Test(expected = ISE.class) public void tesLookupPullingFromCorruptFile() throws IOException { - File snapshotFile = lookupSnapshotTaker.getPersistFile(); + File snapshotFile = lookupSnapshotTaker.getPersistFile(TIER1); Assert.assertTrue(snapshotFile.createNewFile()); byte[] bytes = StringUtils.toUtf8("test corrupt file"); Files.write(bytes, snapshotFile); - lookupSnapshotTaker.pullExistingSnapshot(); + lookupSnapshotTaker.pullExistingSnapshot(TIER1); } @Test @@ -170,7 +144,7 @@ public void testLookupPullingFromNonExistingFile() throws IOException { File directory = temporaryFolder.newFolder(); LookupSnapshotTaker lookupSnapshotTaker = new LookupSnapshotTaker(mapper, directory.getAbsolutePath()); - List actualList = lookupSnapshotTaker.pullExistingSnapshot(); + List actualList = lookupSnapshotTaker.pullExistingSnapshot(TIER1); Assert.assertEquals(Collections.EMPTY_LIST, actualList); } } From ab765b9dae4d3664e4f447f1ecdf80c1ffc792f3 Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Thu, 15 Feb 2018 12:01:41 -0800 Subject: [PATCH 25/82] [Backport] Add metamx emitter, http clients, and metrics packages to druid java-util (#5289) (#5387) * Add metamx emitter, http clients, and metrics packages to druid java-util (#5289) * Add metamx java-util emitter, http clients, and metrics packages to druid java-util * Remove metamx java-util from pom.xml files * Checkstyle fixes * Import fix * TeamCity inspection fixes * Use slf4j, move some version defs to master pom.xml * Use parent jvm-attach-api and maven-surefire-plugin versions * Add ] to log msg, suppress inspection * Remove extra files, fix HttpPostEmitter compile --- .../java/io/druid/utils/CompressionUtils.java | 12 +- docs/content/configuration/index.md | 12 +- docs/content/development/modules.md | 2 +- docs/content/operations/performance-faq.md | 2 +- .../druid/_common/common.runtime.properties | 2 +- .../druid/_common/common.runtime.properties | 2 +- .../ambari-metrics-emitter/pom.xml | 3 +- .../ambari/metrics/AmbariMetricsEmitter.java | 10 +- .../metrics/AmbariMetricsEmitterModule.java | 2 +- .../DruidToTimelineMetricConverter.java | 2 +- .../SendAllTimelineEventConverter.java | 2 +- ...istBasedDruidToTimelineEventConverter.java | 4 +- ...asedDruidToTimelineEventConverterTest.java | 2 +- extensions-contrib/graphite-emitter/pom.xml | 3 +- .../DruidToGraphiteEventConverter.java | 2 +- .../emitter/graphite/GraphiteEmitter.java | 8 +- .../graphite/GraphiteEmitterModule.java | 2 +- .../SendAllGraphiteEventConverter.java | 2 +- .../graphite/WhiteListBasedConverter.java | 2 +- .../graphite/WhiteListBasedConverterTest.java | 2 +- .../kafka-eight-simpleConsumer/pom.xml | 3 +- ...fkaEightSimpleConsumerFirehoseFactory.java | 4 +- extensions-contrib/kafka-emitter/pom.xml | 3 +- .../io/druid/emitter/kafka/KafkaEmitter.java | 8 +- .../emitter/kafka/KafkaEmitterModule.java | 2 +- .../io/druid/client/cache/RedisCache.java | 4 +- extensions-contrib/statsd-emitter/pom.xml | 3 +- .../druid/emitter/statsd/StatsDEmitter.java | 6 +- .../emitter/statsd/StatsDEmitterModule.java | 2 +- .../src/test/java/DimensionConverterTest.java | 2 +- .../src/test/java/StatsDEmitterTest.java | 2 +- .../input/thrift/ThriftInputRowParser.java | 2 +- .../io/druid/segment/MapVirtualColumn.java | 2 +- .../DoublesSketchAggregatorFactory.java | 2 +- .../security/basic/CommonCacheNotifier.java | 12 +- .../authentication/BasicHTTPEscalator.java | 6 +- .../BytesFullResponseHandler.java | 6 +- .../BytesFullResponseHolder.java | 2 +- ...inatorBasicAuthenticatorCacheNotifier.java | 4 +- ...PollingBasicAuthenticatorCacheManager.java | 4 +- ...icAuthenticatorMetadataStorageUpdater.java | 2 +- ...ordinatorBasicAuthorizerCacheNotifier.java | 4 +- ...torPollingBasicAuthorizerCacheManager.java | 4 +- ...BasicAuthorizerMetadataStorageUpdater.java | 2 +- .../security/kerberos/KerberosEscalator.java | 2 +- .../security/kerberos/KerberosHttpClient.java | 8 +- .../kerberos/ResponseCookieHandler.java | 4 +- .../RetryIfUnauthorizedResponseHandler.java | 4 +- extensions-core/hdfs-storage/pom.xml | 11 +- .../storage/hdfs/HdfsDataSegmentKiller.java | 2 +- .../druid/indexing/kafka/KafkaIndexTask.java | 2 +- .../indexing/kafka/KafkaIndexTaskClient.java | 10 +- .../kafka/KafkaIndexTaskClientFactory.java | 2 +- .../kafka/supervisor/KafkaSupervisor.java | 6 +- .../kafka/supervisor/KafkaSupervisorSpec.java | 2 +- .../kafka/KafkaIndexTaskClientTest.java | 8 +- .../indexing/kafka/KafkaIndexTaskTest.java | 8 +- .../namespace/cache/CacheScheduler.java | 4 +- .../NamespaceExtractionCacheManager.java | 2 +- ...ffHeapNamespaceExtractionCacheManager.java | 4 +- ...OnHeapNamespaceExtractionCacheManager.java | 4 +- extensions-core/s3-extensions/pom.xml | 10 +- .../druid/storage/s3/S3DataSegmentPusher.java | 2 +- .../simple-client-sslcontext/pom.xml | 3 +- .../io/druid/https/SSLContextProvider.java | 2 +- .../io/druid/indexing/common/TaskToolbox.java | 4 +- .../indexing/common/TaskToolboxFactory.java | 4 +- .../common/actions/LocalTaskActionClient.java | 2 +- .../actions/RemoteTaskActionClient.java | 2 +- .../actions/SegmentMetadataUpdateAction.java | 2 +- .../common/actions/SegmentNukeAction.java | 2 +- .../SegmentTransactionalInsertAction.java | 2 +- .../common/actions/TaskActionToolbox.java | 2 +- .../indexing/common/task/MergeTaskBase.java | 6 +- .../common/task/RealtimeIndexTask.java | 2 +- .../IngestSegmentFirehoseFactory.java | 2 +- .../indexing/overlord/ForkingTaskRunner.java | 2 +- .../overlord/MetadataTaskStorage.java | 2 +- .../indexing/overlord/RemoteTaskRunner.java | 12 +- .../overlord/RemoteTaskRunnerFactory.java | 2 +- .../druid/indexing/overlord/TaskLockbox.java | 2 +- .../druid/indexing/overlord/TaskMaster.java | 4 +- .../io/druid/indexing/overlord/TaskQueue.java | 6 +- .../indexing/overlord/TaskRunnerUtils.java | 2 +- .../overlord/ThreadPoolTaskRunner.java | 6 +- .../AbstractWorkerProvisioningStrategy.java | 2 +- .../overlord/autoscaling/NoopAutoScaler.java | 2 +- ...ngTaskBasedWorkerProvisioningStrategy.java | 4 +- .../SimpleWorkerProvisioningStrategy.java | 4 +- .../autoscaling/ec2/EC2AutoScaler.java | 2 +- .../supervisor/SupervisorManager.java | 2 +- .../indexing/worker/WorkerTaskMonitor.java | 2 +- .../worker/executor/ExecutorLifecycle.java | 2 +- .../indexing/common/TaskToolboxTest.java | 4 +- .../actions/RemoteTaskActionClientTest.java | 4 +- .../actions/SegmentAllocateActionTest.java | 4 +- .../common/task/RealtimeIndexTaskTest.java | 8 +- .../IngestSegmentFirehoseFactoryTest.java | 2 +- .../overlord/RemoteTaskRunnerTest.java | 4 +- .../overlord/RemoteTaskRunnerTestUtils.java | 2 +- .../indexing/overlord/TaskLifecycleTest.java | 8 +- .../indexing/overlord/TaskLockboxTest.java | 4 +- ...dingTaskBasedProvisioningStrategyTest.java | 6 +- .../SimpleProvisioningStrategyTest.java | 6 +- .../indexing/overlord/http/OverlordTest.java | 4 +- .../clients/ClientInfoResourceTestClient.java | 8 +- .../CoordinatorResourceTestClient.java | 8 +- .../EventReceiverFirehoseTestClient.java | 8 +- .../clients/OverlordResourceTestClient.java | 8 +- .../clients/QueryResourceTestClient.java | 8 +- .../druid/testing/guice/DruidTestModule.java | 14 +- .../org/testng/DruidTestRunnerFactory.java | 8 +- .../indexer/ITRealtimeIndexTaskTest.java | 2 +- .../druid/tests/indexer/ITUnionQueryTest.java | 2 +- .../ITBasicAuthConfigurationTest.java | 12 +- .../resources/indexer/union_select_query.json | 2 +- java-util/pom.xml | 138 ++- .../druid/java/util/common/StreamUtils.java | 28 + .../java/util/common/guava/Sequence.java | 2 +- .../java/util/common/lifecycle/Lifecycle.java | 6 +- .../druid/java/util/common/logger/Logger.java | 6 + .../java/util/emitter/EmittingLogger.java | 139 +++ .../emitter/core/BaseHttpEmittingConfig.java | 152 +++ .../druid/java/util/emitter/core/Batch.java | 332 +++++++ .../util/emitter/core/BatchingStrategy.java | 155 ++++ .../util/emitter/core/ComposingEmitter.java | 113 +++ .../emitter/core/ConcurrentTimeCounter.java | 96 ++ .../util/emitter/core/ContentEncoding.java | 25 + .../emitter/core/EmittedBatchCounter.java | 73 ++ .../druid/java/util/emitter/core/Emitter.java | 38 + .../java/util/emitter/core/Emitters.java | 177 ++++ .../druid/java/util/emitter/core/Event.java | 37 + .../util/emitter/core/FeedUriExtractor.java | 49 + .../util/emitter/core/HttpEmitterConfig.java | 140 +++ .../util/emitter/core/HttpPostEmitter.java | 878 ++++++++++++++++++ .../util/emitter/core/LoggingEmitter.java | 190 ++++ .../emitter/core/LoggingEmitterConfig.java | 66 ++ .../java/util/emitter/core/NoopEmitter.java | 51 + .../emitter/core/ParametrizedUriEmitter.java | 212 +++++ .../core/ParametrizedUriEmitterConfig.java | 55 ++ .../core/ParametrizedUriExtractor.java | 80 ++ .../java/util/emitter/core/UriExtractor.java | 28 + .../core/ZeroCopyByteArrayOutputStream.java | 36 + .../util/emitter/factory/EmitterFactory.java | 39 + .../emitter/factory/HttpEmitterFactory.java | 39 + .../factory/LoggingEmitterFactory.java | 45 + .../emitter/factory/NoopEmitterFactory.java | 42 + .../ParametrizedUriEmitterFactory.java | 39 + .../util/emitter/service/AlertBuilder.java | 90 ++ .../java/util/emitter/service/AlertEvent.java | 185 ++++ .../util/emitter/service/ServiceEmitter.java | 106 +++ .../util/emitter/service/ServiceEvent.java | 29 + .../emitter/service/ServiceEventBuilder.java | 33 + .../emitter/service/ServiceMetricEvent.java | 203 ++++ .../util/http/client/AbstractHttpClient.java | 35 + .../http/client/CredentialedHttpClient.java | 50 + .../http/client/EnforceSslHttpClient.java | 58 ++ .../java/util/http/client/HttpClient.java | 71 ++ .../util/http/client/HttpClientConfig.java | 289 ++++++ .../java/util/http/client/HttpClientInit.java | 231 +++++ .../util/http/client/NettyHttpClient.java | 405 ++++++++ .../druid/java/util/http/client/Request.java | 200 ++++ .../http/client/auth/BasicCredentials.java | 45 + .../util/http/client/auth/Credentials.java | 29 + .../io/AppendableByteArrayInputStream.java | 211 +++++ .../netty/HttpClientPipelineFactory.java | 42 + .../client/pool/ChannelResourceFactory.java | 182 ++++ .../http/client/pool/ResourceContainer.java | 28 + .../http/client/pool/ResourceFactory.java | 29 + .../util/http/client/pool/ResourcePool.java | 301 ++++++ .../http/client/pool/ResourcePoolConfig.java | 69 ++ .../http/client/pool/ResourceVerifier.java | 26 + .../http/client/response/ClientResponse.java | 57 ++ .../client/response/FullResponseHandler.java | 80 ++ .../client/response/FullResponseHolder.java | 63 ++ .../client/response/HttpResponseHandler.java | 54 ++ .../response/InputStreamResponseHandler.java | 74 ++ .../SequenceInputStreamResponseHandler.java | 184 ++++ .../response/StatusResponseHandler.java | 79 ++ .../client/response/StatusResponseHolder.java | 54 ++ .../response/ToStringResponseHandler.java | 78 ++ .../java/util/metrics/AbstractMonitor.java | 54 ++ .../druid/java/util/metrics/CgroupUtil.java | 29 + .../java/util/metrics/CompoundMonitor.java | 75 ++ .../util/metrics/CpuAcctDeltaMonitor.java | 134 +++ .../util/metrics/FeedDefiningMonitor.java | 40 + .../util/metrics/HttpPostEmitterMonitor.java | 91 ++ .../java/util/metrics/JvmCpuMonitor.java | 89 ++ .../druid/java/util/metrics/JvmMonitor.java | 315 +++++++ .../java/util/metrics/JvmPidDiscoverer.java | 80 ++ .../java/util/metrics/JvmThreadsMonitor.java | 83 ++ .../io/druid/java/util/metrics/KeyedDiff.java | 53 ++ .../io/druid/java/util/metrics/Monitor.java | 33 + .../util/metrics/MonitorOfTheMonitors.java | 35 + .../java/util/metrics/MonitorScheduler.java | 142 +++ .../util/metrics/MonitorSchedulerConfig.java | 33 + .../druid/java/util/metrics/MonitorUtils.java | 34 + .../io/druid/java/util/metrics/Monitors.java | 94 ++ .../ParametrizedUriEmitterMonitor.java | 75 ++ .../java/util/metrics/PidDiscoverer.java | 25 + .../java/util/metrics/SigarPidDiscoverer.java | 44 + .../io/druid/java/util/metrics/SigarUtil.java | 85 ++ .../druid/java/util/metrics/SysMonitor.java | 584 ++++++++++++ .../metrics/cgroups/CgroupDiscoverer.java | 32 + .../java/util/metrics/cgroups/CpuAcct.java | 158 ++++ .../metrics/cgroups/ProcCgroupDiscoverer.java | 174 ++++ .../cgroups/ProcPidCgroupDiscoverer.java | 41 + .../cgroups/ProcSelfCgroupDiscoverer.java | 39 + .../emitter/core/ComposingEmitterTest.java | 100 ++ .../core/CustomEmitterFactoryTest.java | 103 ++ .../java/util/emitter/core/EmitterTest.java | 591 ++++++++++++ .../java/util/emitter/core/GoHandler.java | 77 ++ .../java/util/emitter/core/GoHandlers.java | 125 +++ .../emitter/core/HttpEmitterConfigTest.java | 137 +++ .../util/emitter/core/HttpEmitterTest.java | 86 ++ .../core/HttpPostEmitterStressTest.java | 143 +++ .../java/util/emitter/core/IntEvent.java | 57 ++ .../core/LoggingEmitterConfigTest.java | 90 ++ .../util/emitter/core/MockHttpClient.java | 57 ++ .../ParametrizedUriEmitterConfigTest.java | 71 ++ .../core/ParametrizedUriEmitterTest.java | 216 +++++ .../util/emitter/service/AlertEventTest.java | 181 ++++ .../service/ServiceMetricEventTest.java | 292 ++++++ .../java/util/emitter/service/UnitEvent.java | 84 ++ .../util/http/client/AsyncHttpClientTest.java | 115 +++ .../util/http/client/FriendlyServersTest.java | 309 ++++++ .../util/http/client/JankyServersTest.java | 356 +++++++ .../AppendableByteArrayInputStreamTest.java | 261 ++++++ .../http/client/pool/ResourcePoolTest.java | 328 +++++++ ...equenceInputStreamResponseHandlerTest.java | 226 +++++ .../util/metrics/CpuAcctDeltaMonitorTest.java | 105 +++ .../druid/java/util/metrics/GcNameTest.java | 33 + .../java/util/metrics/JvmMonitorTest.java | 148 +++ .../util/metrics/JvmPidDiscovererTest.java | 32 + .../java/util/metrics/MonitorUtilsTest.java | 46 + .../druid/java/util/metrics/MonitorsTest.java | 66 ++ .../java/util/metrics/SigarLoadTest.java | 35 + .../util/metrics/SigarPidDiscovererTest.java | 32 + .../java/util/metrics/StubServiceEmitter.java | 63 ++ .../util/metrics/cgroups/CpuAcctTest.java | 390 ++++++++ .../cgroups/ProcCgroupDiscovererTest.java | 91 ++ .../java/util/metrics/cgroups/TestUtils.java | 63 ++ .../src/test/resources/cpuacct.usage_all | 129 +++ java-util/src/test/resources/keystore.jks | Bin 0 -> 2331 bytes java-util/src/test/resources/log4j2.xml | 32 + java-util/src/test/resources/proc.cgroups | 13 + java-util/src/test/resources/proc.mounts | 36 + java-util/src/test/resources/proc.pid.cgroup | 11 + pom.xml | 27 +- processing/pom.xml | 14 +- .../druid/query/CPUTimeMetricQueryRunner.java | 2 +- .../io/druid/query/DefaultQueryMetrics.java | 4 +- .../druid/query/ExecutorServiceMonitor.java | 7 +- .../druid/query/FluentQueryRunnerBuilder.java | 2 +- .../query/IntervalChunkingQueryRunner.java | 2 +- .../IntervalChunkingQueryRunnerDecorator.java | 2 +- .../query/MetricsEmittingExecutorService.java | 4 +- .../query/MetricsEmittingQueryRunner.java | 2 +- .../java/io/druid/query/QueryMetrics.java | 4 +- .../java/io/druid/query/RetryQueryRunner.java | 2 +- .../first/DoubleFirstAggregatorFactory.java | 2 +- .../first/FloatFirstAggregatorFactory.java | 2 +- .../first/LongFirstAggregatorFactory.java | 2 +- .../last/DoubleLastAggregatorFactory.java | 2 +- .../last/FloatLastAggregatorFactory.java | 2 +- .../last/LongLastAggregatorFactory.java | 2 +- .../epinephelinae/ByteBufferIntList.java | 2 +- .../io/druid/query/search/AutoStrategy.java | 2 +- .../search/DefaultSearchQueryMetrics.java | 2 +- .../query/search/SearchStrategySelector.java | 2 +- .../select/DefaultSelectQueryMetrics.java | 2 +- .../main/java/io/druid/segment/IndexIO.java | 2 +- .../segment/ReferenceCountingSegment.java | 2 +- .../java/io/druid/query/CachingEmitter.java | 4 +- .../druid/query/DefaultQueryMetricsTest.java | 2 +- .../IntervalChunkingQueryRunnerTest.java | 2 +- .../io/druid/query/QueryRunnerTestHelper.java | 4 +- .../DefaultGroupByQueryMetricsTest.java | 2 +- .../search/DefaultSearchQueryMetricsTest.java | 2 +- .../select/DefaultSelectQueryMetricsTest.java | 2 +- .../DefaultTimeseriesQueryMetricsTest.java | 2 +- .../topn/DefaultTopNQueryMetricsTest.java | 2 +- .../serde/HyperUniquesSerdeForTest.java | 2 +- server/pom.xml | 3 +- .../AbstractCuratorServerInventoryView.java | 2 +- .../client/BatchServerInventoryView.java | 2 +- .../io/druid/client/BrokerServerView.java | 4 +- .../druid/client/CachingClusteredClient.java | 2 +- .../io/druid/client/DirectDruidClient.java | 14 +- ...lteredHttpServerInventoryViewProvider.java | 2 +- .../druid/client/HttpServerInventoryView.java | 12 +- .../HttpServerInventoryViewProvider.java | 2 +- .../io/druid/client/ImmutableDruidServer.java | 6 +- .../client/SingleServerInventoryView.java | 2 +- .../java/io/druid/client/cache/Cache.java | 2 +- .../io/druid/client/cache/CacheMonitor.java | 6 +- .../io/druid/client/cache/CaffeineCache.java | 4 +- .../io/druid/client/cache/HybridCache.java | 2 +- .../java/io/druid/client/cache/MapCache.java | 2 +- .../io/druid/client/cache/MemcachedCache.java | 6 +- .../client/coordinator/CoordinatorClient.java | 2 +- .../indexing/IndexingServiceClient.java | 2 +- .../discovery/CuratorDruidLeaderSelector.java | 2 +- .../discovery/CuratorServiceAnnouncer.java | 2 +- .../io/druid/discovery/DruidLeaderClient.java | 10 +- .../guice/CoordinatorDiscoveryModule.java | 2 +- .../java/io/druid/guice/DruidBinders.java | 2 +- .../guice/IndexingServiceDiscoveryModule.java | 2 +- .../io/druid/guice/http/HttpClientModule.java | 6 +- .../io/druid/guice/http/LifecycleUtils.java | 4 +- .../metadata/SQLMetadataRuleManager.java | 2 +- .../metadata/SQLMetadataSegmentManager.java | 2 +- .../SQLMetadataStorageActionHandler.java | 2 +- .../query/lookup/LookupReferencesManager.java | 4 +- .../SegmentLoaderLocalCacheManager.java | 2 +- .../segment/realtime/RealtimeManager.java | 2 +- .../realtime/RealtimeMetricsMonitor.java | 10 +- .../appenderator/AppenderatorImpl.java | 6 +- .../appenderator/AppenderatorPlumber.java | 2 +- .../realtime/appenderator/Appenderators.java | 2 +- .../DefaultRealtimeAppenderatorFactory.java | 2 +- .../appenderator/SinkQuerySegmentWalker.java | 4 +- .../firehose/CombiningFirehoseFactory.java | 2 +- .../EventReceiverFirehoseFactory.java | 2 +- .../firehose/LocalFirehoseFactory.java | 2 +- .../firehose/TimedShutoffFirehoseFactory.java | 2 +- .../realtime/plumber/FlushingPlumber.java | 4 +- .../plumber/FlushingPlumberSchool.java | 2 +- .../realtime/plumber/RealtimePlumber.java | 4 +- .../plumber/RealtimePlumberSchool.java | 2 +- .../server/AsyncQueryForwardingServlet.java | 4 +- .../server/ClientQuerySegmentWalker.java | 2 +- .../java/io/druid/server/QueryLifecycle.java | 2 +- .../druid/server/QueryLifecycleFactory.java | 2 +- .../java/io/druid/server/QueryResource.java | 2 +- .../java/io/druid/server/SegmentManager.java | 2 +- .../druid/server/audit/SQLAuditManager.java | 4 +- .../server/audit/SQLAuditManagerProvider.java | 2 +- .../coordination/SegmentLoadDropHandler.java | 2 +- .../server/coordination/ServerManager.java | 4 +- .../server/coordination/ZkCoordinator.java | 2 +- .../CachingCostBalancerStrategyFactory.java | 2 +- .../coordinator/CostBalancerStrategy.java | 2 +- .../coordinator/CuratorLoadQueuePeon.java | 2 +- .../server/coordinator/DruidCoordinator.java | 4 +- .../DruidCoordinatorRuntimeParams.java | 2 +- .../server/coordinator/HttpLoadQueuePeon.java | 20 +- .../coordinator/LoadQueueTaskMaster.java | 2 +- .../coordinator/ReplicationThrottler.java | 2 +- .../helper/DruidCoordinatorBalancer.java | 2 +- .../helper/DruidCoordinatorLogger.java | 4 +- .../helper/DruidCoordinatorRuleRunner.java | 4 +- .../helper/DruidCoordinatorSegmentMerger.java | 2 +- .../DruidCoordinatorVersionConverter.java | 2 +- .../rules/BroadcastDistributionRule.java | 2 +- .../server/coordinator/rules/LoadRule.java | 2 +- .../emitter/ComposingEmitterModule.java | 4 +- .../druid/server/emitter/EmitterModule.java | 6 +- .../server/emitter/HttpEmitterModule.java | 6 +- .../server/emitter/HttpEmittingMonitor.java | 16 +- .../server/emitter/LogEmitterModule.java | 6 +- .../server/emitter/NoopEmitterModule.java | 4 +- .../emitter/ParametrizedUriEmitterModule.java | 6 +- .../server/http/SegmentListerResource.java | 2 +- .../jetty/JettyServerModule.java | 8 +- .../server/log/EmittingRequestLogger.java | 6 +- .../log/EmittingRequestLoggerProvider.java | 2 +- .../cache/LookupCoordinatorManager.java | 12 +- .../metrics/DruidMonitorSchedulerConfig.java | 2 +- .../druid/server/metrics/DruidSysMonitor.java | 2 +- .../metrics/EventReceiverFirehoseMonitor.java | 10 +- .../metrics/HistoricalMetricsMonitor.java | 6 +- .../druid/server/metrics/MetricsModule.java | 12 +- .../druid/server/metrics/MonitorsConfig.java | 2 +- .../metrics/QueryCountStatsMonitor.java | 8 +- .../server/router/CoordinatorRuleManager.java | 2 +- .../druid/server/router/QueryHostFinder.java | 2 +- .../router/TieredBrokerHostSelector.java | 2 +- .../io/druid/server/security/Escalator.java | 2 +- .../druid/server/security/NoopEscalator.java | 2 +- .../PreResponseAuthorizationCheckFilter.java | 2 +- .../io/druid/client/BrokerServerViewTest.java | 2 +- .../druid/client/CachingQueryRunnerTest.java | 2 +- .../druid/client/DirectDruidClientTest.java | 8 +- .../client/HttpServerInventoryViewTest.java | 6 +- .../client/cache/MemcachedCacheTest.java | 8 +- .../CuratorDruidLeaderSelectorTest.java | 4 +- .../discovery/DruidLeaderClientTest.java | 4 +- .../ComposingEmitterModuleTest.java | 2 +- .../metadata/MetadataSegmentManagerTest.java | 2 +- .../lookup/LookupReferencesManagerTest.java | 6 +- .../SegmentLoaderLocalCacheManagerTest.java | 2 +- .../appenderator/AppenderatorTester.java | 6 +- .../plumber/RealtimePlumberSchoolTest.java | 2 +- .../io/druid/server/QueryResourceTest.java | 4 +- .../coordination/ServerManagerTest.java | 2 +- .../coordination/ZkCoordinatorTest.java | 2 +- .../DruidCoordinatorBalancerProfiler.java | 4 +- .../DruidCoordinatorRuleRunnerTest.java | 6 +- .../DruidCoordinatorSegmentMergerTest.java | 2 +- .../coordinator/HttpLoadQueuePeonTest.java | 6 +- .../coordinator/rules/LoadRuleTest.java | 8 +- .../server/emitter/EmitterModuleTest.java | 4 +- ...eResponseAuthorizationCheckFilterTest.java | 4 +- .../server/initialization/BaseJettyTest.java | 6 +- .../server/initialization/JettyQosTest.java | 8 +- .../server/initialization/JettyTest.java | 8 +- .../cache/LookupCoordinatorManagerTest.java | 16 +- .../metrics/HistoricalMetricsMonitorTest.java | 6 +- .../server/metrics/NoopServiceEmitter.java | 4 +- .../java/io/druid/cli/CliCoordinator.java | 2 +- .../src/main/java/io/druid/cli/CliRouter.java | 2 +- .../io/druid/sql/avatica/AvaticaMonitor.java | 6 +- .../druid/sql/calcite/schema/DruidSchema.java | 2 +- .../druid/sql/calcite/util/CalciteTests.java | 4 +- 415 files changed, 15753 insertions(+), 580 deletions(-) create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/EmittingLogger.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/BaseHttpEmittingConfig.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/Batch.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/BatchingStrategy.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/ComposingEmitter.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/ConcurrentTimeCounter.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/ContentEncoding.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/EmittedBatchCounter.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/Emitter.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/Emitters.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/Event.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/FeedUriExtractor.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/HttpEmitterConfig.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/LoggingEmitter.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/LoggingEmitterConfig.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/NoopEmitter.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/ParametrizedUriEmitter.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/ParametrizedUriEmitterConfig.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/ParametrizedUriExtractor.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/UriExtractor.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/ZeroCopyByteArrayOutputStream.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/factory/EmitterFactory.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/factory/HttpEmitterFactory.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/factory/LoggingEmitterFactory.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/factory/NoopEmitterFactory.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/factory/ParametrizedUriEmitterFactory.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/service/AlertBuilder.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/service/AlertEvent.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/service/ServiceEmitter.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/service/ServiceEvent.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/service/ServiceEventBuilder.java create mode 100644 java-util/src/main/java/io/druid/java/util/emitter/service/ServiceMetricEvent.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/AbstractHttpClient.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/CredentialedHttpClient.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/EnforceSslHttpClient.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/HttpClient.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/HttpClientConfig.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/HttpClientInit.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/NettyHttpClient.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/Request.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/auth/BasicCredentials.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/auth/Credentials.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/io/AppendableByteArrayInputStream.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/netty/HttpClientPipelineFactory.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/pool/ChannelResourceFactory.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/pool/ResourceContainer.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/pool/ResourceFactory.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/pool/ResourcePool.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/pool/ResourcePoolConfig.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/pool/ResourceVerifier.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/response/ClientResponse.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/response/FullResponseHandler.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/response/FullResponseHolder.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/response/HttpResponseHandler.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/response/InputStreamResponseHandler.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/response/SequenceInputStreamResponseHandler.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/response/StatusResponseHandler.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/response/StatusResponseHolder.java create mode 100644 java-util/src/main/java/io/druid/java/util/http/client/response/ToStringResponseHandler.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/AbstractMonitor.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/CgroupUtil.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/CompoundMonitor.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/CpuAcctDeltaMonitor.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/FeedDefiningMonitor.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/HttpPostEmitterMonitor.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/JvmCpuMonitor.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/JvmMonitor.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/JvmPidDiscoverer.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/JvmThreadsMonitor.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/KeyedDiff.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/Monitor.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/MonitorOfTheMonitors.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/MonitorScheduler.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/MonitorSchedulerConfig.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/MonitorUtils.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/Monitors.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/ParametrizedUriEmitterMonitor.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/PidDiscoverer.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/SigarPidDiscoverer.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/SigarUtil.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/SysMonitor.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/cgroups/CgroupDiscoverer.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/cgroups/CpuAcct.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/cgroups/ProcCgroupDiscoverer.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/cgroups/ProcPidCgroupDiscoverer.java create mode 100644 java-util/src/main/java/io/druid/java/util/metrics/cgroups/ProcSelfCgroupDiscoverer.java create mode 100644 java-util/src/test/java/io/druid/java/util/emitter/core/ComposingEmitterTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/emitter/core/CustomEmitterFactoryTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/emitter/core/EmitterTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/emitter/core/GoHandler.java create mode 100644 java-util/src/test/java/io/druid/java/util/emitter/core/GoHandlers.java create mode 100644 java-util/src/test/java/io/druid/java/util/emitter/core/HttpEmitterConfigTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/emitter/core/HttpEmitterTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/emitter/core/HttpPostEmitterStressTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/emitter/core/IntEvent.java create mode 100644 java-util/src/test/java/io/druid/java/util/emitter/core/LoggingEmitterConfigTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/emitter/core/MockHttpClient.java create mode 100644 java-util/src/test/java/io/druid/java/util/emitter/core/ParametrizedUriEmitterConfigTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/emitter/core/ParametrizedUriEmitterTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/emitter/service/AlertEventTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/emitter/service/ServiceMetricEventTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/emitter/service/UnitEvent.java create mode 100644 java-util/src/test/java/io/druid/java/util/http/client/AsyncHttpClientTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/http/client/FriendlyServersTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/http/client/JankyServersTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/http/client/io/AppendableByteArrayInputStreamTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/http/client/pool/ResourcePoolTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/http/client/response/SequenceInputStreamResponseHandlerTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/metrics/CpuAcctDeltaMonitorTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/metrics/GcNameTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/metrics/JvmMonitorTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/metrics/JvmPidDiscovererTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/metrics/MonitorUtilsTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/metrics/MonitorsTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/metrics/SigarLoadTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/metrics/SigarPidDiscovererTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/metrics/StubServiceEmitter.java create mode 100644 java-util/src/test/java/io/druid/java/util/metrics/cgroups/CpuAcctTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/metrics/cgroups/ProcCgroupDiscovererTest.java create mode 100644 java-util/src/test/java/io/druid/java/util/metrics/cgroups/TestUtils.java create mode 100644 java-util/src/test/resources/cpuacct.usage_all create mode 100644 java-util/src/test/resources/keystore.jks create mode 100644 java-util/src/test/resources/log4j2.xml create mode 100644 java-util/src/test/resources/proc.cgroups create mode 100644 java-util/src/test/resources/proc.mounts create mode 100644 java-util/src/test/resources/proc.pid.cgroup diff --git a/api/src/main/java/io/druid/utils/CompressionUtils.java b/api/src/main/java/io/druid/utils/CompressionUtils.java index 6a551e319e0d..2b25186b25fe 100644 --- a/api/src/main/java/io/druid/utils/CompressionUtils.java +++ b/api/src/main/java/io/druid/utils/CompressionUtils.java @@ -36,26 +36,26 @@ public class CompressionUtils private static final Logger log = new Logger(CompressionUtils.class); - @Deprecated // Use com.metamx.common.CompressionUtils.zip + @Deprecated // Use io.druid.java.util.common.CompressionUtils.zip public static long zip(File directory, File outputZipFile) throws IOException { return io.druid.java.util.common.CompressionUtils.zip(directory, outputZipFile); } - @Deprecated // Use com.metamx.common.CompressionUtils.zip + @Deprecated // Use io.druid.java.util.common.CompressionUtils.zip public static long zip(File directory, OutputStream out) throws IOException { return io.druid.java.util.common.CompressionUtils.zip(directory, out); } - @Deprecated // Use com.metamx.common.CompressionUtils.unzip + @Deprecated // Use io.druid.java.util.common.CompressionUtils.unzip public static void unzip(File pulledFile, File outDir) throws IOException { io.druid.java.util.common.CompressionUtils.unzip(pulledFile, outDir); } - @Deprecated // Use com.metamx.common.CompressionUtils.unzip + @Deprecated // Use io.druid.java.util.common.CompressionUtils.unzip public static void unzip(InputStream in, File outDir) throws IOException { io.druid.java.util.common.CompressionUtils.unzip(in, outDir); @@ -63,8 +63,8 @@ public static void unzip(InputStream in, File outDir) throws IOException /** * Uncompress using a gzip uncompress algorithm from the `pulledFile` to the `outDir`. - * Unlike `com.metamx.common.CompressionUtils.gunzip`, this function takes an output *DIRECTORY* and tries to guess the file name. - * It is recommended that the caller use `com.metamx.common.CompressionUtils.gunzip` and specify the output file themselves to ensure names are as expected + * Unlike `io.druid.java.util.common.CompressionUtils.gunzip`, this function takes an output *DIRECTORY* and tries to guess the file name. + * It is recommended that the caller use `io.druid.java.util.common.CompressionUtils.gunzip` and specify the output file themselves to ensure names are as expected * * @param pulledFile The source file * @param outDir The destination directory to put the resulting file diff --git a/docs/content/configuration/index.md b/docs/content/configuration/index.md index 66b8ca41f76f..6719c574edf0 100644 --- a/docs/content/configuration/index.md +++ b/docs/content/configuration/index.md @@ -180,19 +180,19 @@ Druid nodes periodically emit metrics and different metrics monitors can be incl |Property|Description|Default| |--------|-----------|-------| |`druid.monitoring.emissionPeriod`|How often metrics are emitted.|PT1m| -|`druid.monitoring.monitors`|Sets list of Druid monitors used by a node. See below for names and more information. For example, you can specify monitors for a Broker with `druid.monitoring.monitors=["com.metamx.metrics.SysMonitor","com.metamx.metrics.JvmMonitor"]`.|none (no monitors)| +|`druid.monitoring.monitors`|Sets list of Druid monitors used by a node. See below for names and more information. For example, you can specify monitors for a Broker with `druid.monitoring.monitors=["io.druid.java.util.metrics.SysMonitor","io.druid.java.util.metrics.JvmMonitor"]`.|none (no monitors)| The following monitors are available: |Name|Description| |----|-----------| |`io.druid.client.cache.CacheMonitor`|Emits metrics (to logs) about the segment results cache for Historical and Broker nodes. Reports typical cache statistics include hits, misses, rates, and size (bytes and number of entries), as well as timeouts and and errors.| -|`com.metamx.metrics.SysMonitor`|This uses the [SIGAR library](http://www.hyperic.com/products/sigar) to report on various system activities and statuses.| +|`io.druid.java.util.metrics.SysMonitor`|This uses the [SIGAR library](http://www.hyperic.com/products/sigar) to report on various system activities and statuses.| |`io.druid.server.metrics.HistoricalMetricsMonitor`|Reports statistics on Historical nodes.| -|`com.metamx.metrics.JvmMonitor`|Reports various JVM-related statistics.| -|`com.metamx.metrics.JvmCpuMonitor`|Reports statistics of CPU consumption by the JVM.| -|`com.metamx.metrics.CpuAcctDeltaMonitor`|Reports consumed CPU as per the cpuacct cgroup.| -|`com.metamx.metrics.JvmThreadsMonitor`|Reports Thread statistics in the JVM, like numbers of total, daemon, started, died threads.| +|`io.druid.java.util.metrics.JvmMonitor`|Reports various JVM-related statistics.| +|`io.druid.java.util.metrics.JvmCpuMonitor`|Reports statistics of CPU consumption by the JVM.| +|`io.druid.java.util.metrics.CpuAcctDeltaMonitor`|Reports consumed CPU as per the cpuacct cgroup.| +|`io.druid.java.util.metrics.JvmThreadsMonitor`|Reports Thread statistics in the JVM, like numbers of total, daemon, started, died threads.| |`io.druid.segment.realtime.RealtimeMetricsMonitor`|Reports statistics on Realtime nodes.| |`io.druid.server.metrics.EventReceiverFirehoseMonitor`|Reports how many events have been queued in the EventReceiverFirehose.| |`io.druid.server.metrics.QueryCountStatsMonitor`|Reports how many queries have been successful/failed/interrupted.| diff --git a/docs/content/development/modules.md b/docs/content/development/modules.md index be3773bfaf15..db529f3029a3 100644 --- a/docs/content/development/modules.md +++ b/docs/content/development/modules.md @@ -110,7 +110,7 @@ The following example was retrieved from a historical node configured to use Azu 00Z_2015-04-14T02:41:09.484Z 2015-04-14T02:42:33,463 INFO [ZkCoordinator-0] io.druid.guice.JsonConfigurator - Loaded class[class io.druid.storage.azure.AzureAccountConfig] from props[drui d.azure.] as [io.druid.storage.azure.AzureAccountConfig@759c9ad9] -2015-04-14T02:49:08,275 INFO [ZkCoordinator-0] com.metamx.common.CompressionUtils - Unzipping file[/opt/druid/tmp/compressionUtilZipCache1263964429587449785.z +2015-04-14T02:49:08,275 INFO [ZkCoordinator-0] io.druid.java.util.common.CompressionUtils - Unzipping file[/opt/druid/tmp/compressionUtilZipCache1263964429587449785.z ip] to [/opt/druid/zk_druid/dde/2015-01-02T00:00:00.000Z_2015-01-03T00:00:00.000Z/2015-04-14T02:41:09.484Z/0] 2015-04-14T02:49:08,276 INFO [ZkCoordinator-0] io.druid.storage.azure.AzureDataSegmentPuller - Loaded 1196 bytes from [dde/2015-01-02T00:00:00.000Z_2015-01-03 T00:00:00.000Z/2015-04-14T02:41:09.484Z/0/index.zip] to [/opt/druid/zk_druid/dde/2015-01-02T00:00:00.000Z_2015-01-03T00:00:00.000Z/2015-04-14T02:41:09.484Z/0] diff --git a/docs/content/operations/performance-faq.md b/docs/content/operations/performance-faq.md index edeaf77d8aee..1c27c0d3eb94 100644 --- a/docs/content/operations/performance-faq.md +++ b/docs/content/operations/performance-faq.md @@ -64,7 +64,7 @@ Yes, using a `log4j2.xml` similar to the following causes some of the more chatt - + diff --git a/examples/conf-quickstart/druid/_common/common.runtime.properties b/examples/conf-quickstart/druid/_common/common.runtime.properties index fd131b878cce..ee5b284b527f 100644 --- a/examples/conf-quickstart/druid/_common/common.runtime.properties +++ b/examples/conf-quickstart/druid/_common/common.runtime.properties @@ -113,7 +113,7 @@ druid.selectors.coordinator.serviceName=druid/coordinator # Monitoring # -druid.monitoring.monitors=["com.metamx.metrics.JvmMonitor"] +druid.monitoring.monitors=["io.druid.java.util.metrics.JvmMonitor"] druid.emitter=logging druid.emitter.logging.logLevel=info diff --git a/examples/conf/druid/_common/common.runtime.properties b/examples/conf/druid/_common/common.runtime.properties index a018fa01780c..1c967fcd60cf 100644 --- a/examples/conf/druid/_common/common.runtime.properties +++ b/examples/conf/druid/_common/common.runtime.properties @@ -112,7 +112,7 @@ druid.selectors.coordinator.serviceName=druid/coordinator # Monitoring # -druid.monitoring.monitors=["com.metamx.metrics.JvmMonitor"] +druid.monitoring.monitors=["io.druid.java.util.metrics.JvmMonitor"] druid.emitter=logging druid.emitter.logging.logLevel=info diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index a61468d2f5bd..98e86fcb2491 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -47,8 +47,9 @@ provided - com.metamx + io.druid java-util + ${project.parent.version} provided diff --git a/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/AmbariMetricsEmitter.java b/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/AmbariMetricsEmitter.java index 1838e8952c11..f8f2e76fa1fc 100644 --- a/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/AmbariMetricsEmitter.java +++ b/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/AmbariMetricsEmitter.java @@ -20,11 +20,11 @@ package io.druid.emitter.ambari.metrics; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.metamx.common.ISE; -import com.metamx.emitter.core.Emitter; -import com.metamx.emitter.core.Event; -import com.metamx.emitter.service.AlertEvent; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.common.ISE; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.Event; +import io.druid.java.util.emitter.service.AlertEvent; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; import org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink; diff --git a/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/AmbariMetricsEmitterModule.java b/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/AmbariMetricsEmitterModule.java index ad029a087872..6dc5e49edc49 100644 --- a/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/AmbariMetricsEmitterModule.java +++ b/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/AmbariMetricsEmitterModule.java @@ -28,7 +28,7 @@ import com.google.inject.Provides; import com.google.inject.name.Named; import com.google.inject.name.Names; -import com.metamx.emitter.core.Emitter; +import io.druid.java.util.emitter.core.Emitter; import io.druid.guice.JsonConfigProvider; import io.druid.guice.ManageLifecycle; import io.druid.initialization.DruidModule; diff --git a/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/DruidToTimelineMetricConverter.java b/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/DruidToTimelineMetricConverter.java index bcc09f689054..9912ced68436 100644 --- a/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/DruidToTimelineMetricConverter.java +++ b/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/DruidToTimelineMetricConverter.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; diff --git a/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/SendAllTimelineEventConverter.java b/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/SendAllTimelineEventConverter.java index f5be89a23853..2189fac55f0d 100644 --- a/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/SendAllTimelineEventConverter.java +++ b/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/SendAllTimelineEventConverter.java @@ -26,7 +26,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSortedSet; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; /** diff --git a/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverter.java b/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverter.java index 5d2a46290962..cf4bd86821e6 100644 --- a/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverter.java +++ b/extensions-contrib/ambari-metrics-emitter/src/main/java/io/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverter.java @@ -32,8 +32,8 @@ import com.google.common.collect.ImmutableSortedMap; import com.google.common.io.CharStreams; import com.google.common.io.Files; -import com.metamx.common.ISE; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.common.ISE; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.java.util.common.logger.Logger; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; diff --git a/extensions-contrib/ambari-metrics-emitter/src/test/java/io/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverterTest.java b/extensions-contrib/ambari-metrics-emitter/src/test/java/io/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverterTest.java index eeb4048a9b47..be58f3c1934e 100644 --- a/extensions-contrib/ambari-metrics-emitter/src/test/java/io/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverterTest.java +++ b/extensions-contrib/ambari-metrics-emitter/src/test/java/io/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverterTest.java @@ -20,7 +20,7 @@ package io.druid.emitter.ambari.metrics; import com.google.common.collect.Maps; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.DateTimes; import junitparams.JUnitParamsRunner; diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index 70ac15a00fff..a2029f1e185b 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -53,8 +53,9 @@ provided - com.metamx + io.druid java-util + ${project.parent.version} provided diff --git a/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/DruidToGraphiteEventConverter.java b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/DruidToGraphiteEventConverter.java index 673fa2af2af7..243b6119799a 100644 --- a/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/DruidToGraphiteEventConverter.java +++ b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/DruidToGraphiteEventConverter.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceMetricEvent; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = WhiteListBasedConverter.class) diff --git a/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitter.java b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitter.java index cb0fb30d05bf..bba2796862c5 100644 --- a/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitter.java +++ b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitter.java @@ -23,10 +23,10 @@ import com.codahale.metrics.graphite.GraphiteSender; import com.codahale.metrics.graphite.PickledGraphite; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.metamx.emitter.core.Emitter; -import com.metamx.emitter.core.Event; -import com.metamx.emitter.service.AlertEvent; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.Event; +import io.druid.java.util.emitter.service.AlertEvent; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.java.util.common.ISE; import io.druid.java.util.common.logger.Logger; diff --git a/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitterModule.java b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitterModule.java index b7fd38b3681f..12b27213c791 100644 --- a/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitterModule.java +++ b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/GraphiteEmitterModule.java @@ -29,7 +29,7 @@ import com.google.inject.Provides; import com.google.inject.name.Named; import com.google.inject.name.Names; -import com.metamx.emitter.core.Emitter; +import io.druid.java.util.emitter.core.Emitter; import io.druid.guice.JsonConfigProvider; import io.druid.guice.ManageLifecycle; import io.druid.initialization.DruidModule; diff --git a/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/SendAllGraphiteEventConverter.java b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/SendAllGraphiteEventConverter.java index dc26014e1b3f..e3b705684291 100644 --- a/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/SendAllGraphiteEventConverter.java +++ b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/SendAllGraphiteEventConverter.java @@ -26,7 +26,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSortedSet; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import java.util.concurrent.TimeUnit; diff --git a/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/WhiteListBasedConverter.java b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/WhiteListBasedConverter.java index ac53033ca6a3..25018c5f5048 100644 --- a/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/WhiteListBasedConverter.java +++ b/extensions-contrib/graphite-emitter/src/main/java/io/druid/emitter/graphite/WhiteListBasedConverter.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableSortedMap; import com.google.common.io.Files; import com.google.common.io.Resources; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.java.util.common.ISE; import io.druid.java.util.common.logger.Logger; diff --git a/extensions-contrib/graphite-emitter/src/test/java/io/druid/emitter/graphite/WhiteListBasedConverterTest.java b/extensions-contrib/graphite-emitter/src/test/java/io/druid/emitter/graphite/WhiteListBasedConverterTest.java index 11908fb1bb23..10022a4023b8 100644 --- a/extensions-contrib/graphite-emitter/src/test/java/io/druid/emitter/graphite/WhiteListBasedConverterTest.java +++ b/extensions-contrib/graphite-emitter/src/test/java/io/druid/emitter/graphite/WhiteListBasedConverterTest.java @@ -20,7 +20,7 @@ package io.druid.emitter.graphite; import com.google.common.collect.Maps; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.DateTimes; import junitparams.JUnitParamsRunner; diff --git a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml index 4e7ce512421b..1804c8267904 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml +++ b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml @@ -38,8 +38,9 @@ provided - com.metamx + io.druid java-util + ${project.parent.version} provided diff --git a/extensions-contrib/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaEightSimpleConsumerFirehoseFactory.java b/extensions-contrib/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaEightSimpleConsumerFirehoseFactory.java index 0c53007311d8..c65fa808d70c 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaEightSimpleConsumerFirehoseFactory.java +++ b/extensions-contrib/kafka-eight-simpleConsumer/src/main/java/io/druid/firehose/kafka/KafkaEightSimpleConsumerFirehoseFactory.java @@ -25,8 +25,8 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Maps; import com.google.common.io.Closeables; -import com.metamx.common.parsers.ParseException; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.common.parsers.ParseException; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.data.input.ByteBufferInputRowParser; import io.druid.data.input.Committer; import io.druid.data.input.FirehoseFactoryV2; diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index cec80c6dcd5a..a70ac2a9c6da 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -52,8 +52,9 @@ provided - com.metamx + io.druid java-util + ${project.parent.version} provided diff --git a/extensions-contrib/kafka-emitter/src/main/java/io/druid/emitter/kafka/KafkaEmitter.java b/extensions-contrib/kafka-emitter/src/main/java/io/druid/emitter/kafka/KafkaEmitter.java index 7a791be5c84f..7628521dd8dc 100644 --- a/extensions-contrib/kafka-emitter/src/main/java/io/druid/emitter/kafka/KafkaEmitter.java +++ b/extensions-contrib/kafka-emitter/src/main/java/io/druid/emitter/kafka/KafkaEmitter.java @@ -22,10 +22,10 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import com.metamx.emitter.core.Emitter; -import com.metamx.emitter.core.Event; -import com.metamx.emitter.service.AlertEvent; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.Event; +import io.druid.java.util.emitter.service.AlertEvent; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.emitter.kafka.MemoryBoundLinkedBlockingQueue.ObjectContainer; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.lifecycle.LifecycleStart; diff --git a/extensions-contrib/kafka-emitter/src/main/java/io/druid/emitter/kafka/KafkaEmitterModule.java b/extensions-contrib/kafka-emitter/src/main/java/io/druid/emitter/kafka/KafkaEmitterModule.java index 0608b1f0d898..7db5724787f0 100644 --- a/extensions-contrib/kafka-emitter/src/main/java/io/druid/emitter/kafka/KafkaEmitterModule.java +++ b/extensions-contrib/kafka-emitter/src/main/java/io/druid/emitter/kafka/KafkaEmitterModule.java @@ -24,7 +24,7 @@ import com.google.inject.Binder; import com.google.inject.Provides; import com.google.inject.name.Named; -import com.metamx.emitter.core.Emitter; +import io.druid.java.util.emitter.core.Emitter; import io.druid.guice.JsonConfigProvider; import io.druid.guice.ManageLifecycle; import io.druid.initialization.DruidModule; diff --git a/extensions-contrib/redis-cache/src/main/java/io/druid/client/cache/RedisCache.java b/extensions-contrib/redis-cache/src/main/java/io/druid/client/cache/RedisCache.java index 11b63a2e0f92..9b4744fd961c 100644 --- a/extensions-contrib/redis-cache/src/main/java/io/druid/client/cache/RedisCache.java +++ b/extensions-contrib/redis-cache/src/main/java/io/druid/client/cache/RedisCache.java @@ -21,8 +21,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.java.util.common.logger.Logger; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index 542972d2bd6f..7badd860cade 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -45,8 +45,9 @@ provided - com.metamx + io.druid java-util + ${project.parent.version} provided diff --git a/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitter.java b/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitter.java index c949f13f4778..d395146f87e2 100644 --- a/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitter.java +++ b/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitter.java @@ -22,9 +22,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; -import com.metamx.emitter.core.Emitter; -import com.metamx.emitter.core.Event; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.Event; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import com.timgroup.statsd.NonBlockingStatsDClient; import com.timgroup.statsd.StatsDClient; import com.timgroup.statsd.StatsDClientErrorHandler; diff --git a/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitterModule.java b/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitterModule.java index 3d00da34eff1..2d8bd2087c0c 100644 --- a/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitterModule.java +++ b/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitterModule.java @@ -24,7 +24,7 @@ import com.google.inject.Binder; import com.google.inject.Provides; import com.google.inject.name.Named; -import com.metamx.emitter.core.Emitter; +import io.druid.java.util.emitter.core.Emitter; import io.druid.guice.JsonConfigProvider; import io.druid.guice.ManageLifecycle; import io.druid.initialization.DruidModule; diff --git a/extensions-contrib/statsd-emitter/src/test/java/DimensionConverterTest.java b/extensions-contrib/statsd-emitter/src/test/java/DimensionConverterTest.java index b4c8fa33243b..62132a266cf1 100644 --- a/extensions-contrib/statsd-emitter/src/test/java/DimensionConverterTest.java +++ b/extensions-contrib/statsd-emitter/src/test/java/DimensionConverterTest.java @@ -19,7 +19,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.emitter.statsd.DimensionConverter; import io.druid.emitter.statsd.StatsDMetric; import io.druid.java.util.common.DateTimes; diff --git a/extensions-contrib/statsd-emitter/src/test/java/StatsDEmitterTest.java b/extensions-contrib/statsd-emitter/src/test/java/StatsDEmitterTest.java index 531ca7a5dbf5..16175d917a4e 100644 --- a/extensions-contrib/statsd-emitter/src/test/java/StatsDEmitterTest.java +++ b/extensions-contrib/statsd-emitter/src/test/java/StatsDEmitterTest.java @@ -18,7 +18,7 @@ */ import com.fasterxml.jackson.databind.ObjectMapper; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import com.timgroup.statsd.StatsDClient; import io.druid.emitter.statsd.StatsDEmitter; import io.druid.emitter.statsd.StatsDEmitterConfig; diff --git a/extensions-contrib/thrift-extensions/src/main/java/io/druid/data/input/thrift/ThriftInputRowParser.java b/extensions-contrib/thrift-extensions/src/main/java/io/druid/data/input/thrift/ThriftInputRowParser.java index 87169252b8f8..aa523f34993f 100644 --- a/extensions-contrib/thrift-extensions/src/main/java/io/druid/data/input/thrift/ThriftInputRowParser.java +++ b/extensions-contrib/thrift-extensions/src/main/java/io/druid/data/input/thrift/ThriftInputRowParser.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.metamx.common.IAE; +import io.druid.java.util.common.IAE; import com.twitter.elephantbird.mapreduce.io.ThriftWritable; import io.druid.data.input.InputRow; import io.druid.data.input.MapBasedInputRow; diff --git a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java index c17b219a0389..79950efd0875 100644 --- a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java +++ b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/MapVirtualColumn.java @@ -24,7 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; -import com.metamx.common.StringUtils; +import io.druid.java.util.common.StringUtils; import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.DimFilterUtils; diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java index 7ebb40b5d7a5..bfbaaac86f7d 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorFactory.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.common.IAE; +import io.druid.java.util.common.IAE; import com.yahoo.sketches.Util; import com.yahoo.sketches.quantiles.DoublesSketch; import com.yahoo.sketches.quantiles.DoublesUnion; diff --git a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/CommonCacheNotifier.java b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/CommonCacheNotifier.java index 4b7a557197ca..9554928ba60b 100644 --- a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/CommonCacheNotifier.java +++ b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/CommonCacheNotifier.java @@ -21,12 +21,12 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.metamx.emitter.EmittingLogger; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.ClientResponse; -import com.metamx.http.client.response.HttpResponseHandler; -import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.ClientResponse; +import io.druid.java.util.http.client.response.HttpResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHolder; import io.druid.discovery.DiscoveryDruidNode; import io.druid.discovery.DruidNodeDiscovery; import io.druid.discovery.DruidNodeDiscoveryProvider; diff --git a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BasicHTTPEscalator.java b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BasicHTTPEscalator.java index 4029537f5499..45e3a5db7bf6 100644 --- a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BasicHTTPEscalator.java +++ b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BasicHTTPEscalator.java @@ -22,9 +22,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; -import com.metamx.http.client.CredentialedHttpClient; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.auth.BasicCredentials; +import io.druid.java.util.http.client.CredentialedHttpClient; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.auth.BasicCredentials; import io.druid.server.security.AuthenticationResult; import io.druid.server.security.Escalator; diff --git a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BytesFullResponseHandler.java b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BytesFullResponseHandler.java index d8b65a5e8ff0..c548cfc0ac1a 100644 --- a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BytesFullResponseHandler.java +++ b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BytesFullResponseHandler.java @@ -19,9 +19,9 @@ package io.druid.security.basic.authentication; -import com.metamx.http.client.response.ClientResponse; -import com.metamx.http.client.response.FullResponseHolder; -import com.metamx.http.client.response.HttpResponseHandler; +import io.druid.java.util.http.client.response.ClientResponse; +import io.druid.java.util.http.client.response.FullResponseHolder; +import io.druid.java.util.http.client.response.HttpResponseHandler; import org.jboss.netty.handler.codec.http.HttpChunk; import org.jboss.netty.handler.codec.http.HttpResponse; diff --git a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BytesFullResponseHolder.java b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BytesFullResponseHolder.java index a273f5030982..b701468b461c 100644 --- a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BytesFullResponseHolder.java +++ b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BytesFullResponseHolder.java @@ -19,7 +19,7 @@ package io.druid.security.basic.authentication; -import com.metamx.http.client.response.FullResponseHolder; +import io.druid.java.util.http.client.response.FullResponseHolder; import org.jboss.netty.handler.codec.http.HttpResponse; import org.jboss.netty.handler.codec.http.HttpResponseStatus; diff --git a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/db/cache/CoordinatorBasicAuthenticatorCacheNotifier.java b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/db/cache/CoordinatorBasicAuthenticatorCacheNotifier.java index 2eff574cfd91..5ab501b5166f 100644 --- a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/db/cache/CoordinatorBasicAuthenticatorCacheNotifier.java +++ b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/db/cache/CoordinatorBasicAuthenticatorCacheNotifier.java @@ -21,8 +21,8 @@ import com.google.common.base.Preconditions; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.http.client.HttpClient; import io.druid.concurrent.LifecycleLock; import io.druid.discovery.DruidNodeDiscoveryProvider; import io.druid.guice.ManageLifecycle; diff --git a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/db/cache/CoordinatorPollingBasicAuthenticatorCacheManager.java b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/db/cache/CoordinatorPollingBasicAuthenticatorCacheManager.java index dd8bc10501a1..bc9a713b9c14 100644 --- a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/db/cache/CoordinatorPollingBasicAuthenticatorCacheManager.java +++ b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/db/cache/CoordinatorPollingBasicAuthenticatorCacheManager.java @@ -23,8 +23,8 @@ import com.google.common.base.Preconditions; import com.google.inject.Inject; import com.google.inject.Injector; -import com.metamx.emitter.EmittingLogger; -import com.metamx.http.client.Request; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.http.client.Request; import io.druid.client.coordinator.Coordinator; import io.druid.concurrent.LifecycleLock; import io.druid.discovery.DruidLeaderClient; diff --git a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/db/updater/CoordinatorBasicAuthenticatorMetadataStorageUpdater.java b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/db/updater/CoordinatorBasicAuthenticatorMetadataStorageUpdater.java index eacc02422373..872cdc103208 100644 --- a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/db/updater/CoordinatorBasicAuthenticatorMetadataStorageUpdater.java +++ b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/db/updater/CoordinatorBasicAuthenticatorMetadataStorageUpdater.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.common.config.ConfigManager; import io.druid.concurrent.LifecycleLock; import io.druid.guice.ManageLifecycle; diff --git a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authorization/db/cache/CoordinatorBasicAuthorizerCacheNotifier.java b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authorization/db/cache/CoordinatorBasicAuthorizerCacheNotifier.java index 924107cf92a2..9f7a1def62a3 100644 --- a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authorization/db/cache/CoordinatorBasicAuthorizerCacheNotifier.java +++ b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authorization/db/cache/CoordinatorBasicAuthorizerCacheNotifier.java @@ -21,8 +21,8 @@ import com.google.common.base.Preconditions; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.http.client.HttpClient; import io.druid.concurrent.LifecycleLock; import io.druid.discovery.DruidNodeDiscoveryProvider; import io.druid.guice.ManageLifecycle; diff --git a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authorization/db/cache/CoordinatorPollingBasicAuthorizerCacheManager.java b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authorization/db/cache/CoordinatorPollingBasicAuthorizerCacheManager.java index a8c4a1837aa5..5f993d3121f3 100644 --- a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authorization/db/cache/CoordinatorPollingBasicAuthorizerCacheManager.java +++ b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authorization/db/cache/CoordinatorPollingBasicAuthorizerCacheManager.java @@ -23,8 +23,8 @@ import com.google.common.base.Preconditions; import com.google.inject.Inject; import com.google.inject.Injector; -import com.metamx.emitter.EmittingLogger; -import com.metamx.http.client.Request; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.http.client.Request; import io.druid.client.coordinator.Coordinator; import io.druid.concurrent.LifecycleLock; import io.druid.discovery.DruidLeaderClient; diff --git a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authorization/db/updater/CoordinatorBasicAuthorizerMetadataStorageUpdater.java b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authorization/db/updater/CoordinatorBasicAuthorizerMetadataStorageUpdater.java index ca80d19712d6..adc0a60591c5 100644 --- a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authorization/db/updater/CoordinatorBasicAuthorizerMetadataStorageUpdater.java +++ b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authorization/db/updater/CoordinatorBasicAuthorizerMetadataStorageUpdater.java @@ -23,7 +23,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.common.config.ConfigManager; import io.druid.concurrent.LifecycleLock; import io.druid.guice.ManageLifecycle; diff --git a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosEscalator.java b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosEscalator.java index 887b5296d04e..808e1169b64b 100644 --- a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosEscalator.java +++ b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosEscalator.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClient; import io.druid.java.util.common.logger.Logger; import io.druid.server.security.AuthenticationResult; import io.druid.server.security.Escalator; diff --git a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosHttpClient.java b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosHttpClient.java index c79deaf60e34..b1a66a8b2a75 100644 --- a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosHttpClient.java +++ b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosHttpClient.java @@ -24,10 +24,10 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; -import com.metamx.http.client.AbstractHttpClient; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.HttpResponseHandler; +import io.druid.java.util.http.client.AbstractHttpClient; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.HttpResponseHandler; import io.druid.java.util.common.concurrent.Execs; import io.druid.java.util.common.logger.Logger; import org.apache.hadoop.security.UserGroupInformation; diff --git a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/ResponseCookieHandler.java b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/ResponseCookieHandler.java index 5a4d4df0b43e..755cdb96547d 100644 --- a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/ResponseCookieHandler.java +++ b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/ResponseCookieHandler.java @@ -21,8 +21,8 @@ import com.google.common.base.Function; import com.google.common.collect.Maps; -import com.metamx.http.client.response.ClientResponse; -import com.metamx.http.client.response.HttpResponseHandler; +import io.druid.java.util.http.client.response.ClientResponse; +import io.druid.java.util.http.client.response.HttpResponseHandler; import io.druid.java.util.common.logger.Logger; import org.jboss.netty.handler.codec.http.HttpChunk; import org.jboss.netty.handler.codec.http.HttpHeaders; diff --git a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/RetryIfUnauthorizedResponseHandler.java b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/RetryIfUnauthorizedResponseHandler.java index 8d8148842c8b..01a7987e2bac 100644 --- a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/RetryIfUnauthorizedResponseHandler.java +++ b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/RetryIfUnauthorizedResponseHandler.java @@ -19,8 +19,8 @@ package io.druid.security.kerberos; -import com.metamx.http.client.response.ClientResponse; -import com.metamx.http.client.response.HttpResponseHandler; +import io.druid.java.util.http.client.response.ClientResponse; +import io.druid.java.util.http.client.response.HttpResponseHandler; import io.druid.java.util.common.logger.Logger; import org.jboss.netty.handler.codec.http.HttpChunk; import org.jboss.netty.handler.codec.http.HttpResponse; diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index 4d461c297102..ccbe4daf484a 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -44,6 +44,12 @@ ${project.parent.version} provided + + io.druid + java-util + ${project.parent.version} + provided + org.apache.hadoop hadoop-client @@ -135,11 +141,6 @@ - - com.metamx - java-util - provided - org.apache.hadoop hadoop-aws diff --git a/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java index d910ae51b0ac..9ce604e7d203 100644 --- a/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java +++ b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java @@ -20,7 +20,7 @@ package io.druid.storage.hdfs; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.segment.loading.DataSegmentKiller; import io.druid.segment.loading.SegmentLoadingException; import io.druid.timeline.DataSegment; diff --git a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java index 4a2ecbbae10c..26dfb19b712b 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java +++ b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java @@ -44,7 +44,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.data.input.Committer; import io.druid.data.input.InputRow; import io.druid.data.input.impl.InputRowParser; diff --git a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTaskClient.java b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTaskClient.java index 30a06de8c17d..6525d1276318 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTaskClient.java +++ b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTaskClient.java @@ -29,11 +29,11 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.emitter.EmittingLogger; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.FullResponseHandler; -import com.metamx.http.client.response.FullResponseHolder; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.FullResponseHandler; +import io.druid.java.util.http.client.response.FullResponseHolder; import io.druid.indexing.common.RetryPolicy; import io.druid.indexing.common.RetryPolicyConfig; import io.druid.indexing.common.RetryPolicyFactory; diff --git a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTaskClientFactory.java b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTaskClientFactory.java index 869392642003..da4d8c369f94 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTaskClientFactory.java +++ b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTaskClientFactory.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClient; import io.druid.guice.annotations.EscalatedGlobal; import io.druid.guice.annotations.Json; import io.druid.indexing.common.TaskInfoProvider; diff --git a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisor.java b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisor.java index 2a36f251d0a0..7cc79f6a98ad 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisor.java +++ b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisor.java @@ -43,9 +43,9 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.indexing.common.TaskInfoProvider; import io.druid.indexer.TaskLocation; import io.druid.indexing.common.TaskStatus; diff --git a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisorSpec.java b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisorSpec.java index d11709ecc620..2b540e537104 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisorSpec.java +++ b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/supervisor/KafkaSupervisorSpec.java @@ -25,7 +25,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.guice.annotations.Json; import io.druid.indexing.kafka.KafkaIndexTaskClientFactory; import io.druid.indexing.overlord.IndexerMetadataStorageCoordinator; diff --git a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskClientTest.java b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskClientTest.java index 8d328aadbad2..2834cc838a89 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskClientTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskClientTest.java @@ -27,10 +27,10 @@ import com.google.common.collect.Maps; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.FullResponseHandler; -import com.metamx.http.client.response.FullResponseHolder; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.FullResponseHandler; +import io.druid.java.util.http.client.response.FullResponseHolder; import io.druid.indexing.common.TaskInfoProvider; import io.druid.indexer.TaskLocation; import io.druid.indexing.common.TaskStatus; diff --git a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java index 40986a95aab3..ad6459f526f6 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java @@ -38,10 +38,10 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.core.NoopEmitter; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.metrics.MonitorScheduler; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.core.NoopEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.metrics.MonitorScheduler; import io.druid.client.cache.CacheConfig; import io.druid.client.cache.MapCache; import io.druid.data.input.impl.DimensionsSpec; diff --git a/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/CacheScheduler.java b/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/CacheScheduler.java index 38fcce978aac..7669168d1f4a 100644 --- a/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/CacheScheduler.java +++ b/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/CacheScheduler.java @@ -22,8 +22,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Throwables; import com.google.inject.Inject; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.guice.LazySingleton; import io.druid.java.util.common.ISE; import io.druid.java.util.common.StringUtils; diff --git a/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/NamespaceExtractionCacheManager.java b/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/NamespaceExtractionCacheManager.java index 65f646d5af65..49047294c9c0 100644 --- a/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/NamespaceExtractionCacheManager.java +++ b/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/NamespaceExtractionCacheManager.java @@ -22,7 +22,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Throwables; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.java.util.common.concurrent.ExecutorServices; import io.druid.java.util.common.lifecycle.Lifecycle; import io.druid.java.util.common.logger.Logger; diff --git a/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/OffHeapNamespaceExtractionCacheManager.java b/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/OffHeapNamespaceExtractionCacheManager.java index e591f8da90d3..ab069001e39d 100644 --- a/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/OffHeapNamespaceExtractionCacheManager.java +++ b/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/OffHeapNamespaceExtractionCacheManager.java @@ -21,8 +21,8 @@ import com.google.common.base.Throwables; import com.google.inject.Inject; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.java.util.common.lifecycle.Lifecycle; import io.druid.java.util.common.logger.Logger; import io.druid.server.lookup.namespace.NamespaceExtractionConfig; diff --git a/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/OnHeapNamespaceExtractionCacheManager.java b/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/OnHeapNamespaceExtractionCacheManager.java index 6a5e015f2521..d538c8f2da87 100644 --- a/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/OnHeapNamespaceExtractionCacheManager.java +++ b/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/OnHeapNamespaceExtractionCacheManager.java @@ -21,8 +21,8 @@ import com.google.common.primitives.Chars; import com.google.inject.Inject; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.java.util.common.ISE; import io.druid.java.util.common.lifecycle.Lifecycle; import io.druid.java.util.common.logger.Logger; diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index 207369684c12..49ca28462df5 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -44,15 +44,15 @@ ${project.parent.version} provided - - net.java.dev.jets3t - jets3t + io.druid + java-util + ${project.parent.version} provided - com.metamx - java-util + net.java.dev.jets3t + jets3t provided diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java index 333e7fae5dd1..3473cffe4a80 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.CompressionUtils; import io.druid.java.util.common.StringUtils; import io.druid.segment.SegmentUtils; diff --git a/extensions-core/simple-client-sslcontext/pom.xml b/extensions-core/simple-client-sslcontext/pom.xml index 3f2defdd9da3..16eed3db9e10 100644 --- a/extensions-core/simple-client-sslcontext/pom.xml +++ b/extensions-core/simple-client-sslcontext/pom.xml @@ -33,8 +33,9 @@ provided - com.metamx + io.druid java-util + ${project.parent.version} provided diff --git a/extensions-core/simple-client-sslcontext/src/main/java/io/druid/https/SSLContextProvider.java b/extensions-core/simple-client-sslcontext/src/main/java/io/druid/https/SSLContextProvider.java index ccdacf1b733d..ee53c9278e0c 100644 --- a/extensions-core/simple-client-sslcontext/src/main/java/io/druid/https/SSLContextProvider.java +++ b/extensions-core/simple-client-sslcontext/src/main/java/io/druid/https/SSLContextProvider.java @@ -22,7 +22,7 @@ import com.google.common.base.Throwables; import com.google.inject.Inject; import com.google.inject.Provider; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManagerFactory; diff --git a/indexing-service/src/main/java/io/druid/indexing/common/TaskToolbox.java b/indexing-service/src/main/java/io/druid/indexing/common/TaskToolbox.java index 3db6e0f7b907..dd132769192f 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/TaskToolbox.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/TaskToolbox.java @@ -27,8 +27,8 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Multimaps; import com.google.inject.Provider; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.metrics.MonitorScheduler; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.metrics.MonitorScheduler; import io.druid.client.cache.Cache; import io.druid.client.cache.CacheConfig; import io.druid.discovery.DataNodeService; diff --git a/indexing-service/src/main/java/io/druid/indexing/common/TaskToolboxFactory.java b/indexing-service/src/main/java/io/druid/indexing/common/TaskToolboxFactory.java index 43d2abea594c..c17b23fe210a 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/TaskToolboxFactory.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/TaskToolboxFactory.java @@ -23,8 +23,8 @@ import com.google.common.base.Preconditions; import com.google.inject.Inject; import com.google.inject.Provider; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.metrics.MonitorScheduler; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.metrics.MonitorScheduler; import io.druid.client.cache.Cache; import io.druid.client.cache.CacheConfig; import io.druid.discovery.DataNodeService; diff --git a/indexing-service/src/main/java/io/druid/indexing/common/actions/LocalTaskActionClient.java b/indexing-service/src/main/java/io/druid/indexing/common/actions/LocalTaskActionClient.java index 7bce2b6e89ba..0e688ecbfd82 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/actions/LocalTaskActionClient.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/actions/LocalTaskActionClient.java @@ -19,7 +19,7 @@ package io.druid.indexing.common.actions; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.indexing.common.task.Task; import io.druid.indexing.overlord.TaskStorage; import io.druid.java.util.common.ISE; diff --git a/indexing-service/src/main/java/io/druid/indexing/common/actions/RemoteTaskActionClient.java b/indexing-service/src/main/java/io/druid/indexing/common/actions/RemoteTaskActionClient.java index b2e3c5f23c3c..9c1c7e823993 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/actions/RemoteTaskActionClient.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/actions/RemoteTaskActionClient.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; -import com.metamx.http.client.response.FullResponseHolder; +import io.druid.java.util.http.client.response.FullResponseHolder; import io.druid.discovery.DruidLeaderClient; import io.druid.indexing.common.RetryPolicy; import io.druid.indexing.common.RetryPolicyFactory; diff --git a/indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentMetadataUpdateAction.java b/indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentMetadataUpdateAction.java index 2e33d5e15c09..7c55939f4d56 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentMetadataUpdateAction.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentMetadataUpdateAction.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.collect.ImmutableSet; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.indexing.common.task.Task; import io.druid.indexing.overlord.CriticalAction; import io.druid.java.util.common.ISE; diff --git a/indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentNukeAction.java b/indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentNukeAction.java index 044cfebd00fa..dad3ca516460 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentNukeAction.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentNukeAction.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.collect.ImmutableSet; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.indexing.common.task.Task; import io.druid.indexing.overlord.CriticalAction; import io.druid.java.util.common.ISE; diff --git a/indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentTransactionalInsertAction.java b/indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentTransactionalInsertAction.java index c22275def06a..a65498c6a277 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentTransactionalInsertAction.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentTransactionalInsertAction.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.collect.ImmutableSet; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.indexing.common.task.Task; import io.druid.indexing.overlord.CriticalAction; import io.druid.indexing.overlord.DataSourceMetadata; diff --git a/indexing-service/src/main/java/io/druid/indexing/common/actions/TaskActionToolbox.java b/indexing-service/src/main/java/io/druid/indexing/common/actions/TaskActionToolbox.java index 59ca22942159..4d08465b99e4 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/actions/TaskActionToolbox.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/actions/TaskActionToolbox.java @@ -20,7 +20,7 @@ package io.druid.indexing.common.actions; import com.google.inject.Inject; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import io.druid.indexing.overlord.TaskLockbox; import io.druid.indexing.overlord.supervisor.SupervisorManager; diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java b/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java index 2b5574127d65..747cf58d4162 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java @@ -34,9 +34,9 @@ import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import com.google.common.hash.Hashing; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.indexing.common.TaskLock; import io.druid.indexing.common.TaskStatus; import io.druid.indexing.common.TaskToolbox; diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/RealtimeIndexTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/RealtimeIndexTask.java index 0a3a8b48547b..2a6b3b4723ac 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/RealtimeIndexTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/RealtimeIndexTask.java @@ -28,7 +28,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.primitives.Ints; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.data.input.Committer; import io.druid.data.input.Firehose; import io.druid.data.input.FirehoseFactory; diff --git a/indexing-service/src/main/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactory.java b/indexing-service/src/main/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactory.java index b6516c9a47e6..61c245c57d91 100644 --- a/indexing-service/src/main/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactory.java +++ b/indexing-service/src/main/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactory.java @@ -30,7 +30,7 @@ import com.google.common.collect.HashBiMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.data.input.Firehose; import io.druid.data.input.FirehoseFactory; import io.druid.data.input.impl.InputRowParser; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/ForkingTaskRunner.java b/indexing-service/src/main/java/io/druid/indexing/overlord/ForkingTaskRunner.java index ee103420207d..7509c8cd8152 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/ForkingTaskRunner.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/ForkingTaskRunner.java @@ -41,7 +41,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.concurrent.Execs; import io.druid.guice.annotations.Self; import io.druid.indexer.TaskLocation; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/MetadataTaskStorage.java b/indexing-service/src/main/java/io/druid/indexing/overlord/MetadataTaskStorage.java index 479b92e92f38..011aaa5d167e 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/MetadataTaskStorage.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/MetadataTaskStorage.java @@ -28,7 +28,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.indexing.common.TaskLock; import io.druid.indexing.common.TaskStatus; import io.druid.indexing.common.actions.TaskAction; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java b/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java index c04300571d49..53f5e227a5b1 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunner.java @@ -43,12 +43,12 @@ import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; -import com.metamx.emitter.EmittingLogger; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.InputStreamResponseHandler; -import com.metamx.http.client.response.StatusResponseHandler; -import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.InputStreamResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHolder; import io.druid.java.util.common.concurrent.Execs; import io.druid.concurrent.LifecycleLock; import io.druid.curator.CuratorUtils; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunnerFactory.java b/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunnerFactory.java index 5e0b165c0cab..44e7d5015763 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunnerFactory.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/RemoteTaskRunnerFactory.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Supplier; import com.google.inject.Inject; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClient; import io.druid.curator.cache.PathChildrenCacheFactory; import io.druid.guice.annotations.EscalatedGlobal; import io.druid.indexing.overlord.autoscaling.NoopProvisioningStrategy; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/TaskLockbox.java b/indexing-service/src/main/java/io/druid/indexing/overlord/TaskLockbox.java index ef478256d578..5a6c2a297733 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/TaskLockbox.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/TaskLockbox.java @@ -31,7 +31,7 @@ import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.indexing.common.TaskLock; import io.druid.indexing.common.TaskLockType; import io.druid.indexing.common.task.Task; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/TaskMaster.java b/indexing-service/src/main/java/io/druid/indexing/overlord/TaskMaster.java index 4fa12b6decff..f1d14e10f2b0 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/TaskMaster.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/TaskMaster.java @@ -22,8 +22,8 @@ import com.google.common.base.Optional; import com.google.common.base.Throwables; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.indexing.IndexingService; import io.druid.curator.discovery.ServiceAnnouncer; import io.druid.discovery.DruidLeaderSelector; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/TaskQueue.java b/indexing-service/src/main/java/io/druid/indexing/overlord/TaskQueue.java index a360f5c677bf..56b5b2f3710c 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/TaskQueue.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/TaskQueue.java @@ -33,9 +33,9 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.indexing.common.TaskStatus; import io.druid.indexing.common.actions.TaskActionClientFactory; import io.druid.indexing.common.task.Task; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/TaskRunnerUtils.java b/indexing-service/src/main/java/io/druid/indexing/overlord/TaskRunnerUtils.java index 2a6d917d4d73..8067c96edbd1 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/TaskRunnerUtils.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/TaskRunnerUtils.java @@ -19,7 +19,7 @@ package io.druid.indexing.overlord; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.indexer.TaskLocation; import io.druid.indexing.common.TaskStatus; import io.druid.java.util.common.Pair; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/ThreadPoolTaskRunner.java b/indexing-service/src/main/java/io/druid/indexing/overlord/ThreadPoolTaskRunner.java index b7d7008b0937..d24d7e4d8015 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/ThreadPoolTaskRunner.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/ThreadPoolTaskRunner.java @@ -29,9 +29,9 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.java.util.common.concurrent.Execs; import io.druid.concurrent.TaskThreadPriority; import io.druid.guice.annotations.Self; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/AbstractWorkerProvisioningStrategy.java b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/AbstractWorkerProvisioningStrategy.java index ce53188416b1..5ab74b5dd12a 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/AbstractWorkerProvisioningStrategy.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/AbstractWorkerProvisioningStrategy.java @@ -20,7 +20,7 @@ package io.druid.indexing.overlord.autoscaling; import com.google.common.base.Supplier; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.indexing.overlord.WorkerTaskRunner; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.granularity.PeriodGranularity; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/NoopAutoScaler.java b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/NoopAutoScaler.java index a83dc35f1757..a4330c50df5d 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/NoopAutoScaler.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/NoopAutoScaler.java @@ -19,7 +19,7 @@ package io.druid.indexing.overlord.autoscaling; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.UOE; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/PendingTaskBasedWorkerProvisioningStrategy.java b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/PendingTaskBasedWorkerProvisioningStrategy.java index 476f6fdcb9c2..916833b24aaa 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/PendingTaskBasedWorkerProvisioningStrategy.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/PendingTaskBasedWorkerProvisioningStrategy.java @@ -30,8 +30,8 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.inject.Inject; -import com.metamx.common.concurrent.ScheduledExecutors; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.common.concurrent.ScheduledExecutors; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.indexing.common.task.Task; import io.druid.indexing.overlord.ImmutableWorkerInfo; import io.druid.indexing.overlord.WorkerTaskRunner; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/SimpleWorkerProvisioningStrategy.java b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/SimpleWorkerProvisioningStrategy.java index 8ac414f5720f..c5d11ff80819 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/SimpleWorkerProvisioningStrategy.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/SimpleWorkerProvisioningStrategy.java @@ -29,8 +29,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.inject.Inject; -import com.metamx.common.concurrent.ScheduledExecutors; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.common.concurrent.ScheduledExecutors; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.indexing.overlord.ImmutableWorkerInfo; import io.druid.indexing.overlord.TaskRunnerWorkItem; import io.druid.indexing.overlord.WorkerTaskRunner; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java index 60ef3f247341..479253a2b595 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java @@ -36,7 +36,7 @@ import com.google.common.base.Function; import com.google.common.collect.FluentIterable; import com.google.common.collect.Lists; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.indexing.overlord.autoscaling.AutoScaler; import io.druid.indexing.overlord.autoscaling.AutoScalingData; import io.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig; diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorManager.java b/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorManager.java index 1dd4d76d7620..5c1eecc38ff3 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorManager.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorManager.java @@ -23,7 +23,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.indexing.overlord.DataSourceMetadata; import io.druid.java.util.common.Pair; import io.druid.java.util.common.lifecycle.LifecycleStart; diff --git a/indexing-service/src/main/java/io/druid/indexing/worker/WorkerTaskMonitor.java b/indexing-service/src/main/java/io/druid/indexing/worker/WorkerTaskMonitor.java index 5d470102f133..74f5c3357546 100644 --- a/indexing-service/src/main/java/io/druid/indexing/worker/WorkerTaskMonitor.java +++ b/indexing-service/src/main/java/io/druid/indexing/worker/WorkerTaskMonitor.java @@ -26,7 +26,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.indexer.TaskLocation; import io.druid.indexing.common.TaskStatus; import io.druid.indexing.common.task.Task; diff --git a/indexing-service/src/main/java/io/druid/indexing/worker/executor/ExecutorLifecycle.java b/indexing-service/src/main/java/io/druid/indexing/worker/executor/ExecutorLifecycle.java index 144c3512b90f..d06830eaf6d7 100644 --- a/indexing-service/src/main/java/io/druid/indexing/worker/executor/ExecutorLifecycle.java +++ b/indexing-service/src/main/java/io/druid/indexing/worker/executor/ExecutorLifecycle.java @@ -26,7 +26,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.concurrent.Execs; import io.druid.indexing.common.TaskStatus; import io.druid.indexing.common.actions.TaskActionClientFactory; diff --git a/indexing-service/src/test/java/io/druid/indexing/common/TaskToolboxTest.java b/indexing-service/src/test/java/io/druid/indexing/common/TaskToolboxTest.java index 1542b33aef6d..b52a3a351d09 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/TaskToolboxTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/TaskToolboxTest.java @@ -21,8 +21,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.metrics.MonitorScheduler; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.metrics.MonitorScheduler; import io.druid.client.cache.Cache; import io.druid.client.cache.CacheConfig; import io.druid.indexing.common.actions.TaskActionClientFactory; diff --git a/indexing-service/src/test/java/io/druid/indexing/common/actions/RemoteTaskActionClientTest.java b/indexing-service/src/test/java/io/druid/indexing/common/actions/RemoteTaskActionClientTest.java index 7543b0def3dd..b5f79d678ac6 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/actions/RemoteTaskActionClientTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/actions/RemoteTaskActionClientTest.java @@ -20,8 +20,8 @@ package io.druid.indexing.common.actions; import com.fasterxml.jackson.databind.ObjectMapper; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.FullResponseHolder; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.FullResponseHolder; import io.druid.discovery.DruidLeaderClient; import io.druid.indexing.common.RetryPolicyConfig; import io.druid.indexing.common.RetryPolicyFactory; diff --git a/indexing-service/src/test/java/io/druid/indexing/common/actions/SegmentAllocateActionTest.java b/indexing-service/src/test/java/io/druid/indexing/common/actions/SegmentAllocateActionTest.java index b7e82324739c..cef4c48bfeea 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/actions/SegmentAllocateActionTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/actions/SegmentAllocateActionTest.java @@ -25,8 +25,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.indexing.common.TaskLock; import io.druid.indexing.common.task.NoopTask; import io.druid.indexing.common.task.Task; diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java index 521bba77a360..bea1c613145d 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/RealtimeIndexTaskTest.java @@ -30,10 +30,10 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.core.NoopEmitter; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.metrics.MonitorScheduler; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.core.NoopEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.metrics.MonitorScheduler; import io.druid.client.cache.CacheConfig; import io.druid.client.cache.MapCache; import io.druid.data.input.Firehose; diff --git a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java index 3a255ea1fd55..13f2d4f1e88a 100644 --- a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java @@ -31,7 +31,7 @@ import com.google.common.io.Files; import com.google.inject.Binder; import com.google.inject.Module; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java index 996023f36a47..be311c3a39de 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTest.java @@ -28,8 +28,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ListenableFuture; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.indexer.TaskState; import io.druid.indexing.common.IndexingServiceCondition; import io.druid.indexing.common.TaskStatus; diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTestUtils.java b/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTestUtils.java index 7bcdff1530ad..cc4328814077 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTestUtils.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/RemoteTaskRunnerTestUtils.java @@ -24,7 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Supplier; import com.google.common.base.Throwables; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClient; import io.druid.common.guava.DSuppliers; import io.druid.curator.PotentiallyGzippedCompressionProvider; import io.druid.curator.cache.PathChildrenCacheFactory; diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java index 50912834680b..08d72eab3553 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java @@ -34,10 +34,10 @@ import com.google.common.collect.Lists; import com.google.common.collect.Ordering; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.metrics.Monitor; -import com.metamx.metrics.MonitorScheduler; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.metrics.Monitor; +import io.druid.java.util.metrics.MonitorScheduler; import io.druid.client.cache.MapCache; import io.druid.data.input.Firehose; import io.druid.data.input.FirehoseFactory; diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLockboxTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLockboxTest.java index 11266b60def0..33dd21076e20 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLockboxTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLockboxTest.java @@ -21,8 +21,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Iterables; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.indexing.common.TaskLock; import io.druid.indexing.common.TaskLockType; import io.druid.indexing.common.TaskStatus; diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/PendingTaskBasedProvisioningStrategyTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/PendingTaskBasedProvisioningStrategyTest.java index e6afde258183..426eb89378de 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/PendingTaskBasedProvisioningStrategyTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/PendingTaskBasedProvisioningStrategyTest.java @@ -23,9 +23,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceEventBuilder; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEventBuilder; import io.druid.common.guava.DSuppliers; import io.druid.java.util.common.concurrent.Execs; import io.druid.indexer.TaskLocation; diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/SimpleProvisioningStrategyTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/SimpleProvisioningStrategyTest.java index f917d359c64a..2e7294f52391 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/SimpleProvisioningStrategyTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/autoscaling/SimpleProvisioningStrategyTest.java @@ -24,9 +24,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceEventBuilder; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEventBuilder; import io.druid.common.guava.DSuppliers; import io.druid.java.util.common.concurrent.Execs; import io.druid.indexer.TaskLocation; diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/http/OverlordTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/http/OverlordTest.java index b88556cf4fc3..2cf3462aaa6a 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/http/OverlordTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/http/OverlordTest.java @@ -26,8 +26,8 @@ import com.google.common.collect.Lists; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.curator.PotentiallyGzippedCompressionProvider; import io.druid.curator.discovery.NoopServiceAnnouncer; import io.druid.discovery.DruidLeaderSelector; diff --git a/integration-tests/src/main/java/io/druid/testing/clients/ClientInfoResourceTestClient.java b/integration-tests/src/main/java/io/druid/testing/clients/ClientInfoResourceTestClient.java index 9c66444d2119..65bccb904f53 100644 --- a/integration-tests/src/main/java/io/druid/testing/clients/ClientInfoResourceTestClient.java +++ b/integration-tests/src/main/java/io/druid/testing/clients/ClientInfoResourceTestClient.java @@ -24,10 +24,10 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; import com.google.inject.Inject; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.StatusResponseHandler; -import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.StatusResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHolder; import io.druid.java.util.common.ISE; import io.druid.java.util.common.StringUtils; import io.druid.testing.IntegrationTestingConfig; diff --git a/integration-tests/src/main/java/io/druid/testing/clients/CoordinatorResourceTestClient.java b/integration-tests/src/main/java/io/druid/testing/clients/CoordinatorResourceTestClient.java index 9ce846653433..367094cfdafa 100644 --- a/integration-tests/src/main/java/io/druid/testing/clients/CoordinatorResourceTestClient.java +++ b/integration-tests/src/main/java/io/druid/testing/clients/CoordinatorResourceTestClient.java @@ -24,10 +24,10 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; import com.google.inject.Inject; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.StatusResponseHandler; -import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.StatusResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHolder; import io.druid.java.util.common.ISE; import io.druid.java.util.common.RE; import io.druid.java.util.common.StringUtils; diff --git a/integration-tests/src/main/java/io/druid/testing/clients/EventReceiverFirehoseTestClient.java b/integration-tests/src/main/java/io/druid/testing/clients/EventReceiverFirehoseTestClient.java index d41729d592ea..83ebf5fe0138 100644 --- a/integration-tests/src/main/java/io/druid/testing/clients/EventReceiverFirehoseTestClient.java +++ b/integration-tests/src/main/java/io/druid/testing/clients/EventReceiverFirehoseTestClient.java @@ -24,10 +24,10 @@ import com.fasterxml.jackson.jaxrs.smile.SmileMediaTypes; import com.google.common.base.Charsets; import com.google.common.base.Throwables; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.StatusResponseHandler; -import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.StatusResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHolder; import io.druid.java.util.common.jackson.JacksonUtils; import io.druid.java.util.common.ISE; import io.druid.java.util.common.StringUtils; diff --git a/integration-tests/src/main/java/io/druid/testing/clients/OverlordResourceTestClient.java b/integration-tests/src/main/java/io/druid/testing/clients/OverlordResourceTestClient.java index f47d97e4f536..38c6e3d174f1 100644 --- a/integration-tests/src/main/java/io/druid/testing/clients/OverlordResourceTestClient.java +++ b/integration-tests/src/main/java/io/druid/testing/clients/OverlordResourceTestClient.java @@ -25,10 +25,10 @@ import com.google.common.base.Predicates; import com.google.common.base.Throwables; import com.google.inject.Inject; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.StatusResponseHandler; -import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.StatusResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHolder; import io.druid.indexer.TaskState; import io.druid.java.util.common.ISE; import io.druid.java.util.common.RetryUtils; diff --git a/integration-tests/src/main/java/io/druid/testing/clients/QueryResourceTestClient.java b/integration-tests/src/main/java/io/druid/testing/clients/QueryResourceTestClient.java index 8060871ba5db..b6a2c6bcce30 100644 --- a/integration-tests/src/main/java/io/druid/testing/clients/QueryResourceTestClient.java +++ b/integration-tests/src/main/java/io/druid/testing/clients/QueryResourceTestClient.java @@ -25,10 +25,10 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; import com.google.inject.Inject; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.StatusResponseHandler; -import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.StatusResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHolder; import io.druid.java.util.common.ISE; import io.druid.java.util.common.StringUtils; import io.druid.query.Query; diff --git a/integration-tests/src/main/java/io/druid/testing/guice/DruidTestModule.java b/integration-tests/src/main/java/io/druid/testing/guice/DruidTestModule.java index b2cb8b00c133..9383a032a39d 100644 --- a/integration-tests/src/main/java/io/druid/testing/guice/DruidTestModule.java +++ b/integration-tests/src/main/java/io/druid/testing/guice/DruidTestModule.java @@ -24,13 +24,13 @@ import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Provides; -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.emitter.core.LoggingEmitter; -import com.metamx.emitter.core.LoggingEmitterConfig; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.http.client.CredentialedHttpClient; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.auth.BasicCredentials; +import io.druid.java.util.common.lifecycle.Lifecycle; +import io.druid.java.util.emitter.core.LoggingEmitter; +import io.druid.java.util.emitter.core.LoggingEmitterConfig; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.http.client.CredentialedHttpClient; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.auth.BasicCredentials; import io.druid.curator.CuratorConfig; import io.druid.guice.JsonConfigProvider; import io.druid.guice.ManageLifecycle; diff --git a/integration-tests/src/main/java/org/testng/DruidTestRunnerFactory.java b/integration-tests/src/main/java/org/testng/DruidTestRunnerFactory.java index bece170369d8..ed809088e003 100644 --- a/integration-tests/src/main/java/org/testng/DruidTestRunnerFactory.java +++ b/integration-tests/src/main/java/org/testng/DruidTestRunnerFactory.java @@ -23,10 +23,10 @@ import com.google.common.base.Throwables; import com.google.inject.Injector; import com.google.inject.Key; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.StatusResponseHandler; -import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.StatusResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHolder; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.lifecycle.Lifecycle; import io.druid.java.util.common.logger.Logger; diff --git a/integration-tests/src/test/java/io/druid/tests/indexer/ITRealtimeIndexTaskTest.java b/integration-tests/src/test/java/io/druid/tests/indexer/ITRealtimeIndexTaskTest.java index 530250c14a6e..54d31540b204 100644 --- a/integration-tests/src/test/java/io/druid/tests/indexer/ITRealtimeIndexTaskTest.java +++ b/integration-tests/src/test/java/io/druid/tests/indexer/ITRealtimeIndexTaskTest.java @@ -21,7 +21,7 @@ import com.google.common.base.Throwables; import com.google.inject.Inject; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClient; import io.druid.curator.discovery.ServerDiscoveryFactory; import io.druid.curator.discovery.ServerDiscoverySelector; import io.druid.java.util.common.DateTimes; diff --git a/integration-tests/src/test/java/io/druid/tests/indexer/ITUnionQueryTest.java b/integration-tests/src/test/java/io/druid/tests/indexer/ITUnionQueryTest.java index 645f66dddae1..392a39358746 100644 --- a/integration-tests/src/test/java/io/druid/tests/indexer/ITUnionQueryTest.java +++ b/integration-tests/src/test/java/io/druid/tests/indexer/ITUnionQueryTest.java @@ -22,7 +22,7 @@ import com.beust.jcommander.internal.Lists; import com.google.common.base.Throwables; import com.google.inject.Inject; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClient; import io.druid.curator.discovery.ServerDiscoveryFactory; import io.druid.curator.discovery.ServerDiscoverySelector; import io.druid.java.util.common.DateTimes; diff --git a/integration-tests/src/test/java/io/druid/tests/security/ITBasicAuthConfigurationTest.java b/integration-tests/src/test/java/io/druid/tests/security/ITBasicAuthConfigurationTest.java index d03ce64b346a..b8ecd4f71f36 100644 --- a/integration-tests/src/test/java/io/druid/tests/security/ITBasicAuthConfigurationTest.java +++ b/integration-tests/src/test/java/io/druid/tests/security/ITBasicAuthConfigurationTest.java @@ -24,12 +24,12 @@ import com.google.common.base.Charsets; import com.google.common.base.Throwables; import com.google.inject.Inject; -import com.metamx.http.client.CredentialedHttpClient; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.auth.BasicCredentials; -import com.metamx.http.client.response.StatusResponseHandler; -import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.java.util.http.client.CredentialedHttpClient; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.auth.BasicCredentials; +import io.druid.java.util.http.client.response.StatusResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHolder; import io.druid.guice.annotations.Client; import io.druid.java.util.common.ISE; import io.druid.java.util.common.StringUtils; diff --git a/integration-tests/src/test/resources/indexer/union_select_query.json b/integration-tests/src/test/resources/indexer/union_select_query.json index 76e4d0be63ed..f35e682c313a 100644 --- a/integration-tests/src/test/resources/indexer/union_select_query.json +++ b/integration-tests/src/test/resources/indexer/union_select_query.json @@ -75,7 +75,7 @@ } }, { - "segmentId": "wikipedia_index_test0_2013-08-31T0com.metamx.common.ISE: one or more twitter queries failed0:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:43.993Z", + "segmentId": "wikipedia_index_test0_2013-08-31T0io.druid.java.util.common.ISE: one or more twitter queries failed0:00:00.000Z_2013-09-01T00:00:00.000Z_2014-05-01T15:27:43.993Z", "offset": 0, "event": { "timestamp": "2013-08-31T03:32:45.000Z", diff --git a/java-util/pom.xml b/java-util/pom.xml index 27fa96e77fa5..71fc76a9c7e9 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -38,6 +38,12 @@ + + UTF-8 + 1.6.5 + ${sigar.base.version}.132 + + org.slf4j @@ -85,6 +91,12 @@ + + net.thisptr + jackson-jq + + + com.google.code.findbugs jsr305 @@ -93,6 +105,37 @@ javax.validation validation-api + + org.asynchttpclient + async-http-client + + + + + org.hyperic + sigar + ${sigar.version} + true + + + org.hyperic + sigar-dist + ${sigar.version} + zip + + provided + + + org.gridkit.lab + jvm-attach-api + + + + + io.netty + netty + @@ -111,10 +154,46 @@ test true - - net.thisptr - jackson-jq - + + org.apache.logging.log4j + log4j-api + test + + + org.apache.logging.log4j + log4j-core + test + + + org.apache.logging.log4j + log4j-slf4j-impl + test + + + org.apache.logging.log4j + log4j-1.2-api + test + + + org.apache.logging.log4j + log4j-jul + test + + + org.eclipse.jetty + jetty-server + test + + + org.eclipse.jetty + jetty-servlet + test + + + it.unimi.dsi + fastutil + test + @@ -141,7 +220,56 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-sigar-lib-to-resources + generate-resources + + unpack-dependencies + + + org.hyperic + sigar-dist + **/sigar-bin/lib/* + **/sigar-bin/lib/*jar + + ${project.build.directory} + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + java.library.path + ${project.build.directory}/hyperic-sigar-${sigar.base.version}/sigar-bin/lib/ + + + + + + + + + ${project.build.directory}/hyperic-sigar-${sigar.base.version}/sigar-bin/lib + + + - \ No newline at end of file + + + sigar + https://repository.jboss.org/nexus/content/repositories/thirdparty-uploads/ + + + + diff --git a/java-util/src/main/java/io/druid/java/util/common/StreamUtils.java b/java-util/src/main/java/io/druid/java/util/common/StreamUtils.java index 079d6c9c810b..23ff9d621ae3 100644 --- a/java-util/src/main/java/io/druid/java/util/common/StreamUtils.java +++ b/java-util/src/main/java/io/druid/java/util/common/StreamUtils.java @@ -25,6 +25,9 @@ import com.google.common.io.ByteSource; import com.google.common.io.ByteStreams; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -34,6 +37,31 @@ */ public class StreamUtils { + /** + * Copy from an input stream to a file (and buffer it) and close the input stream. + *

+ * It is highly recommended to use FileUtils.retryCopy whenever possible, and not use a raw `InputStream` + * + * @param is The input stream to copy bytes from. `is` is closed regardless of the copy result. + * @param file The file to copy bytes to. Any parent directories are automatically created. + * + * @return The count of bytes written to the file + * + * @throws IOException + */ + public static long copyToFileAndClose(InputStream is, File file) throws IOException + { + file.getParentFile().mkdirs(); + try (OutputStream os = new BufferedOutputStream(new FileOutputStream(file))) { + final long result = ByteStreams.copy(is, os); + // Workarround for http://hg.openjdk.java.net/jdk8/jdk8/jdk/rev/759aa847dcaf + os.flush(); + return result; + } + finally { + is.close(); + } + } /** * Copy from `is` to `os` and close the streams regardless of the result. diff --git a/java-util/src/main/java/io/druid/java/util/common/guava/Sequence.java b/java-util/src/main/java/io/druid/java/util/common/guava/Sequence.java index 20fab62adc20..77d711b8a987 100644 --- a/java-util/src/main/java/io/druid/java/util/common/guava/Sequence.java +++ b/java-util/src/main/java/io/druid/java/util/common/guava/Sequence.java @@ -34,7 +34,7 @@ * methods get called and other resources get cleaned up whenever processing is complete. Without this inversion * it is very easy to unintentionally leak resources when iterating over something that is backed by a resource. *

- * Sequences also expose {#see com.metamx.common.guava.Yielder} Yielder objects which allow you to implement a + * Sequences also expose {#see io.druid.java.util.common.guava.Yielder} Yielder objects which allow you to implement a * continuation over the Sequence. Yielder do not offer the same guarantees of automatic resource management * as the accumulate method, but they are Closeable and will do the proper cleanup when close() is called on them. */ diff --git a/java-util/src/main/java/io/druid/java/util/common/lifecycle/Lifecycle.java b/java-util/src/main/java/io/druid/java/util/common/lifecycle/Lifecycle.java index 05999be54566..eb8771adc48a 100644 --- a/java-util/src/main/java/io/druid/java/util/common/lifecycle/Lifecycle.java +++ b/java-util/src/main/java/io/druid/java/util/common/lifecycle/Lifecycle.java @@ -403,8 +403,7 @@ public void start() throws Exception for (Annotation annotation : method.getAnnotations()) { if (annotation.annotationType() .getCanonicalName() - .equals("io.druid.java.util.common.lifecycle.LifecycleStart") || - annotation.annotationType().getCanonicalName().equals("com.metamx.common.lifecycle.LifecycleStart")) { + .equals("io.druid.java.util.common.lifecycle.LifecycleStart")) { doStart = true; break; } @@ -424,8 +423,7 @@ public void stop() for (Annotation annotation : method.getAnnotations()) { if (annotation.annotationType() .getCanonicalName() - .equals("io.druid.java.util.common.lifecycle.LifecycleStop") || - annotation.annotationType().getCanonicalName().equals("com.metamx.common.lifecycle.LifecycleStop")) { + .equals("io.druid.java.util.common.lifecycle.LifecycleStop")) { doStop = true; break; } diff --git a/java-util/src/main/java/io/druid/java/util/common/logger/Logger.java b/java-util/src/main/java/io/druid/java/util/common/logger/Logger.java index 656de5f97c07..0d8189a0a08e 100644 --- a/java-util/src/main/java/io/druid/java/util/common/logger/Logger.java +++ b/java-util/src/main/java/io/druid/java/util/common/logger/Logger.java @@ -38,6 +38,12 @@ public Logger(Class clazz) log = LoggerFactory.getLogger(clazz); } + @Override + public String toString() + { + return StringUtils.format("Logger{name=[%s], class[%s]}", log.getName(), log.getClass()); + } + public void trace(String message, Object... formatArgs) { if (log.isTraceEnabled()) { diff --git a/java-util/src/main/java/io/druid/java/util/emitter/EmittingLogger.java b/java-util/src/main/java/io/druid/java/util/emitter/EmittingLogger.java new file mode 100644 index 000000000000..55c679d4991b --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/EmittingLogger.java @@ -0,0 +1,139 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.collect.Maps; +import io.druid.java.util.common.ISE; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.logger.Logger; +import io.druid.java.util.emitter.service.AlertBuilder; +import io.druid.java.util.emitter.service.ServiceEmitter; + +import java.io.PrintWriter; +import java.io.StringWriter; + +/** + */ +public class EmittingLogger extends Logger +{ + private static volatile ServiceEmitter emitter = null; + + private final String className; + + public static void registerEmitter(ServiceEmitter emitter) + { + Preconditions.checkNotNull(emitter); + EmittingLogger.emitter = emitter; + } + + public EmittingLogger(String className) + { + super(className); + + this.className = className; + } + + public EmittingLogger(Class clazz) + { + super(clazz); + + this.className = clazz.getName(); + } + + public AlertBuilder makeAlert(String message, Object... objects) + { + return makeAlert(null, message, objects); + } + + public AlertBuilder makeAlert(Throwable t, String message, Object... objects) + { + if (emitter == null) { + final String errorMessage = StringUtils.format( + "Emitter not initialized! Cannot alert. Please make sure to call %s.registerEmitter()", this.getClass() + ); + + error(errorMessage); + throw new ISE(errorMessage); + } + + final AlertBuilder retVal = new EmittingAlertBuilder(t, StringUtils.format(message, objects), emitter) + .addData("class", className); + + if (t != null) { + final StringWriter trace = new StringWriter(); + final PrintWriter pw = new PrintWriter(trace); + t.printStackTrace(pw); + retVal.addData("exceptionType", t.getClass()); + retVal.addData("exceptionMessage", t.getMessage()); + retVal.addData("exceptionStackTrace", trace.toString()); + } + + return retVal; + } + + public class EmittingAlertBuilder extends AlertBuilder + { + private final Throwable t; + + private volatile boolean emitted = false; + + private EmittingAlertBuilder(Throwable t, String description, ServiceEmitter emitter) + { + super(description, emitter); + this.t = t; + } + + @Override + public void emit() + { + logIt("%s: %s"); + + emitted = true; + + super.emit(); + } + + @Override + protected void finalize() throws Throwable + { + if (!emitted) { + logIt("Alert not emitted, emitting. %s: %s"); + super.emit(); + } + } + + private void logIt(String format) + { + if (t == null) { + error(format, description, dataMap); + } else { + // Filter out the stack trace from the message, because it should be in the logline already if it's wanted. + error( + t, + format, + description, + Maps.filterKeys(dataMap, Predicates.not(Predicates.equalTo("exceptionStackTrace"))) + ); + } + } + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/BaseHttpEmittingConfig.java b/java-util/src/main/java/io/druid/java/util/emitter/core/BaseHttpEmittingConfig.java new file mode 100644 index 000000000000..7078a3bf679e --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/BaseHttpEmittingConfig.java @@ -0,0 +1,152 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.Min; + +public class BaseHttpEmittingConfig +{ + public static final long DEFAULT_FLUSH_MILLIS = 60 * 1000; + public static final int DEFAULT_FLUSH_COUNTS = 500; + public static final int DEFAULT_MAX_BATCH_SIZE = 5 * 1024 * 1024; + /** + * Do not time out in case flushTimeOut is not set + */ + public static final long DEFAULT_FLUSH_TIME_OUT = Long.MAX_VALUE; + public static final String DEFAULT_BASIC_AUTHENTICATION = null; + public static final BatchingStrategy DEFAULT_BATCHING_STRATEGY = BatchingStrategy.ARRAY; + public static final ContentEncoding DEFAULT_CONTENT_ENCODING = null; + public static final int DEFAULT_BATCH_QUEUE_SIZE_LIMIT = 50; + public static final float DEFAULT_HTTP_TIMEOUT_ALLOWANCE_FACTOR = 2.0f; + /** + * The default value effective doesn't set the min timeout + */ + public static final int DEFAULT_MIN_HTTP_TIMEOUT_MILLIS = 0; + + @Min(1) + @JsonProperty + long flushMillis = DEFAULT_FLUSH_MILLIS; + + @Min(0) + @JsonProperty + int flushCount = DEFAULT_FLUSH_COUNTS; + + @Min(0) + @JsonProperty + long flushTimeOut = DEFAULT_FLUSH_TIME_OUT; + + @JsonProperty + String basicAuthentication = DEFAULT_BASIC_AUTHENTICATION; + + @JsonProperty + BatchingStrategy batchingStrategy = DEFAULT_BATCHING_STRATEGY; + + @Min(0) + @JsonProperty + int maxBatchSize = DEFAULT_MAX_BATCH_SIZE; + + @JsonProperty + ContentEncoding contentEncoding = DEFAULT_CONTENT_ENCODING; + + @Min(0) + @JsonProperty + int batchQueueSizeLimit = DEFAULT_BATCH_QUEUE_SIZE_LIMIT; + + @Min(1) + @JsonProperty + float httpTimeoutAllowanceFactor = DEFAULT_HTTP_TIMEOUT_ALLOWANCE_FACTOR; + + @Min(0) + @JsonProperty + int minHttpTimeoutMillis = DEFAULT_MIN_HTTP_TIMEOUT_MILLIS; + + public long getFlushMillis() + { + return flushMillis; + } + + public int getFlushCount() + { + return flushCount; + } + + public long getFlushTimeOut() + { + return flushTimeOut; + } + + public String getBasicAuthentication() + { + return basicAuthentication; + } + + public BatchingStrategy getBatchingStrategy() + { + return batchingStrategy; + } + + public int getMaxBatchSize() + { + return maxBatchSize; + } + + public ContentEncoding getContentEncoding() + { + return contentEncoding; + } + + public int getBatchQueueSizeLimit() + { + return batchQueueSizeLimit; + } + + public float getHttpTimeoutAllowanceFactor() + { + return httpTimeoutAllowanceFactor; + } + + public int getMinHttpTimeoutMillis() + { + return minHttpTimeoutMillis; + } + + @Override + public String toString() + { + return "BaseHttpEmittingConfig{" + toStringBase() + '}'; + } + + protected String toStringBase() + { + return + "flushMillis=" + flushMillis + + ", flushCount=" + flushCount + + ", flushTimeOut=" + flushTimeOut + + ", basicAuthentication='" + basicAuthentication + '\'' + + ", batchingStrategy=" + batchingStrategy + + ", maxBatchSize=" + maxBatchSize + + ", contentEncoding=" + contentEncoding + + ", batchQueueSizeLimit=" + batchQueueSizeLimit + + ", httpTimeoutAllowanceFactor=" + httpTimeoutAllowanceFactor + + ", minHttpTimeoutMillis=" + minHttpTimeoutMillis; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/Batch.java b/java-util/src/main/java/io/druid/java/util/emitter/core/Batch.java new file mode 100644 index 000000000000..16dbab4532f1 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/Batch.java @@ -0,0 +1,332 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.google.common.base.Preconditions; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.AbstractQueuedLongSynchronizer; + +/** + * Buffer for batched data + synchronization state. + *

+ * The state structure ({@link AbstractQueuedLongSynchronizer#state}): + * Bits 0-30 - bufferWatermark + * Bit 31 - always 0 + * Bits 32-62 - "parties" (the number of concurrent writers) + * Bit 63 - sealed flag + *

+ * Writer threads (callers of {@link HttpPostEmitter#emit(Event)}) are eligible to come, increment bufferWatermark and + * write data into the buffer, as long as sealed flag is false. + *

+ * {@link HttpPostEmitter#emittingThread} is eligible to emit the buffer, when sealed flag=true and parties=0 (all + * writes are completed). See {@link #isEmittingAllowed(long)}. + *

+ * In this class, "lock" means "increment number of parties by 1", i. e. lock the emitter thread from emitting this + * batch. "Unlock" means "decrement number of parties by 1". + */ +class Batch extends AbstractQueuedLongSynchronizer +{ + private static final long PARTY = 1L << 32; + private static final long SEAL_BIT = 1L << 63; + + private static int bufferWatermark(long state) + { + return (int) state; + } + + private static int parties(long state) + { + return ((int) (state >>> 32)) & Integer.MAX_VALUE; + } + + private static boolean isSealed(long state) + { + // The highest bit is 1. + return state < 0; + } + + private static boolean isEmittingAllowed(long state) + { + return isSealed(state) && parties(state) == 0; + } + + /** + * Tags (arg values) to transmit request from {@link #releaseShared(long)} to {@link #tryReleaseShared(long)}. + */ + private static final long UNLOCK_TAG = 0; + private static final long UNLOCK_AND_SEAL_TAG = 1; + private static final long SEAL_TAG = 2; + + /** + * The emitter this batch belongs to. + */ + private final HttpPostEmitter emitter; + + /** + * The data buffer of the batch. + */ + final byte[] buffer; + + /** + * Ordering number of this batch, as they filled & emitted in {@link HttpPostEmitter} serially, starting from 0. + */ + final int batchNumber; + + /** + * The number of events in this batch, needed for event count-based batch emitting. + */ + final AtomicInteger eventCount = new AtomicInteger(0); + + /** + * The time when the first event was written into this batch, needed for timeout-based batch emitting. + */ + private long firstEventTimestamp = -1; + + Batch(HttpPostEmitter emitter, byte[] buffer, int batchNumber) + { + this.emitter = emitter; + this.buffer = buffer; + this.batchNumber = batchNumber; + } + + int getSealedBufferWatermark() + { + long state = getState(); + Preconditions.checkState(isSealed(state)); + return bufferWatermark(state); + } + + /** + * Tries to add (write) event to the batch, returns true, if successful. If fails, no subsequent attempts to add event + * to this batch will succeed, the next batch should be taken. + */ + boolean tryAddEvent(byte[] event) + { + while (true) { + long state = getState(); + if (isSealed(state)) { + return false; + } + int bufferWatermark = bufferWatermark(state); + if (bufferWatermark == 0) { + if (tryAddFirstEvent(event)) { + return true; + } + } else if (newBufferWatermark(bufferWatermark, event) <= emitter.maxBufferWatermark) { + if (tryAddNonFirstEvent(state, event)) { + return true; + } + } else { + seal(); + return false; + } + } + } + + private boolean tryAddFirstEvent(byte[] event) + { + if (!tryReserveFirstEventSizeAndLock(event)) { + return false; + } + try { + int bufferOffset = emitter.batchingStrategy.writeBatchStart(buffer); + writeEvent(event, bufferOffset); + eventCount.incrementAndGet(); + firstEventTimestamp = System.currentTimeMillis(); + return true; + } + finally { + unlock(); + } + } + + private boolean tryReserveFirstEventSizeAndLock(byte[] event) + { + return compareAndSetState(0, emitter.batchingStrategy.batchStartLength() + event.length + PARTY); + } + + private int newBufferWatermark(int bufferWatermark, byte[] eventBytes) + { + return bufferWatermark + emitter.batchingStrategy.separatorLength() + eventBytes.length; + } + + private boolean tryAddNonFirstEvent(long state, byte[] event) + { + int bufferOffset = tryReserveEventSizeAndLock(state, emitter.batchingStrategy.separatorLength() + event.length); + if (bufferOffset < 0) { + return false; + } + try { + bufferOffset = emitter.batchingStrategy.writeMessageSeparator(buffer, bufferOffset); + writeEvent(event, bufferOffset); + return true; + } + finally { + unlockAndSealIfNeeded(); + } + } + + /** + * Returns the buffer offset at which the caller has reserved the ability to write `size` bytes exclusively, + * or negative number, if the reservation attempt failed. + */ + private int tryReserveEventSizeAndLock(long state, int size) + { + Preconditions.checkArgument(size > 0); + int bufferWatermark = bufferWatermark(state); + while (true) { + if (compareAndSetState(state, state + size + PARTY)) { + return bufferWatermark; + } + state = getState(); + if (isSealed(state)) { + return -1; + } + bufferWatermark = bufferWatermark(state); + int newBufferWatermark = bufferWatermark + size; + Preconditions.checkState(newBufferWatermark > 0); + if (newBufferWatermark > emitter.maxBufferWatermark) { + return -1; + } + } + } + + private void unlockAndSealIfNeeded() + { + if (eventCount.incrementAndGet() >= emitter.config.getFlushCount()) { + unlockAndSeal(); + } else { + long timeSinceFirstEvent = System.currentTimeMillis() - firstEventTimestamp; + if (firstEventTimestamp > 0 && timeSinceFirstEvent > emitter.config.getFlushMillis()) { + unlockAndSeal(); + } else { + unlock(); + } + } + } + + void sealIfFlushNeeded() + { + long timeSinceFirstEvent = System.currentTimeMillis() - firstEventTimestamp; + if (firstEventTimestamp > 0 && timeSinceFirstEvent > emitter.config.getFlushMillis()) { + seal(); + } + } + + private void writeEvent(byte[] event, int bufferOffset) + { + System.arraycopy(event, 0, buffer, bufferOffset, event.length); + } + + + private void unlock() + { + releaseShared(UNLOCK_TAG); + } + + private void unlockAndSeal() + { + releaseShared(UNLOCK_AND_SEAL_TAG); + } + + void seal() + { + releaseShared(SEAL_TAG); + } + + @Override + protected boolean tryReleaseShared(long tag) + { + if (tag == UNLOCK_TAG) { + while (true) { + long state = getState(); + int parties = parties(state); + if (parties == 0) { + throw new IllegalMonitorStateException(); + } + long newState = state - PARTY; + if (compareAndSetState(state, newState)) { + return isEmittingAllowed(newState); + } + } + } else if (tag == UNLOCK_AND_SEAL_TAG) { + while (true) { + long state = getState(); + int parties = parties(state); + if (parties == 0) { + throw new IllegalMonitorStateException(); + } + long newState = (state - PARTY) | SEAL_BIT; + if (compareAndSetState(state, newState)) { + // Ensures only one thread calls emitter.onSealExclusive() for each batch. + if (!isSealed(state)) { + emitter.onSealExclusive( + this, + firstEventTimestamp > 0 ? System.currentTimeMillis() - firstEventTimestamp : -1 + ); + } + return isEmittingAllowed(newState); + } + } + } else if (tag == SEAL_TAG) { + while (true) { + long state = getState(); + if (isSealed(state)) { + // Returning false, despite acquisition could be possible now, because this thread actually didn't update the + // state, i. e. didn't "release" in AbstractQueuedLongSynchronizer's terms. + return false; + } + long newState = state | SEAL_BIT; + if (compareAndSetState(state, newState)) { + emitter.onSealExclusive( + this, + firstEventTimestamp > 0 ? System.currentTimeMillis() - firstEventTimestamp : -1 + ); + return isEmittingAllowed(newState); + } + } + } else { + throw new IllegalStateException("Unknown tag: " + tag); + } + } + + void awaitEmittingAllowed() + { + acquireShared(1); + } + + @Override + protected long tryAcquireShared(long ignored) + { + return isEmittingAllowed(getState()) ? 1 : -1; + } + + @Override + public String toString() + { + long state = getState(); + return "Batch{" + + "bufferWatermark=" + bufferWatermark(state) + + ", parties=" + parties(state) + + ", isSealed=" + isSealed(state) + + "}"; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/BatchingStrategy.java b/java-util/src/main/java/io/druid/java/util/emitter/core/BatchingStrategy.java new file mode 100644 index 000000000000..b94f894b42c6 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/BatchingStrategy.java @@ -0,0 +1,155 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +public enum BatchingStrategy +{ + ARRAY { + + @Override + public int batchStartLength() + { + return 1; + } + + @Override + public int separatorLength() + { + return 1; + } + + @Override + public int batchEndLength() + { + return 2; + } + + @Override + public int writeBatchStart(byte[] buffer) + { + buffer[0] = '['; + return batchStartLength(); + } + + @Override + public int writeMessageSeparator(byte[] buffer, int bufferOffset) + { + buffer[bufferOffset] = ','; + return bufferOffset + separatorLength(); + } + + @Override + public int writeBatchEnd(byte[] buffer, int bufferOffset) + { + buffer[bufferOffset] = ']'; + buffer[bufferOffset + 1] = '\n'; + return bufferOffset + batchEndLength(); + } + }, + NEWLINES { + + @Override + public int batchStartLength() + { + return 0; + } + + @Override + public int separatorLength() + { + return 1; + } + + @Override + public int batchEndLength() + { + return 1; + } + + @Override + public int writeBatchStart(byte[] buffer) + { + // Write nothing + return batchStartLength(); + } + + @Override + public int writeMessageSeparator(byte[] buffer, int bufferOffset) + { + buffer[bufferOffset] = '\n'; + return bufferOffset + separatorLength(); + } + + @Override + public int writeBatchEnd(byte[] buffer, int bufferOffset) + { + return writeMessageSeparator(buffer, bufferOffset); + } + }, + ONLY_EVENTS { + @Override + public int batchStartLength() + { + return 0; + } + + @Override + public int separatorLength() + { + return 0; + } + + @Override + public int batchEndLength() + { + return 0; + } + + @Override + public int writeBatchStart(byte[] buffer) + { + return 0; + } + + @Override + public int writeMessageSeparator(byte[] buffer, int bufferOffset) + { + return bufferOffset; + } + + @Override + public int writeBatchEnd(byte[] buffer, int bufferOffset) + { + return bufferOffset; + } + }; + + public abstract int batchStartLength(); + + public abstract int separatorLength(); + + public abstract int batchEndLength(); + + public abstract int writeBatchStart(byte[] buffer); + + public abstract int writeMessageSeparator(byte[] buffer, int bufferOffset); + + public abstract int writeBatchEnd(byte[] buffer, int bufferOffset); +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/ComposingEmitter.java b/java-util/src/main/java/io/druid/java/util/emitter/core/ComposingEmitter.java new file mode 100644 index 000000000000..b2d5b9fc8afb --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/ComposingEmitter.java @@ -0,0 +1,113 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.google.common.base.Preconditions; +import io.druid.java.util.common.lifecycle.LifecycleStart; +import io.druid.java.util.common.lifecycle.LifecycleStop; +import io.druid.java.util.common.logger.Logger; + +import java.io.IOException; +import java.util.List; + +public class ComposingEmitter implements Emitter +{ + private static Logger log = new Logger(ComposingEmitter.class); + + private final List emitters; + + public ComposingEmitter(List emitters) + { + this.emitters = Preconditions.checkNotNull(emitters, "null emitters"); + } + + @Override + @LifecycleStart + public void start() + { + log.info("Starting Composing Emitter."); + + for (Emitter e : emitters) { + log.info("Starting emitter %s.", e.getClass().getName()); + e.start(); + } + } + + @Override + public void emit(Event event) + { + for (Emitter e : emitters) { + e.emit(event); + } + } + + @Override + public void flush() throws IOException + { + boolean fail = false; + log.info("Flushing Composing Emitter."); + + for (Emitter e : emitters) { + try { + log.info("Flushing emitter %s.", e.getClass().getName()); + e.flush(); + } + catch (IOException ex) { + log.error(ex, "Failed to flush emitter [%s]", e.getClass().getName()); + fail = true; + } + } + + if (fail) { + throw new IOException("failed to flush one or more emitters"); + } + } + + @Override + @LifecycleStop + public void close() throws IOException + { + boolean fail = false; + log.info("Closing Composing Emitter."); + + for (Emitter e : emitters) { + try { + log.info("Closing emitter %s.", e.getClass().getName()); + e.close(); + } + catch (IOException ex) { + log.error(ex, "Failed to close emitter [%s]", e.getClass().getName()); + fail = true; + } + } + + if (fail) { + throw new IOException("failed to close one or more emitters"); + } + } + + @Override + public String toString() + { + return "ComposingEmitter{" + + "emitters=" + emitters + + '}'; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/ConcurrentTimeCounter.java b/java-util/src/main/java/io/druid/java/util/emitter/core/ConcurrentTimeCounter.java new file mode 100644 index 000000000000..4b836789f7ae --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/ConcurrentTimeCounter.java @@ -0,0 +1,96 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.google.common.primitives.UnsignedInts; + +import java.util.concurrent.atomic.AtomicLong; + +/** + * A class to accumulate simple stats of some time points. All methods are safe to use from multiple threads. + */ +public class ConcurrentTimeCounter +{ + /** Lower 32 bits for sum of {@link #add}ed times, higher 32 bits for the count */ + private final AtomicLong timeSumAndCount = new AtomicLong(0L); + /** Lower 32 bits for the max {@link #add}ed time, 63th bit for indication if any value is added. */ + private final AtomicLong max = new AtomicLong(-1); + /** Similar to {@link #max} */ + private final AtomicLong min = new AtomicLong(-1); + + public void add(int time) + { + long x = (1L << 32) | time; + timeSumAndCount.addAndGet(x); + updateMax(time); + updateMin(time); + } + + private void updateMax(int time) + { + long max; + do { + max = this.max.get(); + if (max >= 0 && ((int) max) >= time) { + return; + } + } while (!this.max.compareAndSet(max, UnsignedInts.toLong(time))); + } + + private void updateMin(int time) + { + long min; + do { + min = this.min.get(); + if (min >= 0 && ((int) min) <= time) { + return; + } + } while (!this.min.compareAndSet(min, UnsignedInts.toLong(time))); + } + + public long getTimeSumAndCountAndReset() + { + return timeSumAndCount.getAndSet(0L); + } + + public int getAndResetMaxTime() + { + long max = this.max.getAndSet(-1); + // If max < 0, means no times added yet, then return 0 + return max >= 0 ? (int) max : 0; + } + + public int getAndResetMinTime() + { + long min = this.min.getAndSet(-1); + // If min < 0, means no times added yet, then return 0 + return min >= 0 ? (int) min : 0; + } + + public static int timeSum(long timeSumAndCount) + { + return (int) timeSumAndCount; + } + + public static int count(long timeSumAndCount) + { + return (int) (timeSumAndCount >> 32); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/ContentEncoding.java b/java-util/src/main/java/io/druid/java/util/emitter/core/ContentEncoding.java new file mode 100644 index 000000000000..ad45c24de2be --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/ContentEncoding.java @@ -0,0 +1,25 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +public enum ContentEncoding +{ + GZIP +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/EmittedBatchCounter.java b/java-util/src/main/java/io/druid/java/util/emitter/core/EmittedBatchCounter.java new file mode 100644 index 000000000000..860e6957fe94 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/EmittedBatchCounter.java @@ -0,0 +1,73 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import java.util.concurrent.Phaser; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + + +final class EmittedBatchCounter +{ + /** + * Max {@link Phaser}'s phase, specified in it's javadoc. Then it wraps to zero. + */ + private static final int MAX_PHASE = Integer.MAX_VALUE; + + static int nextBatchNumber(int prevBatchNumber) + { + return (prevBatchNumber + 1) & MAX_PHASE; + } + + private final Phaser phaser = new Phaser(1); + + void batchEmitted() + { + phaser.arrive(); + } + + void awaitBatchEmitted(int batchNumberToAwait, long timeout, TimeUnit unit) + throws InterruptedException, TimeoutException + { + batchNumberToAwait &= MAX_PHASE; + int currentBatch = phaser.getPhase(); + checkNotTerminated(currentBatch); + while (comparePhases(batchNumberToAwait, currentBatch) >= 0) { + currentBatch = phaser.awaitAdvanceInterruptibly(currentBatch, timeout, unit); + checkNotTerminated(currentBatch); + } + } + + private static int comparePhases(int phase1, int phase2) + { + int diff = (phase1 - phase2) & MAX_PHASE; + if (diff == 0) { + return 0; + } + return diff < MAX_PHASE / 2 ? 1 : -1; + } + + private void checkNotTerminated(int phase) + { + if (phase < 0) { + throw new IllegalStateException("Phaser[" + phaser + "] unexpectedly terminated."); + } + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/Emitter.java b/java-util/src/main/java/io/druid/java/util/emitter/core/Emitter.java new file mode 100644 index 000000000000..a3fdf88b92e2 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/Emitter.java @@ -0,0 +1,38 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import java.io.Closeable; +import java.io.Flushable; +import java.io.IOException; + +/** + */ +public interface Emitter extends Closeable, Flushable +{ + void start(); + void emit(Event event); + + @Override + void flush() throws IOException; + + @Override + void close() throws IOException; +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/Emitters.java b/java-util/src/main/java/io/druid/java/util/emitter/core/Emitters.java new file mode 100644 index 000000000000..537233c9c4f4 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/Emitters.java @@ -0,0 +1,177 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Maps; +import io.druid.java.util.common.IAE; +import io.druid.java.util.common.ISE; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.lifecycle.Lifecycle; +import io.druid.java.util.common.logger.Logger; +import io.druid.java.util.emitter.factory.EmitterFactory; +import org.asynchttpclient.AsyncHttpClient; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +public class Emitters +{ + private static final Logger log = new Logger(Emitters.class); + + private static final String LOG_EMITTER_PROP = "io.druid.java.util.emitter.logging"; + private static final String HTTP_EMITTER_PROP = "io.druid.java.util.emitter.http"; + private static final String CUSTOM_EMITTER_TYPE_PROP = "io.druid.java.util.emitter.type"; + + public static Emitter create(Properties props, AsyncHttpClient httpClient, Lifecycle lifecycle) + { + return create(props, httpClient, new ObjectMapper(), lifecycle); + } + + public static Emitter create( + Properties props, + AsyncHttpClient httpClient, + ObjectMapper jsonMapper, + Lifecycle lifecycle + ) + { + Map jsonified = Maps.newHashMap(); + if (props.getProperty(LOG_EMITTER_PROP) != null) { + jsonified = makeLoggingMap(props); + jsonified.put("type", "logging"); + } else if (props.getProperty(HTTP_EMITTER_PROP) != null) { + jsonified = makeHttpMap(props); + jsonified.put("type", "http"); + } else if (props.getProperty(CUSTOM_EMITTER_TYPE_PROP) != null) { + jsonified = makeCustomFactoryMap(props); + } else { + throw new ISE( + "Unknown type of emitter. Please set [%s], [%s] or provide registered subtype of io.druid.java.util.emitter.core.factory.EmitterFactory via [%s]", + LOG_EMITTER_PROP, + HTTP_EMITTER_PROP, + CUSTOM_EMITTER_TYPE_PROP + ); + } + return jsonMapper.convertValue(jsonified, EmitterFactory.class).makeEmitter(jsonMapper, httpClient, lifecycle); + } + + // Package-visible for unit tests + + static Map makeHttpMap(Properties props) + { + Map httpMap = Maps.newHashMap(); + + final String urlProperty = "io.druid.java.util.emitter.http.url"; + + final String baseUrl = props.getProperty(urlProperty); + if (baseUrl == null) { + throw new IAE("Property[%s] must be set", urlProperty); + } + + httpMap.put("recipientBaseUrl", baseUrl); + httpMap.put("flushMillis", Long.parseLong(props.getProperty("io.druid.java.util.emitter.flushMillis", "60000"))); + httpMap.put("flushCount", Integer.parseInt(props.getProperty("io.druid.java.util.emitter.flushCount", "300"))); + /** + * The defaultValue for "io.druid.java.util.emitter.http.flushTimeOut" must be same as {@link HttpEmitterConfig.DEFAULT_FLUSH_TIME_OUT} + * */ + httpMap.put( + "flushTimeOut", + Long.parseLong(props.getProperty( + "io.druid.java.util.emitter.http.flushTimeOut", + String.valueOf(Long.MAX_VALUE) + )) + ); + if (props.containsKey("io.druid.java.util.emitter.http.basicAuthentication")) { + httpMap.put("basicAuthentication", props.getProperty("io.druid.java.util.emitter.http.basicAuthentication")); + } + if (props.containsKey("io.druid.java.util.emitter.http.batchingStrategy")) { + httpMap.put( + "batchingStrategy", + StringUtils.toUpperCase(props.getProperty("io.druid.java.util.emitter.http.batchingStrategy")) + ); + } + if (props.containsKey("io.druid.java.util.emitter.http.maxBatchSize")) { + httpMap.put("maxBatchSize", Integer.parseInt(props.getProperty("io.druid.java.util.emitter.http.maxBatchSize"))); + } + if (props.containsKey("io.druid.java.util.emitter.http.batchQueueSizeLimit")) { + httpMap.put( + "batchQueueSizeLimit", + Integer.parseInt(props.getProperty("io.druid.java.util.emitter.http.batchQueueSizeLimit")) + ); + } + if (props.containsKey("io.druid.java.util.emitter.http.httpTimeoutAllowanceFactor")) { + httpMap.put( + "httpTimeoutAllowanceFactor", + Float.parseFloat(props.getProperty("io.druid.java.util.emitter.http.httpTimeoutAllowanceFactor")) + ); + } + if (props.containsKey("io.druid.java.util.emitter.http.minHttpTimeoutMillis")) { + httpMap.put( + "minHttpTimeoutMillis", + Float.parseFloat(props.getProperty("io.druid.java.util.emitter.http.minHttpTimeoutMillis")) + ); + } + return httpMap; + } + + // Package-visible for unit tests + static Map makeLoggingMap(Properties props) + { + Map loggingMap = Maps.newHashMap(); + + loggingMap.put( + "loggerClass", props.getProperty("io.druid.java.util.emitter.logging.class", LoggingEmitter.class.getName()) + ); + loggingMap.put( + "logLevel", props.getProperty("io.druid.java.util.emitter.logging.level", "debug") + ); + return loggingMap; + } + + static Map makeCustomFactoryMap(Properties props) + { + Map factoryMap = Maps.newHashMap(); + String prefix = "io.druid.java.util.emitter."; + + for (Map.Entry entry : props.entrySet()) { + String key = entry.getKey().toString(); + if (key.startsWith(prefix)) { + String combinedKey = key.substring(prefix.length()); + Map currentLevelJson = factoryMap; + String currentKey = null; + String[] keyPath = combinedKey.split("\\."); + + for (int i = 0; i < keyPath.length - 1; i++) { + String keyPart = keyPath[i]; + Object nextLevelJson = currentLevelJson.get(keyPart); + if (nextLevelJson == null) { + nextLevelJson = new HashMap(); + currentLevelJson.put(keyPart, nextLevelJson); + } + currentLevelJson = (Map) nextLevelJson; + } + + currentLevelJson.put(keyPath[keyPath.length - 1], entry.getValue()); + } + } + return factoryMap; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/Event.java b/java-util/src/main/java/io/druid/java/util/emitter/core/Event.java new file mode 100644 index 000000000000..b9588ab40a0d --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/Event.java @@ -0,0 +1,37 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import org.joda.time.DateTime; + +import java.util.Map; + +/** + */ +public interface Event +{ + Map toMap(); + + String getFeed(); + + DateTime getCreatedTime(); + + boolean isSafeToBuffer(); +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/FeedUriExtractor.java b/java-util/src/main/java/io/druid/java/util/emitter/core/FeedUriExtractor.java new file mode 100644 index 000000000000..dbac2aa7408b --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/FeedUriExtractor.java @@ -0,0 +1,49 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import io.druid.java.util.common.StringUtils; + +import java.net.URI; +import java.net.URISyntaxException; + +public class FeedUriExtractor implements UriExtractor +{ + private String uriPattern; + + public FeedUriExtractor(String uriPattern) + { + this.uriPattern = uriPattern; + } + + @Override + public URI apply(Event event) throws URISyntaxException + { + return new URI(StringUtils.format(uriPattern, event.getFeed())); + } + + @Override + public String toString() + { + return "FeedUriExtractor{" + + "uriPattern='" + uriPattern + '\'' + + '}'; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/HttpEmitterConfig.java b/java-util/src/main/java/io/druid/java/util/emitter/core/HttpEmitterConfig.java new file mode 100644 index 000000000000..1db1b840ff73 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/HttpEmitterConfig.java @@ -0,0 +1,140 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.NotNull; + +/** + */ +public class HttpEmitterConfig extends BaseHttpEmittingConfig +{ + @NotNull + @JsonProperty + String recipientBaseUrl = null; + + /** + * For JSON deserialization only. In other cases use {@link Builder} + */ + public HttpEmitterConfig() {} + + public HttpEmitterConfig(BaseHttpEmittingConfig base, String recipientBaseUrl) + { + this.flushMillis = base.flushMillis; + this.flushCount = base.flushCount; + this.flushTimeOut = base.flushTimeOut; + this.recipientBaseUrl = recipientBaseUrl; + this.basicAuthentication = base.basicAuthentication; + this.batchingStrategy = base.batchingStrategy; + this.maxBatchSize = base.maxBatchSize; + this.contentEncoding = base.contentEncoding; + this.batchQueueSizeLimit = base.batchQueueSizeLimit; + this.httpTimeoutAllowanceFactor = base.httpTimeoutAllowanceFactor; + this.minHttpTimeoutMillis = base.minHttpTimeoutMillis; + } + + public String getRecipientBaseUrl() + { + return recipientBaseUrl; + } + + @Override + public String toString() + { + return "HttpEmitterConfig{" + + toStringBase() + + ", recipientBaseUrl=\'" + recipientBaseUrl + '\'' + + '}'; + } + + public static class Builder extends HttpEmitterConfig + { + public Builder(String recipientBaseUrl) + { + this.recipientBaseUrl = recipientBaseUrl; + } + + public Builder setFlushMillis(long flushMillis) + { + this.flushMillis = flushMillis; + return this; + } + + public Builder setFlushCount(int flushCount) + { + this.flushCount = flushCount; + return this; + } + + public Builder setFlushTimeOut(long flushTimeOut) + { + this.flushTimeOut = flushTimeOut; + return this; + } + + public Builder setBasicAuthentication(String basicAuthentication) + { + this.basicAuthentication = basicAuthentication; + return this; + } + + public Builder setBatchingStrategy(BatchingStrategy batchingStrategy) + { + this.batchingStrategy = batchingStrategy; + return this; + } + + public Builder setMaxBatchSize(int maxBatchSize) + { + this.maxBatchSize = maxBatchSize; + return this; + } + + public Builder setContentEncoding(ContentEncoding contentEncoding) + { + this.contentEncoding = contentEncoding; + return this; + } + + public Builder setBatchQueueSizeLimit(int batchQueueSizeLimit) + { + this.batchQueueSizeLimit = batchQueueSizeLimit; + return this; + } + + public Builder setHttpTimeoutAllowanceFactor(float httpTimeoutAllowanceFactor) + { + this.httpTimeoutAllowanceFactor = httpTimeoutAllowanceFactor; + return this; + } + + public Builder setMinHttpTimeoutMillis(int minHttpTimeoutMillis) + { + this.minHttpTimeoutMillis = minHttpTimeoutMillis; + return this; + } + + public HttpEmitterConfig build() + { + return new HttpEmitterConfig(this, recipientBaseUrl); + } + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java b/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java new file mode 100644 index 000000000000..a9303d9ccaed --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java @@ -0,0 +1,878 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Throwables; +import com.google.common.primitives.Ints; +import io.druid.java.util.common.ISE; +import io.druid.java.util.common.RetryUtils; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.lifecycle.LifecycleStart; +import io.druid.java.util.common.lifecycle.LifecycleStop; +import io.druid.java.util.common.logger.Logger; +import io.netty.handler.codec.http.HttpHeaders; +import org.asynchttpclient.AsyncHttpClient; +import org.asynchttpclient.ListenableFuture; +import org.asynchttpclient.RequestBuilder; +import org.asynchttpclient.Response; + +import java.io.Closeable; +import java.io.Flushable; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.Base64; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; +import java.util.zip.GZIPOutputStream; + +public class HttpPostEmitter implements Flushable, Closeable, Emitter +{ + private static final int MAX_EVENT_SIZE = 1023 * 1024; // Set max size slightly less than 1M to allow for metadata + + private static final int MAX_SEND_RETRIES = 3; + + /** + * Threshold of the size of {@link #buffersToEmit} when switch from using {@link + * BaseHttpEmittingConfig#getHttpTimeoutAllowanceFactor()} to {@link #EQUILIBRIUM_ALLOWANCE_FACTOR} + */ + private static final int EMIT_QUEUE_THRESHOLD_1 = 5; + + /** + * Threshold of the size of {@link #buffersToEmit} when switch from using {@link #EQUILIBRIUM_ALLOWANCE_FACTOR} + * to {@link #TIGHT_ALLOWANCE_FACTOR}. + */ + private static final int EMIT_QUEUE_THRESHOLD_2 = 10; + + /** + * 0.9 is to give room for unexpected latency or time out not being respected rigorously. + */ + private static final double EQUILIBRIUM_ALLOWANCE_FACTOR = 0.9; + + private static final double TIGHT_ALLOWANCE_FACTOR = 0.5; + + /** + * Used in {@link EmittingThread#emitLargeEvents()} to ensure fair emitting of both large events and batched events. + */ + private static final byte[] LARGE_EVENTS_STOP = new byte[]{}; + + private static final Logger log = new Logger(HttpPostEmitter.class); + private static final AtomicInteger instanceCounter = new AtomicInteger(); + + final BatchingStrategy batchingStrategy; + final HttpEmitterConfig config; + private final int bufferSize; + final int maxBufferWatermark; + private final int largeEventThreshold; + private final AsyncHttpClient client; + private final ObjectMapper jsonMapper; + private final String url; + + private final ConcurrentLinkedQueue buffersToReuse = new ConcurrentLinkedQueue<>(); + /** + * "Approximate" because not exactly atomically synchronized with {@link #buffersToReuse} updates. {@link + * ConcurrentLinkedQueue#size()} is not used, because it's O(n). + */ + private final AtomicInteger approximateBuffersToReuseCount = new AtomicInteger(); + + /** + * concurrentBatch.get() == null means the service is closed. + */ + private final AtomicReference concurrentBatch = new AtomicReference<>(); + + private final ConcurrentLinkedDeque buffersToEmit = new ConcurrentLinkedDeque<>(); + /** + * See {@link #approximateBuffersToReuseCount} + */ + private final AtomicInteger approximateBuffersToEmitCount = new AtomicInteger(); + /** + * See {@link #approximateBuffersToReuseCount} + */ + private final AtomicLong approximateEventsToEmitCount = new AtomicLong(); + + private final ConcurrentLinkedQueue largeEventsToEmit = new ConcurrentLinkedQueue<>(); + /** + * See {@link #approximateBuffersToReuseCount} + */ + private final AtomicInteger approximateLargeEventsToEmitCount = new AtomicInteger(); + + private final EmittedBatchCounter emittedBatchCounter = new EmittedBatchCounter(); + private final EmittingThread emittingThread; + private final AtomicLong totalEmittedEvents = new AtomicLong(); + private final AtomicInteger allocatedBuffers = new AtomicInteger(); + private final AtomicInteger droppedBuffers = new AtomicInteger(); + + private volatile long lastFillTimeMillis; + private final ConcurrentTimeCounter batchFillingTimeCounter = new ConcurrentTimeCounter(); + + private final Object startLock = new Object(); + private final CountDownLatch startLatch = new CountDownLatch(1); + private boolean running = false; + + public HttpPostEmitter(HttpEmitterConfig config, AsyncHttpClient client) + { + this(config, client, new ObjectMapper()); + } + + public HttpPostEmitter(HttpEmitterConfig config, AsyncHttpClient client, ObjectMapper jsonMapper) + { + batchingStrategy = config.getBatchingStrategy(); + final int batchOverhead = batchingStrategy.batchStartLength() + batchingStrategy.batchEndLength(); + Preconditions.checkArgument( + config.getMaxBatchSize() >= MAX_EVENT_SIZE + batchOverhead, + StringUtils.format( + "maxBatchSize must be greater than MAX_EVENT_SIZE[%,d] + overhead[%,d].", + MAX_EVENT_SIZE, + batchOverhead + ) + ); + this.config = config; + this.bufferSize = config.getMaxBatchSize(); + this.maxBufferWatermark = bufferSize - batchingStrategy.batchEndLength(); + // Chosen so that if event size < largeEventThreshold, at least 2 events could fit the standard buffer. + this.largeEventThreshold = (bufferSize - batchOverhead - batchingStrategy.separatorLength()) / 2; + this.client = client; + this.jsonMapper = jsonMapper; + try { + this.url = new URL(config.getRecipientBaseUrl()).toString(); + } + catch (MalformedURLException e) { + throw new ISE(e, "Bad URL: %s", config.getRecipientBaseUrl()); + } + emittingThread = new EmittingThread(config); + concurrentBatch.set(new Batch(this, acquireBuffer(), 0)); + // lastFillTimeMillis must not be 0, minHttpTimeoutMillis could be. + lastFillTimeMillis = Math.max(config.minHttpTimeoutMillis, 1); + } + + @Override + @LifecycleStart + public void start() + { + synchronized (startLock) { + if (!running) { + if (startLatch.getCount() == 0) { + throw new IllegalStateException("Already started."); + } + running = true; + startLatch.countDown(); + emittingThread.start(); + } + } + } + + private void awaitStarted() + { + try { + if (!startLatch.await(1, TimeUnit.SECONDS)) { + throw new RejectedExecutionException("Service is not started."); + } + if (isTerminated()) { + throw new RejectedExecutionException("Service is closed."); + } + } + catch (InterruptedException e) { + log.debug("Interrupted waiting for start"); + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + private boolean isTerminated() + { + return concurrentBatch.get() == null; + } + + @Override + public void emit(Event event) + { + emitAndReturnBatch(event); + } + + @VisibleForTesting + Batch emitAndReturnBatch(Event event) + { + awaitStarted(); + + final byte[] eventBytes = eventToBytes(event); + + if (eventBytes.length > MAX_EVENT_SIZE) { + log.error( + "Event too large to emit (%,d > %,d): %s ...", + eventBytes.length, + MAX_EVENT_SIZE, + StringUtils.fromUtf8(ByteBuffer.wrap(eventBytes), 1024) + ); + return null; + } + + if (eventBytes.length > largeEventThreshold) { + writeLargeEvent(eventBytes); + return null; + } + + while (true) { + Batch batch = concurrentBatch.get(); + if (batch == null) { + throw new RejectedExecutionException("Service is closed."); + } + if (batch.tryAddEvent(eventBytes)) { + return batch; + } + // Spin loop, until the thread calling onSealExclusive() updates the concurrentBatch. This update becomes visible + // eventually, because concurrentBatch.get() is a volatile read. + } + } + + private byte[] eventToBytes(Event event) + { + try { + return jsonMapper.writeValueAsBytes(event); + } + catch (IOException e) { + throw Throwables.propagate(e); + } + } + + private void writeLargeEvent(byte[] eventBytes) + { + // It's better to drop the oldest, not latest event, but dropping the oldest is not easy to implement, because + // LARGE_EVENTS_STOP could be added into the queue concurrently. So just not adding the latest event. + // >, not >=, because largeEventsToEmit could contain LARGE_EVENTS_STOP + if (approximateBuffersToEmitCount.get() > config.getBatchQueueSizeLimit()) { + log.error( + "largeEventsToEmit queue size reached the limit [%d], dropping the latest large event", + config.getBatchQueueSizeLimit() + ); + } else { + largeEventsToEmit.add(eventBytes); + approximateLargeEventsToEmitCount.incrementAndGet(); + approximateEventsToEmitCount.incrementAndGet(); + } + wakeUpEmittingThread(); + } + + /** + * Called from {@link Batch} only once for each Batch in existence. + */ + void onSealExclusive(Batch batch, long elapsedTimeMillis) + { + batchFillingTimeCounter.add((int) Math.max(elapsedTimeMillis, 0)); + if (elapsedTimeMillis > 0) { + // If elapsedTimeMillis is 0 or negative, it's likely because System.currentTimeMillis() is not monotonic, so not + // accounting this time for determining batch sending timeout. + lastFillTimeMillis = elapsedTimeMillis; + } + addBatchToEmitQueue(batch); + wakeUpEmittingThread(); + if (!isTerminated()) { + int nextBatchNumber = EmittedBatchCounter.nextBatchNumber(batch.batchNumber); + if (!concurrentBatch.compareAndSet(batch, new Batch(this, acquireBuffer(), nextBatchNumber))) { + // If compareAndSet failed, the service is closed concurrently. + Preconditions.checkState(isTerminated()); + } + } + } + + private void addBatchToEmitQueue(Batch batch) + { + limitBuffersToEmitSize(); + buffersToEmit.addLast(batch); + approximateBuffersToEmitCount.incrementAndGet(); + approximateEventsToEmitCount.addAndGet(batch.eventCount.get()); + } + + private void limitBuffersToEmitSize() + { + if (approximateBuffersToEmitCount.get() >= config.getBatchQueueSizeLimit()) { + Batch droppedBatch = buffersToEmit.pollFirst(); + if (droppedBatch != null) { + batchFinalized(); + approximateBuffersToEmitCount.decrementAndGet(); + approximateEventsToEmitCount.addAndGet(-droppedBatch.eventCount.get()); + droppedBuffers.incrementAndGet(); + log.error( + "buffersToEmit queue size reached the limit [%d], dropping the oldest buffer to emit", + config.getBatchQueueSizeLimit() + ); + } + } + } + + private void batchFinalized() + { + // Notify HttpPostEmitter.flush(), that the batch is emitted, or failed, or dropped. + emittedBatchCounter.batchEmitted(); + } + + private Batch pollBatchFromEmitQueue() + { + Batch result = buffersToEmit.pollFirst(); + if (result == null) { + return null; + } + approximateBuffersToEmitCount.decrementAndGet(); + approximateEventsToEmitCount.addAndGet(-result.eventCount.get()); + return result; + } + + private void wakeUpEmittingThread() + { + LockSupport.unpark(emittingThread); + } + + @Override + public void flush() throws IOException + { + awaitStarted(); + flush(concurrentBatch.get()); + } + + private void flush(Batch batch) throws IOException + { + if (batch == null) { + return; + } + batch.seal(); + try { + // This check doesn't always awaits for this exact batch to be emitted, because another batch could be dropped + // from the queue ahead of this one, in limitBuffersToEmitSize(). But there is no better way currently to wait for + // the exact batch, and it's not that important. + emittedBatchCounter.awaitBatchEmitted(batch.batchNumber, config.getFlushTimeOut(), TimeUnit.MILLISECONDS); + } + catch (TimeoutException e) { + String message = StringUtils.format("Timed out after [%d] millis during flushing", config.getFlushTimeOut()); + throw new IOException(message, e); + } + catch (InterruptedException e) { + log.debug("Thread Interrupted"); + Thread.currentThread().interrupt(); + throw new IOException("Thread Interrupted while flushing", e); + } + } + + @Override + @LifecycleStop + public void close() throws IOException + { + synchronized (startLock) { + if (running) { + running = false; + Batch lastBatch = concurrentBatch.getAndSet(null); + flush(lastBatch); + emittingThread.shuttingDown = true; + // EmittingThread is interrupted after the last batch is flushed. + emittingThread.interrupt(); + } + } + } + + @Override + public String toString() + { + return "HttpPostEmitter{" + + "config=" + config + + '}'; + } + + private class EmittingThread extends Thread + { + private final ArrayDeque failedBuffers = new ArrayDeque<>(); + /** + * "Approximate", because not exactly synchronized with {@link #failedBuffers} updates. Not using size() on + * {@link #failedBuffers}, because access to it is not synchronized, while approximateFailedBuffersCount is queried + * not within EmittingThread. + */ + private final AtomicInteger approximateFailedBuffersCount = new AtomicInteger(); + + private final ConcurrentTimeCounter successfulSendingTimeCounter = new ConcurrentTimeCounter(); + private final ConcurrentTimeCounter failedSendingTimeCounter = new ConcurrentTimeCounter(); + + /** + * Cache the exception. Need an exception because {@link RetryUtils} operates only via exceptions. + */ + private final TimeoutException timeoutLessThanMinimumException; + + private boolean shuttingDown = false; + private ZeroCopyByteArrayOutputStream gzipBaos; + + EmittingThread(HttpEmitterConfig config) + { + super("HttpPostEmitter-" + instanceCounter.incrementAndGet()); + setDaemon(true); + timeoutLessThanMinimumException = new TimeoutException( + "Timeout less than minimum [" + config.getMinHttpTimeoutMillis() + "] ms." + ); + // To not showing and writing nonsense and misleading stack trace in logs. + timeoutLessThanMinimumException.setStackTrace(new StackTraceElement[]{}); + } + + @Override + public void run() + { + while (true) { + boolean needsToShutdown = needsToShutdown(); + try { + emitLargeEvents(); + emitBatches(); + tryEmitOneFailedBuffer(); + + if (needsToShutdown) { + tryEmitAndDrainAllFailedBuffers(); + // Make GC life easier + drainBuffersToReuse(); + return; + } + } + catch (Throwable t) { + log.error(t, "Uncaught exception in EmittingThread.run()"); + } + if (failedBuffers.isEmpty()) { + // Waiting for 1/2 of config.getFlushMillis() in order to flush events not more than 50% later than specified. + // If nanos=0 parkNanos() doesn't wait at all, then we don't want. + long waitNanos = Math.max(TimeUnit.MILLISECONDS.toNanos(config.getFlushMillis()) / 2, 1); + LockSupport.parkNanos(HttpPostEmitter.this, waitNanos); + } + } + } + + private boolean needsToShutdown() + { + boolean needsToShutdown = Thread.interrupted() || shuttingDown; + if (needsToShutdown) { + Batch lastBatch = concurrentBatch.getAndSet(null); + if (lastBatch != null) { + lastBatch.seal(); + } + } else { + Batch batch = concurrentBatch.get(); + if (batch != null) { + batch.sealIfFlushNeeded(); + } else { + needsToShutdown = true; + } + } + return needsToShutdown; + } + + private void emitBatches() + { + for (Batch batch; (batch = pollBatchFromEmitQueue()) != null; ) { + emit(batch); + } + } + + private void emit(final Batch batch) + { + // Awaits until all concurrent event writers finish copy their event bytes to the buffer. This call provides + // memory visibility guarantees. + batch.awaitEmittingAllowed(); + try { + final int bufferWatermark = batch.getSealedBufferWatermark(); + if (bufferWatermark == 0) { // sealed while empty + return; + } + int eventCount = batch.eventCount.get(); + log.debug( + "Sending batch #%d to url[%s], event count[%d], bytes[%d]", + batch.batchNumber, + url, + eventCount, + bufferWatermark + ); + int bufferEndOffset = batchingStrategy.writeBatchEnd(batch.buffer, bufferWatermark); + + if (sendWithRetries(batch.buffer, bufferEndOffset, eventCount, true)) { + buffersToReuse.add(batch.buffer); + approximateBuffersToReuseCount.incrementAndGet(); + } else { + limitFailedBuffersSize(); + failedBuffers.addLast(new FailedBuffer(batch.buffer, bufferEndOffset, eventCount)); + approximateFailedBuffersCount.incrementAndGet(); + } + } + finally { + batchFinalized(); + } + } + + private void limitFailedBuffersSize() + { + if (failedBuffers.size() >= config.getBatchQueueSizeLimit()) { + failedBuffers.removeFirst(); + approximateFailedBuffersCount.decrementAndGet(); + droppedBuffers.incrementAndGet(); + log.error( + "failedBuffers queue size reached the limit [%d], dropping the oldest failed buffer", + config.getBatchQueueSizeLimit() + ); + } + } + + @SuppressWarnings("ArrayEquality") + private void emitLargeEvents() + { + if (largeEventsToEmit.isEmpty()) { + return; + } + // Don't try to emit large events until exhaustion, to avoid starvation of "normal" batches, if large event + // posting rate is too high, though it should never happen in practice. + largeEventsToEmit.add(LARGE_EVENTS_STOP); + for (byte[] largeEvent; (largeEvent = largeEventsToEmit.poll()) != LARGE_EVENTS_STOP; ) { + emitLargeEvent(largeEvent); + approximateLargeEventsToEmitCount.decrementAndGet(); + approximateEventsToEmitCount.decrementAndGet(); + } + } + + private void emitLargeEvent(byte[] eventBytes) + { + byte[] buffer = acquireBuffer(); + int bufferOffset = batchingStrategy.writeBatchStart(buffer); + System.arraycopy(eventBytes, 0, buffer, bufferOffset, eventBytes.length); + bufferOffset += eventBytes.length; + bufferOffset = batchingStrategy.writeBatchEnd(buffer, bufferOffset); + if (sendWithRetries(buffer, bufferOffset, 1, true)) { + buffersToReuse.add(buffer); + approximateBuffersToReuseCount.incrementAndGet(); + } else { + limitFailedBuffersSize(); + failedBuffers.addLast(new FailedBuffer(buffer, bufferOffset, 1)); + approximateFailedBuffersCount.incrementAndGet(); + } + } + + private void tryEmitOneFailedBuffer() + { + FailedBuffer failedBuffer = failedBuffers.peekFirst(); + if (failedBuffer != null) { + if (sendWithRetries(failedBuffer.buffer, failedBuffer.length, failedBuffer.eventCount, false)) { + // Remove from the queue of failed buffer. + failedBuffers.pollFirst(); + approximateFailedBuffersCount.decrementAndGet(); + // Don't add the failed buffer back to the buffersToReuse queue here, because in a situation when we were not + // able to emit events for a while we don't have a way to discard buffers that were used to accumulate events + // during that period, if they are added back to buffersToReuse. For instance it may result in having 100 + // buffers in rotation even if we need just 2. + } + } + } + + private void tryEmitAndDrainAllFailedBuffers() + { + for (FailedBuffer failedBuffer; (failedBuffer = failedBuffers.pollFirst()) != null; ) { + sendWithRetries(failedBuffer.buffer, failedBuffer.length, failedBuffer.eventCount, false); + approximateFailedBuffersCount.decrementAndGet(); + } + } + + /** + * Returns true if sent successfully. + */ + private boolean sendWithRetries(final byte[] buffer, final int length, final int eventCount, boolean withTimeout) + { + long deadLineMillis = System.currentTimeMillis() + sendRequestTimeoutMillis(lastFillTimeMillis); + try { + RetryUtils.retry( + new Callable() + { + @Override + public Void call() throws Exception + { + send(buffer, length); + return null; + } + }, + new Predicate() + { + @Override + public boolean apply(Throwable e) + { + if (withTimeout && deadLineMillis - System.currentTimeMillis() <= 0) { // overflow-aware + return false; + } + if (e == timeoutLessThanMinimumException) { + return false; // Doesn't make sense to retry, because the result will be the same. + } + return !(e instanceof InterruptedException); + } + }, + MAX_SEND_RETRIES + ); + totalEmittedEvents.addAndGet(eventCount); + return true; + } + catch (InterruptedException e) { + return false; + } + catch (Exception e) { + if (e == timeoutLessThanMinimumException) { + log.debug(e, "Failed to send events to url[%s] with timeout less than minimum", config.getRecipientBaseUrl()); + } else { + log.error(e, "Failed to send events to url[%s]", config.getRecipientBaseUrl()); + } + return false; + } + } + + private void send(byte[] buffer, int length) throws Exception + { + long lastFillTimeMillis = HttpPostEmitter.this.lastFillTimeMillis; + final long timeoutMillis = sendRequestTimeoutMillis(lastFillTimeMillis); + if (timeoutMillis < config.getMinHttpTimeoutMillis()) { + throw timeoutLessThanMinimumException; + } + long sendingStartMs = System.currentTimeMillis(); + + final RequestBuilder request = new RequestBuilder("POST"); + request.setUrl(url); + byte[] payload; + int payloadLength; + ContentEncoding contentEncoding = config.getContentEncoding(); + if (contentEncoding != null) { + switch (contentEncoding) { + case GZIP: + try (GZIPOutputStream gzipOutputStream = acquireGzipOutputStream(length)) { + gzipOutputStream.write(buffer, 0, length); + } + payload = gzipBaos.getBuffer(); + payloadLength = gzipBaos.size(); + request.setHeader(HttpHeaders.Names.CONTENT_ENCODING, HttpHeaders.Values.GZIP); + break; + default: + throw new ISE("Unsupported content encoding [%s]", contentEncoding.name()); + } + } else { + payload = buffer; + payloadLength = length; + } + + + request.setHeader(HttpHeaders.Names.CONTENT_TYPE, "application/json"); + request.setHeader(HttpHeaders.Names.CONTENT_LENGTH, String.valueOf(payloadLength)); + request.setBody(ByteBuffer.wrap(payload, 0, payloadLength)); + + if (config.getBasicAuthentication() != null) { + final String[] parts = config.getBasicAuthentication().split(":", 2); + final String user = parts[0]; + final String password = parts.length > 1 ? parts[1] : ""; + String encoded = Base64.getEncoder().encodeToString((user + ':' + password).getBytes(StandardCharsets.UTF_8)); + request.setHeader(HttpHeaders.Names.AUTHORIZATION, "Basic " + encoded); + } + + request.setRequestTimeout(Ints.saturatedCast(timeoutMillis)); + + ListenableFuture future = client.executeRequest(request); + Response response; + try { + // Don't use Future.get(timeout), because we want to avoid sending the same data twice, in case the send + // succeeds finally, but after the timeout. + response = future.get(); + } + catch (ExecutionException e) { + accountFailedSending(sendingStartMs); + if (e.getCause() instanceof TimeoutException) { + log.error( + "Timing out emitter batch send, last batch fill time [%,d] ms, timeout [%,d] ms", + lastFillTimeMillis, + timeoutMillis + ); + } + throw e; + } + + if (response.getStatusCode() == 413) { + accountFailedSending(sendingStartMs); + throw new ISE( + "Received HTTP status 413 from [%s]. Batch size of [%d] may be too large, " + + "try adjusting maxBatchSizeBatch property", + config.getRecipientBaseUrl(), + config.getMaxBatchSize() + ); + } + + if (response.getStatusCode() / 100 != 2) { + accountFailedSending(sendingStartMs); + throw new ISE( + "Emissions of events not successful[%d: %s], with message[%s].", + response.getStatusCode(), + response.getStatusText(), + response.getResponseBody(StandardCharsets.UTF_8).trim() + ); + } + + accountSuccessfulSending(sendingStartMs); + } + + private long sendRequestTimeoutMillis(long lastFillTimeMillis) + { + int emitQueueSize = approximateBuffersToEmitCount.get(); + if (emitQueueSize < EMIT_QUEUE_THRESHOLD_1) { + return (long) (lastFillTimeMillis * config.httpTimeoutAllowanceFactor); + } + if (emitQueueSize < EMIT_QUEUE_THRESHOLD_2) { + // The idea is to not let buffersToEmit queue to grow faster than we can emit buffers. + return (long) (lastFillTimeMillis * EQUILIBRIUM_ALLOWANCE_FACTOR); + } + // If buffersToEmit still grows, try to restrict even more + return (long) (lastFillTimeMillis * TIGHT_ALLOWANCE_FACTOR); + } + + private void accountSuccessfulSending(long sendingStartMs) + { + successfulSendingTimeCounter.add((int) Math.max(System.currentTimeMillis() - sendingStartMs, 0)); + } + + private void accountFailedSending(long sendingStartMs) + { + failedSendingTimeCounter.add((int) Math.max(System.currentTimeMillis() - sendingStartMs, 0)); + } + + GZIPOutputStream acquireGzipOutputStream(int length) throws IOException + { + if (gzipBaos == null) { + gzipBaos = new ZeroCopyByteArrayOutputStream(length); + } else { + gzipBaos.reset(); + } + return new GZIPOutputStream(gzipBaos, true); + } + } + + private static class FailedBuffer + { + final byte[] buffer; + final int length; + final int eventCount; + + private FailedBuffer(byte[] buffer, int length, int eventCount) + { + this.buffer = buffer; + this.length = length; + this.eventCount = eventCount; + } + } + + private byte[] acquireBuffer() + { + byte[] buffer = buffersToReuse.poll(); + if (buffer == null) { + buffer = new byte[bufferSize]; + allocatedBuffers.incrementAndGet(); + } else { + approximateBuffersToReuseCount.decrementAndGet(); + } + return buffer; + } + + private void drainBuffersToReuse() + { + while (buffersToReuse.poll() != null) { + approximateBuffersToReuseCount.decrementAndGet(); + } + } + + /** + * This and the following methods are public for external monitoring purposes. + */ + public int getTotalAllocatedBuffers() + { + return allocatedBuffers.get(); + } + + public int getBuffersToEmit() + { + return approximateBuffersToEmitCount.get(); + } + + public int getBuffersToReuse() + { + return approximateBuffersToReuseCount.get(); + } + + public int getFailedBuffers() + { + return emittingThread.approximateFailedBuffersCount.get(); + } + + public int getDroppedBuffers() + { + return droppedBuffers.get(); + } + + public long getTotalEmittedEvents() + { + return totalEmittedEvents.get(); + } + + public long getEventsToEmit() + { + return approximateEventsToEmitCount.get(); + } + + public long getLargeEventsToEmit() + { + return approximateLargeEventsToEmitCount.get(); + } + + public ConcurrentTimeCounter getBatchFillingTimeCounter() + { + return batchFillingTimeCounter; + } + + public ConcurrentTimeCounter getSuccessfulSendingTimeCounter() + { + return emittingThread.successfulSendingTimeCounter; + } + + public ConcurrentTimeCounter getFailedSendingTimeCounter() + { + return emittingThread.successfulSendingTimeCounter; + } + + @VisibleForTesting + void waitForEmission(int batchNumber) throws Exception + { + emittedBatchCounter.awaitBatchEmitted(batchNumber, 10, TimeUnit.SECONDS); + } + + @VisibleForTesting + void joinEmitterThread() throws InterruptedException + { + emittingThread.join(); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/LoggingEmitter.java b/java-util/src/main/java/io/druid/java/util/emitter/core/LoggingEmitter.java new file mode 100644 index 000000000000..5bf21d88e0e2 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/LoggingEmitter.java @@ -0,0 +1,190 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +/** + */ + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.lifecycle.LifecycleStart; +import io.druid.java.util.common.lifecycle.LifecycleStop; +import io.druid.java.util.common.logger.Logger; + +import java.io.IOException; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + */ +public class LoggingEmitter implements Emitter +{ + private final Logger log; + private final Level level; + private final ObjectMapper jsonMapper; + + private final AtomicBoolean started = new AtomicBoolean(false); + + public LoggingEmitter(LoggingEmitterConfig config, ObjectMapper jsonMapper) + { + this(new Logger(config.getLoggerClass()), Level.toLevel(config.getLogLevel()), jsonMapper); + } + + public LoggingEmitter(Logger log, Level level, ObjectMapper jsonMapper) + { + this.log = log; + this.level = level; + this.jsonMapper = jsonMapper; + } + + @Override + @LifecycleStart + public void start() + { + final boolean alreadyStarted = started.getAndSet(true); + if (!alreadyStarted) { + final String message = "Start: started [%s]"; + switch (level) { + case TRACE: + if (log.isTraceEnabled()) { + log.trace(message, started.get()); + } + break; + case DEBUG: + if (log.isDebugEnabled()) { + log.debug(message, started.get()); + } + break; + case INFO: + if (log.isInfoEnabled()) { + log.info(message, started.get()); + } + break; + case WARN: + log.warn(message, started.get()); + break; + case ERROR: + log.error(message, started.get()); + break; + } + } + } + + @Override + public void emit(Event event) + { + synchronized (started) { + if (!started.get()) { + throw new RejectedExecutionException("Service not started."); + } + } + try { + final String message = "Event [%s]"; + switch (level) { + case TRACE: + if (log.isTraceEnabled()) { + log.trace(message, jsonMapper.writeValueAsString(event)); + } + break; + case DEBUG: + if (log.isDebugEnabled()) { + log.debug(message, jsonMapper.writeValueAsString(event)); + } + break; + case INFO: + if (log.isInfoEnabled()) { + log.info(message, jsonMapper.writeValueAsString(event)); + } + break; + case WARN: + log.warn(message, jsonMapper.writeValueAsString(event)); + break; + case ERROR: + log.error(message, jsonMapper.writeValueAsString(event)); + break; + } + } + catch (Exception e) { + log.warn(e, "Failed to generate json"); + } + } + + @Override + public void flush() throws IOException + { + + } + + @Override + @LifecycleStop + public void close() throws IOException + { + final boolean wasStarted = started.getAndSet(false); + if (wasStarted) { + final String message = "Close: started [%s]"; + switch (level) { + case TRACE: + if (log.isTraceEnabled()) { + log.trace(message, started.get()); + } + break; + case DEBUG: + if (log.isDebugEnabled()) { + log.debug(message, started.get()); + } + break; + case INFO: + if (log.isInfoEnabled()) { + log.info(message, started.get()); + } + break; + case WARN: + log.warn(message, started.get()); + break; + case ERROR: + log.error(message, started.get()); + break; + } + } + } + + @Override + public String toString() + { + return "LoggingEmitter{" + + "log=" + log + + ", level=" + level + + '}'; + } + + public enum Level + { + TRACE, + DEBUG, + INFO, + WARN, + ERROR; + + public static Level toLevel(String name) + { + return Level.valueOf(StringUtils.toUpperCase(name)); + } + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/LoggingEmitterConfig.java b/java-util/src/main/java/io/druid/java/util/emitter/core/LoggingEmitterConfig.java new file mode 100644 index 000000000000..c4aba3de01c4 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/LoggingEmitterConfig.java @@ -0,0 +1,66 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.NotNull; + +/** + */ +public class LoggingEmitterConfig +{ + @NotNull + @JsonProperty + private String loggerClass = LoggingEmitter.class.getName(); + + @NotNull + @JsonProperty + private String logLevel = "info"; + + public String getLoggerClass() + { + return loggerClass; + } + + public void setLoggerClass(String loggerClass) + { + this.loggerClass = loggerClass; + } + + public String getLogLevel() + { + return logLevel; + } + + public void setLogLevel(String logLevel) + { + this.logLevel = logLevel; + } + + @Override + public String toString() + { + return "LoggingEmitterConfig{" + + "loggerClass='" + loggerClass + '\'' + + ", logLevel='" + logLevel + '\'' + + '}'; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/NoopEmitter.java b/java-util/src/main/java/io/druid/java/util/emitter/core/NoopEmitter.java new file mode 100644 index 000000000000..13897ddd3900 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/NoopEmitter.java @@ -0,0 +1,51 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import java.io.IOException; + +/** + */ +public class NoopEmitter implements Emitter +{ + @Override + public void start() + { + // Do nothing + } + + @Override + public void emit(Event event) + { + // Do nothing + } + + @Override + public void flush() throws IOException + { + // Do nothing + } + + @Override + public void close() throws IOException + { + // Do nothing + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/ParametrizedUriEmitter.java b/java-util/src/main/java/io/druid/java/util/emitter/core/ParametrizedUriEmitter.java new file mode 100644 index 000000000000..18934c4f3fb7 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/ParametrizedUriEmitter.java @@ -0,0 +1,212 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableSet; +import io.druid.java.util.common.lifecycle.Lifecycle; +import io.druid.java.util.common.lifecycle.LifecycleStart; +import io.druid.java.util.common.lifecycle.LifecycleStop; +import io.druid.java.util.common.logger.Logger; +import org.asynchttpclient.AsyncHttpClient; + +import javax.annotation.concurrent.GuardedBy; +import java.io.Closeable; +import java.io.Flushable; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiConsumer; + +public class ParametrizedUriEmitter implements Flushable, Closeable, Emitter +{ + private static final Logger log = new Logger(ParametrizedUriEmitter.class); + private static final Set ONLY_FEED_PARAM = ImmutableSet.of("feed"); + + private static UriExtractor makeUriExtractor(ParametrizedUriEmitterConfig config) + { + final String baseUri = config.getRecipientBaseUrlPattern(); + final ParametrizedUriExtractor parametrizedUriExtractor = new ParametrizedUriExtractor(baseUri); + UriExtractor uriExtractor = parametrizedUriExtractor; + if (ONLY_FEED_PARAM.equals(parametrizedUriExtractor.getParams())) { + uriExtractor = new FeedUriExtractor(baseUri.replace("{feed}", "%s")); + } + return uriExtractor; + } + + /** + * Type should be ConcurrentHashMap, not {@link java.util.concurrent.ConcurrentMap}, because the latter _doesn't_ + * guarantee that the lambda passed to {@link java.util.Map#computeIfAbsent} is executed at most once. + */ + private final ConcurrentHashMap emitters = new ConcurrentHashMap<>(); + private final UriExtractor uriExtractor; + private final Object startCloseLock = new Object(); + @GuardedBy("startCloseLock") + private boolean started = false; + @GuardedBy("startCloseLock") + private boolean closed = false; + private final Lifecycle innerLifecycle = new Lifecycle(); + private final AsyncHttpClient client; + private final ObjectMapper jsonMapper; + private final ParametrizedUriEmitterConfig config; + + public ParametrizedUriEmitter( + ParametrizedUriEmitterConfig config, + AsyncHttpClient client, + ObjectMapper jsonMapper + ) + { + this(config, client, jsonMapper, makeUriExtractor(config)); + } + + public ParametrizedUriEmitter( + ParametrizedUriEmitterConfig config, + AsyncHttpClient client, + ObjectMapper jsonMapper, + UriExtractor uriExtractor + ) + { + this.config = config; + this.client = client; + this.jsonMapper = jsonMapper; + this.uriExtractor = uriExtractor; + } + + @Override + @LifecycleStart + public void start() + { + // Use full synchronized instead of atomic flag, because otherwise some thread may think that the emitter is already + // started while it's in the process of starting by another thread. + synchronized (startCloseLock) { + if (started) { + return; + } + started = true; + try { + innerLifecycle.start(); + } + catch (RuntimeException e) { + throw e; + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + @Override + public void emit(Event event) + { + try { + URI uri = uriExtractor.apply(event); + HttpPostEmitter emitter = emitters.get(uri); + if (emitter == null) { + try { + emitter = emitters.computeIfAbsent(uri, u -> { + try { + return innerLifecycle.addMaybeStartManagedInstance( + new HttpPostEmitter( + config.buildHttpEmitterConfig(u.toString()), + client, + jsonMapper + ) + ); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + }); + } + catch (RuntimeException e) { + log.error(e, "Error while creating or starting an HttpPostEmitter for URI[%s]", uri); + return; + } + } + emitter.emit(event); + } + catch (URISyntaxException e) { + log.error(e, "Failed to extract URI for event[%s]", event.toMap()); + } + } + + @Override + @LifecycleStop + public void close() throws IOException + { + // Use full synchronized instead of atomic flag, because otherwise some thread may think that the emitter is already + // closed while it's in the process of closing by another thread. + synchronized (startCloseLock) { + if (closed) { + return; + } + closed = true; + innerLifecycle.stop(); + } + } + + @Override + public void flush() throws IOException + { + Exception thrown = null; + for (HttpPostEmitter httpPostEmitter : emitters.values()) { + try { + httpPostEmitter.flush(); + } + catch (Exception e) { + // If flush was interrupted, exit the loop + if (Thread.currentThread().isInterrupted()) { + if (thrown != null) { + e.addSuppressed(thrown); + } + throw Throwables.propagate(e); + } + if (thrown == null) { + thrown = e; + } else { + if (thrown != e) { + thrown.addSuppressed(e); + } + } + } + } + if (thrown != null) { + throw Throwables.propagate(thrown); + } + } + + public void forEachEmitter(BiConsumer action) + { + emitters.forEach(action); + } + + @Override + public String toString() + { + return "ParametrizedUriEmitter{" + + "emitters=" + emitters.keySet() + + ", uriExtractor=" + uriExtractor + + ", config=" + config + + '}'; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/ParametrizedUriEmitterConfig.java b/java-util/src/main/java/io/druid/java/util/emitter/core/ParametrizedUriEmitterConfig.java new file mode 100644 index 000000000000..ba70ea550885 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/ParametrizedUriEmitterConfig.java @@ -0,0 +1,55 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.NotNull; + +public class ParametrizedUriEmitterConfig +{ + private static final BaseHttpEmittingConfig DEFAULT_HTTP_EMITTING_CONFIG = new BaseHttpEmittingConfig(); + + @NotNull + @JsonProperty + private String recipientBaseUrlPattern; + + @JsonProperty("httpEmitting") + private BaseHttpEmittingConfig httpEmittingConfig = DEFAULT_HTTP_EMITTING_CONFIG; + + public String getRecipientBaseUrlPattern() + { + return recipientBaseUrlPattern; + } + + public HttpEmitterConfig buildHttpEmitterConfig(String baseUri) + { + return new HttpEmitterConfig(httpEmittingConfig, baseUri); + } + + @Override + public String toString() + { + return "ParametrizedUriEmitterConfig{" + + "recipientBaseUrlPattern='" + recipientBaseUrlPattern + '\'' + + ", httpEmittingConfig=" + httpEmittingConfig + + '}'; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/ParametrizedUriExtractor.java b/java-util/src/main/java/io/druid/java/util/emitter/core/ParametrizedUriExtractor.java new file mode 100644 index 000000000000..6d29bd58ebcd --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/ParametrizedUriExtractor.java @@ -0,0 +1,80 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import io.druid.java.util.common.StringUtils; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ParametrizedUriExtractor implements UriExtractor +{ + private String uriPattern; + private Set params; + + public ParametrizedUriExtractor(String uriPattern) + { + this.uriPattern = uriPattern; + Matcher keyMatcher = Pattern.compile("\\{([^\\}]+)\\}").matcher(uriPattern); + params = new HashSet<>(); + while (keyMatcher.find()) { + params.add(keyMatcher.group(1)); + } + } + + public Set getParams() + { + return params; + } + + @Override + public URI apply(Event event) throws URISyntaxException + { + Map eventMap = event.toMap(); + String processedUri = uriPattern; + for (String key : params) { + Object paramValue = eventMap.get(key); + if (paramValue == null) { + throw new IllegalArgumentException(StringUtils.format( + "ParametrizedUriExtractor with pattern %s requires %s to be set in event, but found %s", + uriPattern, + key, + eventMap + )); + } + processedUri = processedUri.replace(StringUtils.format("{%s}", key), paramValue.toString()); + } + return new URI(processedUri); + } + + @Override + public String toString() + { + return "ParametrizedUriExtractor{" + + "uriPattern='" + uriPattern + '\'' + + ", params=" + params + + '}'; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/UriExtractor.java b/java-util/src/main/java/io/druid/java/util/emitter/core/UriExtractor.java new file mode 100644 index 000000000000..42e57487c5a3 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/UriExtractor.java @@ -0,0 +1,28 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import java.net.URI; +import java.net.URISyntaxException; + +public interface UriExtractor +{ + URI apply(Event event) throws URISyntaxException; +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/ZeroCopyByteArrayOutputStream.java b/java-util/src/main/java/io/druid/java/util/emitter/core/ZeroCopyByteArrayOutputStream.java new file mode 100644 index 000000000000..15e6b86935cd --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/ZeroCopyByteArrayOutputStream.java @@ -0,0 +1,36 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import java.io.ByteArrayOutputStream; + +class ZeroCopyByteArrayOutputStream extends ByteArrayOutputStream +{ + + ZeroCopyByteArrayOutputStream(int capacity) + { + super(capacity); + } + + byte[] getBuffer() + { + return buf; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/factory/EmitterFactory.java b/java-util/src/main/java/io/druid/java/util/emitter/factory/EmitterFactory.java new file mode 100644 index 000000000000..c45ff6402471 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/factory/EmitterFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.factory; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.java.util.common.lifecycle.Lifecycle; +import io.druid.java.util.emitter.core.Emitter; +import org.asynchttpclient.AsyncHttpClient; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "http", value = HttpEmitterFactory.class), + @JsonSubTypes.Type(name = "logging", value = LoggingEmitterFactory.class), + @JsonSubTypes.Type(name = "parametrized", value = ParametrizedUriEmitterFactory.class), + @JsonSubTypes.Type(name = "noop", value = NoopEmitterFactory.class), +}) +public interface EmitterFactory +{ + Emitter makeEmitter(ObjectMapper objectMapper, AsyncHttpClient httpClient, Lifecycle lifecycle); +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/factory/HttpEmitterFactory.java b/java-util/src/main/java/io/druid/java/util/emitter/factory/HttpEmitterFactory.java new file mode 100644 index 000000000000..fd43bd56111d --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/factory/HttpEmitterFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.factory; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.java.util.common.lifecycle.Lifecycle; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.HttpEmitterConfig; +import io.druid.java.util.emitter.core.HttpPostEmitter; +import org.asynchttpclient.AsyncHttpClient; + +public class HttpEmitterFactory extends HttpEmitterConfig implements EmitterFactory +{ + + @Override + public Emitter makeEmitter(ObjectMapper objectMapper, AsyncHttpClient httpClient, Lifecycle lifecycle) + { + Emitter retVal = new HttpPostEmitter(this, httpClient, objectMapper); + lifecycle.addManagedInstance(retVal); + return retVal; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/factory/LoggingEmitterFactory.java b/java-util/src/main/java/io/druid/java/util/emitter/factory/LoggingEmitterFactory.java new file mode 100644 index 000000000000..5e7ee9325e16 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/factory/LoggingEmitterFactory.java @@ -0,0 +1,45 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.factory; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.java.util.common.lifecycle.Lifecycle; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.LoggingEmitter; +import io.druid.java.util.emitter.core.LoggingEmitterConfig; +import org.asynchttpclient.AsyncHttpClient; + +public class LoggingEmitterFactory extends LoggingEmitterConfig implements EmitterFactory +{ + public LoggingEmitterFactory() {} + + @Override + public Emitter makeEmitter(ObjectMapper objectMapper, AsyncHttpClient httpClient, Lifecycle lifecycle) + { + return makeEmitter(objectMapper, lifecycle); + } + + public Emitter makeEmitter(ObjectMapper objectMapper, Lifecycle lifecycle) + { + Emitter retVal = new LoggingEmitter(this, objectMapper); + lifecycle.addManagedInstance(retVal); + return retVal; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/factory/NoopEmitterFactory.java b/java-util/src/main/java/io/druid/java/util/emitter/factory/NoopEmitterFactory.java new file mode 100644 index 000000000000..1e89c7b87f50 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/factory/NoopEmitterFactory.java @@ -0,0 +1,42 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.factory; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.java.util.common.lifecycle.Lifecycle; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.NoopEmitter; +import org.asynchttpclient.AsyncHttpClient; + +public class NoopEmitterFactory implements EmitterFactory +{ + @Override + public Emitter makeEmitter(ObjectMapper objectMapper, AsyncHttpClient httpClient, Lifecycle lifecycle) + { + return makeEmitter(lifecycle); + } + + public Emitter makeEmitter(Lifecycle lifecycle) + { + Emitter retVal = new NoopEmitter(); + lifecycle.addManagedInstance(retVal); + return retVal; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/factory/ParametrizedUriEmitterFactory.java b/java-util/src/main/java/io/druid/java/util/emitter/factory/ParametrizedUriEmitterFactory.java new file mode 100644 index 000000000000..0c6956be9aaa --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/factory/ParametrizedUriEmitterFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.factory; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.java.util.common.lifecycle.Lifecycle; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.ParametrizedUriEmitter; +import io.druid.java.util.emitter.core.ParametrizedUriEmitterConfig; +import org.asynchttpclient.AsyncHttpClient; + +public class ParametrizedUriEmitterFactory extends ParametrizedUriEmitterConfig implements EmitterFactory +{ + + @Override + public Emitter makeEmitter(ObjectMapper objectMapper, AsyncHttpClient httpClient, Lifecycle lifecycle) + { + final Emitter retVal = new ParametrizedUriEmitter(this, httpClient, objectMapper); + lifecycle.addManagedInstance(retVal); + return retVal; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/service/AlertBuilder.java b/java-util/src/main/java/io/druid/java/util/emitter/service/AlertBuilder.java new file mode 100644 index 000000000000..705da242c28f --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/service/AlertBuilder.java @@ -0,0 +1,90 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.service; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import io.druid.java.util.common.DateTimes; +import io.druid.java.util.common.StringUtils; + +import java.util.Map; + +/** +*/ +public class AlertBuilder extends ServiceEventBuilder +{ + protected final Map dataMap = Maps.newLinkedHashMap(); + protected final String description; + protected final ServiceEmitter emitter; + + protected AlertEvent.Severity severity = AlertEvent.Severity.DEFAULT; + + public static AlertBuilder create(String descriptionFormat, Object... objects) + { + return AlertBuilder.createEmittable(null, descriptionFormat, objects); + } + + public static AlertBuilder createEmittable(ServiceEmitter emitter, String descriptionFormat, Object... objects) + { + return new AlertBuilder(StringUtils.format(descriptionFormat, objects), emitter); + } + + protected AlertBuilder( + String description, + ServiceEmitter emitter + ) + { + this.description = description; + this.emitter = emitter; + } + + public AlertBuilder addData(String identifier, Object value) + { + dataMap.put(identifier, value); + return this; + } + + public AlertBuilder addData(Map data) + { + dataMap.putAll(data); + return this; + } + + public AlertBuilder severity(AlertEvent.Severity severity) + { + this.severity = severity; + return this; + } + + @Override + public AlertEvent build(ImmutableMap serviceDimensions) + { + return new AlertEvent(DateTimes.nowUtc(), serviceDimensions, severity, description, dataMap); + } + + public void emit() + { + if (emitter == null) { + throw new UnsupportedOperationException("Emitter is null, cannot emit."); + } + + emitter.emit(this); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/service/AlertEvent.java b/java-util/src/main/java/io/druid/java/util/emitter/service/AlertEvent.java new file mode 100644 index 000000000000..81fc432e84c3 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/service/AlertEvent.java @@ -0,0 +1,185 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.service; + +import com.fasterxml.jackson.annotation.JsonValue; +import com.google.common.collect.ImmutableMap; +import io.druid.java.util.common.DateTimes; +import org.joda.time.DateTime; + +import java.util.Collections; +import java.util.Map; + +/** + */ +public class AlertEvent implements ServiceEvent +{ + private final ImmutableMap serviceDimensions; + private final Severity severity; + private final String description; + private final DateTime createdTime; + + private final Map dataMap; + + public AlertEvent( + DateTime createdTime, + ImmutableMap serviceDimensions, + Severity severity, + String description, + Map dataMap + ) + { + this.createdTime = createdTime; + this.serviceDimensions = serviceDimensions; + this.severity = severity; + this.description = description; + this.dataMap = dataMap; + } + + public AlertEvent( + DateTime createdTime, + String service, + String host, + Severity severity, + String description, + Map dataMap + ) + { + this(createdTime, ImmutableMap.of("service", service, "host", host), severity, description, dataMap); + } + + public AlertEvent( + String service, + String host, + Severity severity, + String description, + Map dataMap + ) + { + this(DateTimes.nowUtc(), service, host, severity, description, dataMap); + } + + public AlertEvent( + String service, + String host, + String description, + Map dataMap + ) + { + this(DateTimes.nowUtc(), service, host, Severity.DEFAULT, description, dataMap); + } + + public AlertEvent( + String service, + String host, + String description + ) + { + this(DateTimes.nowUtc(), service, host, Severity.DEFAULT, description, ImmutableMap.of()); + } + + @Override + public DateTime getCreatedTime() + { + return createdTime; + } + + @Override + public String getFeed() + { + return "alerts"; + } + + @Override + public String getService() + { + return serviceDimensions.get("service"); + } + + @Override + public String getHost() + { + return serviceDimensions.get("host"); + } + + @Override + public boolean isSafeToBuffer() + { + return false; + } + + public Severity getSeverity() + { + return severity; + } + + public String getDescription() + { + return description; + } + + public Map getDataMap() + { + return Collections.unmodifiableMap(dataMap); + } + + @Override + @JsonValue + public Map toMap() + { + return ImmutableMap.builder() + .put("feed", getFeed()) + .put("timestamp", createdTime.toString()) + .putAll(serviceDimensions) + .put("severity", severity.toString()) + .put("description", description) + .put("data", dataMap) + .build(); + } + + public enum Severity + { + ANOMALY { + @Override + public String toString() + { + return "anomaly"; + } + }, + + COMPONENT_FAILURE { + @Override + public String toString() + { + return "component-failure"; + } + }, + + SERVICE_FAILURE { + @Override + public String toString() + { + return "service-failure"; + } + }; + + public static final Severity DEFAULT = COMPONENT_FAILURE; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/service/ServiceEmitter.java b/java-util/src/main/java/io/druid/java/util/emitter/service/ServiceEmitter.java new file mode 100644 index 000000000000..f18fdf09cfac --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/service/ServiceEmitter.java @@ -0,0 +1,106 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.service; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import io.druid.java.util.common.lifecycle.LifecycleStart; +import io.druid.java.util.common.lifecycle.LifecycleStop; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.Event; + +import java.io.IOException; + +public class ServiceEmitter implements Emitter +{ + private final ImmutableMap serviceDimensions; + private final Emitter emitter; + + public ServiceEmitter(String service, String host, Emitter emitter) + { + this(service, host, emitter, ImmutableMap.of()); + } + + public ServiceEmitter( + String service, + String host, + Emitter emitter, + ImmutableMap otherServiceDimensions + ) + { + this.serviceDimensions = ImmutableMap + .builder() + .put("service", Preconditions.checkNotNull(service)) + .put("host", Preconditions.checkNotNull(host)) + .putAll(otherServiceDimensions) + .build(); + this.emitter = emitter; + } + + public String getService() + { + return serviceDimensions.get("service"); + } + + public String getHost() + { + return serviceDimensions.get("host"); + } + + @Override + @LifecycleStart + public void start() + { + emitter.start(); + } + + @Override + public void emit(Event event) + { + emitter.emit(event); + } + + public void emit(ServiceEventBuilder builder) + { + emit(builder.build(serviceDimensions)); + } + + @Override + public void flush() throws IOException + { + emitter.flush(); + } + + @Override + @LifecycleStop + public void close() throws IOException + { + emitter.close(); + } + + @Override + public String toString() + { + return "ServiceEmitter{" + + "serviceDimensions=" + serviceDimensions + + ", emitter=" + emitter + + '}'; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/service/ServiceEvent.java b/java-util/src/main/java/io/druid/java/util/emitter/service/ServiceEvent.java new file mode 100644 index 000000000000..4e983cd464dc --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/service/ServiceEvent.java @@ -0,0 +1,29 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.service; + +import io.druid.java.util.emitter.core.Event; + +public interface ServiceEvent extends Event +{ + String getService(); + + String getHost(); +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/service/ServiceEventBuilder.java b/java-util/src/main/java/io/druid/java/util/emitter/service/ServiceEventBuilder.java new file mode 100644 index 000000000000..0fc45e57a400 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/service/ServiceEventBuilder.java @@ -0,0 +1,33 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.service; + +import com.google.common.collect.ImmutableMap; +import io.druid.java.util.emitter.core.Event; + +public abstract class ServiceEventBuilder +{ + public abstract X build(ImmutableMap serviceDimensions); + + public X build(String service, String host) + { + return build(ImmutableMap.of("service", service, "host", host)); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/service/ServiceMetricEvent.java b/java-util/src/main/java/io/druid/java/util/emitter/service/ServiceMetricEvent.java new file mode 100644 index 000000000000..0c4bc9b549bc --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/emitter/service/ServiceMetricEvent.java @@ -0,0 +1,203 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.service; + +import com.fasterxml.jackson.annotation.JsonValue; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import io.druid.java.util.common.DateTimes; +import io.druid.java.util.common.ISE; +import org.joda.time.DateTime; + +import java.util.Arrays; +import java.util.Map; + +/** + */ +public class ServiceMetricEvent implements ServiceEvent +{ + public static Builder builder() + { + return new Builder(); + } + + private final DateTime createdTime; + private final ImmutableMap serviceDims; + private final Map userDims; + private final String feed; + private final String metric; + private final Number value; + + private ServiceMetricEvent( + DateTime createdTime, + ImmutableMap serviceDims, + Map userDims, + String feed, + String metric, + Number value + ) + { + this.createdTime = createdTime != null ? createdTime : DateTimes.nowUtc(); + this.serviceDims = serviceDims; + this.userDims = userDims; + this.feed = feed; + this.metric = metric; + this.value = value; + } + + @Override + public DateTime getCreatedTime() + { + return createdTime; + } + + @Override + public String getFeed() + { + return feed; + } + + @Override + public String getService() + { + return serviceDims.get("service"); + } + + @Override + public String getHost() + { + return serviceDims.get("host"); + } + + public Map getUserDims() + { + return ImmutableMap.copyOf(userDims); + } + + public String getMetric() + { + return metric; + } + + public Number getValue() + { + return value; + } + + @Override + public boolean isSafeToBuffer() + { + return true; + } + + @Override + @JsonValue + public Map toMap() + { + return ImmutableMap.builder() + .put("feed", getFeed()) + .put("timestamp", createdTime.toString()) + .putAll(serviceDims) + .put("metric", metric) + .put("value", value) + .putAll( + Maps.filterEntries( + userDims, + new Predicate>() + { + @Override + public boolean apply(Map.Entry input) + { + return input.getKey() != null; + } + } + ) + ) + .build(); + } + + public static class Builder + { + private final Map userDims = Maps.newTreeMap(); + private String feed = "metrics"; + + public Builder setFeed(String feed) + { + this.feed = feed; + return this; + } + + public Builder setDimension(String dim, String[] values) + { + userDims.put(dim, Arrays.asList(values)); + return this; + } + + public Builder setDimension(String dim, String value) + { + userDims.put(dim, value); + return this; + } + + public Object getDimension(String dim) + { + return userDims.get(dim); + } + + public ServiceEventBuilder build( + final String metric, + final Number value + ) + { + return build(null, metric, value); + } + + public ServiceEventBuilder build( + final DateTime createdTime, + final String metric, + final Number value + ) + { + if (Double.isNaN(value.doubleValue())) { + throw new ISE("Value of NaN is not allowed!"); + } + if (Double.isInfinite(value.doubleValue())) { + throw new ISE("Value of Infinite is not allowed!"); + } + + return new ServiceEventBuilder() + { + @Override + public ServiceMetricEvent build(ImmutableMap serviceDimensions) + { + return new ServiceMetricEvent( + createdTime, + serviceDimensions, + userDims, + feed, + metric, + value + ); + } + }; + } + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/AbstractHttpClient.java b/java-util/src/main/java/io/druid/java/util/http/client/AbstractHttpClient.java new file mode 100644 index 000000000000..8335aa561097 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/AbstractHttpClient.java @@ -0,0 +1,35 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client; + +import com.google.common.util.concurrent.ListenableFuture; +import io.druid.java.util.http.client.response.HttpResponseHandler; + +public abstract class AbstractHttpClient implements HttpClient +{ + @Override + public ListenableFuture go( + final Request request, + final HttpResponseHandler handler + ) + { + return go(request, handler, null); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/CredentialedHttpClient.java b/java-util/src/main/java/io/druid/java/util/http/client/CredentialedHttpClient.java new file mode 100644 index 000000000000..d56894433530 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/CredentialedHttpClient.java @@ -0,0 +1,50 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client; + +import com.google.common.util.concurrent.ListenableFuture; +import io.druid.java.util.http.client.auth.Credentials; +import io.druid.java.util.http.client.response.HttpResponseHandler; +import org.joda.time.Duration; + +/** + */ +public class CredentialedHttpClient extends AbstractHttpClient +{ + + private final Credentials creds; + private final HttpClient delegate; + + public CredentialedHttpClient(Credentials creds, HttpClient delegate) + { + this.creds = creds; + this.delegate = delegate; + } + + @Override + public ListenableFuture go( + Request request, + HttpResponseHandler handler, + Duration requestReadTimeout + ) + { + return delegate.go(creds.addCredentials(request), handler, requestReadTimeout); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/EnforceSslHttpClient.java b/java-util/src/main/java/io/druid/java/util/http/client/EnforceSslHttpClient.java new file mode 100644 index 000000000000..c2e41d1c270a --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/EnforceSslHttpClient.java @@ -0,0 +1,58 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client; + +import com.google.common.util.concurrent.ListenableFuture; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.http.client.response.HttpResponseHandler; +import org.joda.time.Duration; + +import java.net.URL; + +/** + */ +public class EnforceSslHttpClient extends AbstractHttpClient +{ + + private final HttpClient delegate; + + public EnforceSslHttpClient( + HttpClient delegate + ) + { + this.delegate = delegate; + } + + @Override + public ListenableFuture go( + Request request, + HttpResponseHandler handler, + Duration requestReadTimeout + ) + { + URL url = request.getUrl(); + + if (!"https".equals(url.getProtocol())) { + throw new IllegalArgumentException(StringUtils.format("Requests must be over https, got[%s].", url)); + } + + return delegate.go(request, handler, requestReadTimeout); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/HttpClient.java b/java-util/src/main/java/io/druid/java/util/http/client/HttpClient.java new file mode 100644 index 000000000000..5950b863ed5e --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/HttpClient.java @@ -0,0 +1,71 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client; + +import com.google.common.util.concurrent.ListenableFuture; +import io.druid.java.util.http.client.response.HttpResponseHandler; +import org.joda.time.Duration; + +/** + * Interface for Async HTTP client libraries. + */ +public interface HttpClient +{ + /** + * Submit a request and process the response with the given response handler. + *

+ * Note that the Request object passed in to the HttpClient *may* be mutated by the actual client. This is + * largely done by composed clients, but the contract is that mutation is possible. It is the caller's + * responsibility to pass in a copy of the Request object if they want to have an object that is not mutated. + * + * @param request Request to process, this *may* be mutated by the client + * @param handler An asynchronous response handler that will be used to process results from the http call + * @param The type of the intermediate results from the handler + * @param The type of the final results that the returned ListenableFuture will contain + * + * @return A listenable future that will eventually provide an object of type Final + */ + ListenableFuture go( + Request request, + HttpResponseHandler handler + ); + + /** + * Submit a request and process the response with the given response handler. + *

+ * Note that the Request object passed in to the HttpClient *may* be mutated by the actual client. This is + * largely done by composed clients, but the contract is that mutation is possible. It is the caller's + * responsibility to pass in a copy of the Request object if they want to have an object that is not mutated. + * + * @param request Request to process, this *may* be mutated by the client + * @param handler An asynchronous response handler that will be used to process results from the http call + * @param readTimeout Read timeout to use for this request. Leave null to use the default readTimeout. Set to zero + * to disable timeouts for this request. + * @param The type of the intermediate results from the handler + * @param The type of the final results that the returned ListenableFuture will contain + * + * @return A listenable future that will eventually provide an object of type Final + */ + ListenableFuture go( + Request request, + HttpResponseHandler handler, + Duration readTimeout + ); +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/HttpClientConfig.java b/java-util/src/main/java/io/druid/java/util/http/client/HttpClientConfig.java new file mode 100644 index 000000000000..54e458052720 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/HttpClientConfig.java @@ -0,0 +1,289 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client; + +import org.joda.time.Duration; +import org.joda.time.Period; + +import javax.net.ssl.SSLContext; + +/** + */ +public class HttpClientConfig +{ + public enum CompressionCodec + { + IDENTITY { + @Override + public String getEncodingString() + { + return "identity"; + } + }, + GZIP { + @Override + public String getEncodingString() + { + return "gzip"; + } + }, + DEFLATE { + @Override + public String getEncodingString() + { + return "deflate"; + } + }; + + /** + * Get the header-ified name of this encoding, which should go in "Accept-Encoding" and + * "Content-Encoding" headers. This is not just the lowercasing of the enum name, since + * we may one day support x- encodings like LZ4, which would likely be an enum named + * "LZ4" that has an encoding string like "x-lz4". + * + * @return encoding name + */ + public abstract String getEncodingString(); + } + + public static final CompressionCodec DEFAULT_COMPRESSION_CODEC = CompressionCodec.GZIP; + + // Default from NioClientSocketChannelFactory.DEFAULT_BOSS_COUNT, which is private: + private static final int DEFAULT_BOSS_COUNT = 1; + + // Default from SelectorUtil.DEFAULT_IO_THREADS, which is private: + private static final int DEFAULT_WORKER_COUNT = Runtime.getRuntime().availableProcessors() * 2; + + private static final Duration DEFAULT_UNUSED_CONNECTION_TIMEOUT_DURATION = new Period("PT4M").toStandardDuration(); + + public static Builder builder() + { + return new Builder(); + } + + private final int numConnections; + private final SSLContext sslContext; + private final Duration readTimeout; + private final Duration sslHandshakeTimeout; + private final int bossPoolSize; + private final int workerPoolSize; + private final CompressionCodec compressionCodec; + private final Duration unusedConnectionTimeoutDuration; + + @Deprecated // Use the builder instead + public HttpClientConfig( + int numConnections, + SSLContext sslContext + ) + { + this( + numConnections, + sslContext, + Duration.ZERO, + null, + DEFAULT_BOSS_COUNT, + DEFAULT_WORKER_COUNT, + DEFAULT_COMPRESSION_CODEC, + DEFAULT_UNUSED_CONNECTION_TIMEOUT_DURATION + ); + } + + @Deprecated // Use the builder instead + public HttpClientConfig( + int numConnections, + SSLContext sslContext, + Duration readTimeout + ) + { + this( + numConnections, + sslContext, + readTimeout, + null, + DEFAULT_BOSS_COUNT, + DEFAULT_WORKER_COUNT, + DEFAULT_COMPRESSION_CODEC, + DEFAULT_UNUSED_CONNECTION_TIMEOUT_DURATION + ); + } + + @Deprecated // Use the builder instead + public HttpClientConfig( + int numConnections, + SSLContext sslContext, + Duration readTimeout, + Duration sslHandshakeTimeout + ) + { + this( + numConnections, + sslContext, + readTimeout, + sslHandshakeTimeout, + DEFAULT_BOSS_COUNT, + DEFAULT_WORKER_COUNT, + DEFAULT_COMPRESSION_CODEC, + DEFAULT_UNUSED_CONNECTION_TIMEOUT_DURATION + ); + } + + private HttpClientConfig( + int numConnections, + SSLContext sslContext, + Duration readTimeout, + Duration sslHandshakeTimeout, + int bossPoolSize, + int workerPoolSize, + CompressionCodec compressionCodec, + Duration unusedConnectionTimeoutDuration + ) + { + this.numConnections = numConnections; + this.sslContext = sslContext; + this.readTimeout = readTimeout; + this.sslHandshakeTimeout = sslHandshakeTimeout; + this.bossPoolSize = bossPoolSize; + this.workerPoolSize = workerPoolSize; + this.compressionCodec = compressionCodec; + this.unusedConnectionTimeoutDuration = unusedConnectionTimeoutDuration; + } + + public int getNumConnections() + { + return numConnections; + } + + public SSLContext getSslContext() + { + return sslContext; + } + + public Duration getReadTimeout() + { + return readTimeout; + } + + public Duration getSslHandshakeTimeout() + { + return sslHandshakeTimeout; + } + + public int getBossPoolSize() + { + return bossPoolSize; + } + + public int getWorkerPoolSize() + { + return workerPoolSize; + } + + public CompressionCodec getCompressionCodec() + { + return compressionCodec; + } + + public Duration getUnusedConnectionTimeoutDuration() + { + return unusedConnectionTimeoutDuration; + } + + public static class Builder + { + private int numConnections = 1; + private SSLContext sslContext = null; + private Duration readTimeout = null; + private Duration sslHandshakeTimeout = null; + private int bossCount = DEFAULT_BOSS_COUNT; + private int workerCount = DEFAULT_WORKER_COUNT; + private CompressionCodec compressionCodec = DEFAULT_COMPRESSION_CODEC; + private Duration unusedConnectionTimeoutDuration = DEFAULT_UNUSED_CONNECTION_TIMEOUT_DURATION; + + private Builder() {} + + public Builder withNumConnections(int numConnections) + { + this.numConnections = numConnections; + return this; + } + + public Builder withSslContext(SSLContext sslContext) + { + this.sslContext = sslContext; + return this; + } + + public Builder withSslContext(String keyStorePath, String keyStorePassword) + { + this.sslContext = HttpClientInit.sslContextWithTrustedKeyStore(keyStorePath, keyStorePassword); + return this; + } + + public Builder withReadTimeout(Duration readTimeout) + { + this.readTimeout = readTimeout; + return this; + } + + public Builder withSslHandshakeTimeout(Duration sslHandshakeTimeout) + { + this.sslHandshakeTimeout = sslHandshakeTimeout; + return this; + } + + public Builder withBossCount(int bossCount) + { + this.bossCount = bossCount; + return this; + } + + public Builder withWorkerCount(int workerCount) + { + this.workerCount = workerCount; + return this; + } + + public Builder withCompressionCodec(CompressionCodec compressionCodec) + { + this.compressionCodec = compressionCodec; + return this; + } + + public Builder withUnusedConnectionTimeoutDuration(Duration unusedConnectionTimeoutDuration) + { + this.unusedConnectionTimeoutDuration = unusedConnectionTimeoutDuration; + return this; + } + + public HttpClientConfig build() + { + return new HttpClientConfig( + numConnections, + sslContext, + readTimeout, + sslHandshakeTimeout, + bossCount, + workerCount, + compressionCodec, + unusedConnectionTimeoutDuration + ); + } + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/HttpClientInit.java b/java-util/src/main/java/io/druid/java/util/http/client/HttpClientInit.java new file mode 100644 index 000000000000..1141829a9606 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/HttpClientInit.java @@ -0,0 +1,231 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client; + +import com.google.common.base.Throwables; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.druid.java.util.common.guava.CloseQuietly; +import io.druid.java.util.common.lifecycle.Lifecycle; +import io.druid.java.util.http.client.netty.HttpClientPipelineFactory; +import io.druid.java.util.http.client.pool.ChannelResourceFactory; +import io.druid.java.util.http.client.pool.ResourcePool; +import io.druid.java.util.http.client.pool.ResourcePoolConfig; +import org.jboss.netty.bootstrap.ClientBootstrap; +import org.jboss.netty.channel.socket.nio.NioClientBossPool; +import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory; +import org.jboss.netty.channel.socket.nio.NioWorkerPool; +import org.jboss.netty.logging.InternalLoggerFactory; +import org.jboss.netty.logging.Slf4JLoggerFactory; +import org.jboss.netty.util.HashedWheelTimer; +import org.jboss.netty.util.ThreadNameDeterminer; +import org.jboss.netty.util.Timer; +import org.joda.time.Duration; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +/** + */ +public class HttpClientInit +{ + public static HttpClient createClient(HttpClientConfig config, Lifecycle lifecycle) + { + try { + // We need to use the full constructor in order to set a ThreadNameDeterminer. The other parameters are taken + // from the defaults in HashedWheelTimer's other constructors. + final HashedWheelTimer timer = new HashedWheelTimer( + new ThreadFactoryBuilder().setDaemon(true) + .setNameFormat("HttpClient-Timer-%s") + .build(), + ThreadNameDeterminer.CURRENT, + 100, + TimeUnit.MILLISECONDS, + 512 + ); + lifecycle.addMaybeStartHandler( + new Lifecycle.Handler() + { + @Override + public void start() throws Exception + { + timer.start(); + } + + @Override + public void stop() + { + timer.stop(); + } + } + ); + return lifecycle.addMaybeStartManagedInstance( + new NettyHttpClient( + new ResourcePool<>( + new ChannelResourceFactory( + createBootstrap(lifecycle, timer, config.getBossPoolSize(), config.getWorkerPoolSize()), + config.getSslContext(), + timer, + config.getSslHandshakeTimeout() == null ? -1 : config.getSslHandshakeTimeout().getMillis() + ), + new ResourcePoolConfig( + config.getNumConnections(), + config.getUnusedConnectionTimeoutDuration().getMillis() + ) + ), + config.getReadTimeout(), + config.getCompressionCodec(), + timer + ) + ); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + + @Deprecated + public static HttpClient createClient(ResourcePoolConfig config, final SSLContext sslContext, Lifecycle lifecycle) + { + return createClient( + new HttpClientConfig(config.getMaxPerKey(), sslContext, Duration.ZERO), + lifecycle + ); + } + + @Deprecated // use createClient directly + public static ClientBootstrap createBootstrap(Lifecycle lifecycle, Timer timer) + { + final HttpClientConfig defaultConfig = HttpClientConfig.builder().build(); + return createBootstrap(lifecycle, timer, defaultConfig.getBossPoolSize(), defaultConfig.getWorkerPoolSize()); + } + + @Deprecated // use createClient directly + public static ClientBootstrap createBootstrap(Lifecycle lifecycle) + { + final Timer timer = new HashedWheelTimer(new ThreadFactoryBuilder().setDaemon(true).build()); + return createBootstrap(lifecycle, timer); + } + + public static SSLContext sslContextWithTrustedKeyStore(final String keyStorePath, final String keyStorePassword) + { + FileInputStream in = null; + try { + final KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType()); + + in = new FileInputStream(keyStorePath); + ks.load(in, keyStorePassword.toCharArray()); + in.close(); + + final TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + tmf.init(ks); + final SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, tmf.getTrustManagers(), null); + + return sslContext; + } + catch (CertificateException e) { + throw Throwables.propagate(e); + } + catch (NoSuchAlgorithmException e) { + throw Throwables.propagate(e); + } + catch (KeyStoreException e) { + throw Throwables.propagate(e); + } + catch (KeyManagementException e) { + throw Throwables.propagate(e); + } + catch (FileNotFoundException e) { + throw Throwables.propagate(e); + } + catch (IOException e) { + throw Throwables.propagate(e); + } + finally { + CloseQuietly.close(in); + } + } + + private static ClientBootstrap createBootstrap(Lifecycle lifecycle, Timer timer, int bossPoolSize, int workerPoolSize) + { + final NioClientBossPool bossPool = new NioClientBossPool( + Executors.newCachedThreadPool( + new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat("HttpClient-Netty-Boss-%s") + .build() + ), + bossPoolSize, + timer, + ThreadNameDeterminer.CURRENT + ); + + final NioWorkerPool workerPool = new NioWorkerPool( + Executors.newCachedThreadPool( + new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat("HttpClient-Netty-Worker-%s") + .build() + ), + workerPoolSize, + ThreadNameDeterminer.CURRENT + ); + + final ClientBootstrap bootstrap = new ClientBootstrap(new NioClientSocketChannelFactory(bossPool, workerPool)); + + bootstrap.setOption("keepAlive", true); + bootstrap.setPipelineFactory(new HttpClientPipelineFactory()); + + InternalLoggerFactory.setDefaultFactory(new Slf4JLoggerFactory()); + + try { + lifecycle.addMaybeStartHandler( + new Lifecycle.Handler() + { + @Override + public void start() throws Exception + { + } + + @Override + public void stop() + { + bootstrap.releaseExternalResources(); + } + } + ); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + + return bootstrap; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/NettyHttpClient.java b/java-util/src/main/java/io/druid/java/util/http/client/NettyHttpClient.java new file mode 100644 index 000000000000..0d343a06f1ab --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/NettyHttpClient.java @@ -0,0 +1,405 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Multimap; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import io.druid.java.util.common.IAE; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.lifecycle.LifecycleStart; +import io.druid.java.util.common.lifecycle.LifecycleStop; +import io.druid.java.util.common.logger.Logger; +import io.druid.java.util.http.client.pool.ResourceContainer; +import io.druid.java.util.http.client.pool.ResourcePool; +import io.druid.java.util.http.client.response.ClientResponse; +import io.druid.java.util.http.client.response.HttpResponseHandler; +import org.jboss.netty.channel.Channel; +import org.jboss.netty.channel.ChannelException; +import org.jboss.netty.channel.ChannelFuture; +import org.jboss.netty.channel.ChannelFutureListener; +import org.jboss.netty.channel.ChannelHandlerContext; +import org.jboss.netty.channel.ChannelStateEvent; +import org.jboss.netty.channel.ExceptionEvent; +import org.jboss.netty.channel.MessageEvent; +import org.jboss.netty.channel.SimpleChannelUpstreamHandler; +import org.jboss.netty.handler.codec.http.DefaultHttpRequest; +import org.jboss.netty.handler.codec.http.HttpChunk; +import org.jboss.netty.handler.codec.http.HttpHeaders; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpRequest; +import org.jboss.netty.handler.codec.http.HttpResponse; +import org.jboss.netty.handler.codec.http.HttpVersion; +import org.jboss.netty.handler.timeout.ReadTimeoutHandler; +import org.jboss.netty.util.Timer; +import org.joda.time.Duration; + +import java.net.URL; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + */ +public class NettyHttpClient extends AbstractHttpClient +{ + private static final Logger log = new Logger(NettyHttpClient.class); + + private static final String READ_TIMEOUT_HANDLER_NAME = "read-timeout"; + private static final String LAST_HANDLER_NAME = "last-handler"; + + private final Timer timer; + private final ResourcePool pool; + private final HttpClientConfig.CompressionCodec compressionCodec; + private final Duration defaultReadTimeout; + + public NettyHttpClient( + ResourcePool pool + ) + { + this(pool, null, HttpClientConfig.DEFAULT_COMPRESSION_CODEC, null); + } + + NettyHttpClient( + ResourcePool pool, + Duration defaultReadTimeout, + HttpClientConfig.CompressionCodec compressionCodec, + Timer timer + ) + { + this.pool = Preconditions.checkNotNull(pool, "pool"); + this.defaultReadTimeout = defaultReadTimeout; + this.compressionCodec = Preconditions.checkNotNull(compressionCodec); + this.timer = timer; + + if (defaultReadTimeout != null && defaultReadTimeout.getMillis() > 0) { + Preconditions.checkNotNull(timer, "timer"); + } + } + + @LifecycleStart + public void start() + { + } + + @LifecycleStop + public void stop() + { + pool.close(); + } + + public HttpClient withReadTimeout(Duration readTimeout) + { + return new NettyHttpClient(pool, readTimeout, compressionCodec, timer); + } + + public NettyHttpClient withTimer(Timer timer) + { + return new NettyHttpClient(pool, defaultReadTimeout, compressionCodec, timer); + } + + @Override + public ListenableFuture go( + final Request request, + final HttpResponseHandler handler, + final Duration requestReadTimeout + ) + { + final HttpMethod method = request.getMethod(); + final URL url = request.getUrl(); + final Multimap headers = request.getHeaders(); + + final String requestDesc = StringUtils.format("%s %s", method, url); + if (log.isDebugEnabled()) { + log.debug("[%s] starting", requestDesc); + } + + // Block while acquiring a channel from the pool, then complete the request asynchronously. + final Channel channel; + final String hostKey = getPoolKey(url); + final ResourceContainer channelResourceContainer = pool.take(hostKey); + final ChannelFuture channelFuture = channelResourceContainer.get().awaitUninterruptibly(); + if (!channelFuture.isSuccess()) { + channelResourceContainer.returnResource(); // Some other poor sap will have to deal with it... + return Futures.immediateFailedFuture( + new ChannelException( + "Faulty channel in resource pool", + channelFuture.getCause() + ) + ); + } else { + channel = channelFuture.getChannel(); + } + + final String urlFile = Strings.nullToEmpty(url.getFile()); + final HttpRequest httpRequest = new DefaultHttpRequest( + HttpVersion.HTTP_1_1, + method, + urlFile.isEmpty() ? "/" : urlFile + ); + + if (!headers.containsKey(HttpHeaders.Names.HOST)) { + httpRequest.headers().add(HttpHeaders.Names.HOST, getHost(url)); + } + + // If Accept-Encoding is set in the Request, use that. Otherwise use the default from "compressionCodec". + if (!headers.containsKey(HttpHeaders.Names.ACCEPT_ENCODING)) { + httpRequest.headers().set(HttpHeaders.Names.ACCEPT_ENCODING, compressionCodec.getEncodingString()); + } + + for (Map.Entry> entry : headers.asMap().entrySet()) { + String key = entry.getKey(); + + for (String obj : entry.getValue()) { + httpRequest.headers().add(key, obj); + } + } + + if (request.hasContent()) { + httpRequest.setContent(request.getContent()); + } + + final long readTimeout = getReadTimeout(requestReadTimeout); + final SettableFuture retVal = SettableFuture.create(); + + if (readTimeout > 0) { + channel.getPipeline().addLast( + READ_TIMEOUT_HANDLER_NAME, + new ReadTimeoutHandler(timer, readTimeout, TimeUnit.MILLISECONDS) + ); + } + + channel.getPipeline().addLast( + LAST_HANDLER_NAME, + new SimpleChannelUpstreamHandler() + { + private volatile ClientResponse response = null; + + @Override + public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception + { + if (log.isDebugEnabled()) { + log.debug("[%s] messageReceived: %s", requestDesc, e.getMessage()); + } + try { + Object msg = e.getMessage(); + + if (msg instanceof HttpResponse) { + HttpResponse httpResponse = (HttpResponse) msg; + if (log.isDebugEnabled()) { + log.debug("[%s] Got response: %s", requestDesc, httpResponse.getStatus()); + } + + response = handler.handleResponse(httpResponse); + if (response.isFinished()) { + retVal.set((Final) response.getObj()); + } + + if (!httpResponse.isChunked()) { + finishRequest(); + } + } else if (msg instanceof HttpChunk) { + HttpChunk httpChunk = (HttpChunk) msg; + if (log.isDebugEnabled()) { + log.debug( + "[%s] Got chunk: %sB, last=%s", + requestDesc, + httpChunk.getContent().readableBytes(), + httpChunk.isLast() + ); + } + + if (httpChunk.isLast()) { + finishRequest(); + } else { + response = handler.handleChunk(response, httpChunk); + if (response.isFinished() && !retVal.isDone()) { + retVal.set((Final) response.getObj()); + } + } + } else { + throw new IllegalStateException(StringUtils.format("Unknown message type[%s]", msg.getClass())); + } + } + catch (Exception ex) { + log.warn(ex, "[%s] Exception thrown while processing message, closing channel.", requestDesc); + + if (!retVal.isDone()) { + retVal.set(null); + } + channel.close(); + channelResourceContainer.returnResource(); + + throw ex; + } + } + + private void finishRequest() + { + ClientResponse finalResponse = handler.done(response); + if (!finalResponse.isFinished()) { + throw new IllegalStateException( + StringUtils.format( + "[%s] Didn't get a completed ClientResponse Object from [%s]", + requestDesc, + handler.getClass() + ) + ); + } + if (!retVal.isDone()) { + retVal.set(finalResponse.getObj()); + } + removeHandlers(); + channelResourceContainer.returnResource(); + } + + @Override + public void exceptionCaught(ChannelHandlerContext context, ExceptionEvent event) throws Exception + { + if (log.isDebugEnabled()) { + final Throwable cause = event.getCause(); + if (cause == null) { + log.debug("[%s] Caught exception", requestDesc); + } else { + log.debug(cause, "[%s] Caught exception", requestDesc); + } + } + + retVal.setException(event.getCause()); + // response is non-null if we received initial chunk and then exception occurs + if (response != null) { + handler.exceptionCaught(response, event.getCause()); + } + removeHandlers(); + try { + channel.close(); + } + catch (Exception e) { + // ignore + } + finally { + channelResourceContainer.returnResource(); + } + + context.sendUpstream(event); + } + + @Override + public void channelDisconnected(ChannelHandlerContext context, ChannelStateEvent event) throws Exception + { + if (log.isDebugEnabled()) { + log.debug("[%s] Channel disconnected", requestDesc); + } + // response is non-null if we received initial chunk and then exception occurs + if (response != null) { + handler.exceptionCaught(response, new ChannelException("Channel disconnected")); + } + channel.close(); + channelResourceContainer.returnResource(); + if (!retVal.isDone()) { + log.warn("[%s] Channel disconnected before response complete", requestDesc); + retVal.setException(new ChannelException("Channel disconnected")); + } + context.sendUpstream(event); + } + + private void removeHandlers() + { + if (readTimeout > 0) { + channel.getPipeline().remove(READ_TIMEOUT_HANDLER_NAME); + } + channel.getPipeline().remove(LAST_HANDLER_NAME); + } + } + ); + + channel.write(httpRequest).addListener( + new ChannelFutureListener() + { + @Override + public void operationComplete(ChannelFuture future) throws Exception + { + if (!future.isSuccess()) { + channel.close(); + channelResourceContainer.returnResource(); + if (!retVal.isDone()) { + retVal.setException( + new ChannelException( + StringUtils.format("[%s] Failed to write request to channel", requestDesc), + future.getCause() + ) + ); + } + } + } + } + ); + + return retVal; + } + + private long getReadTimeout(Duration requestReadTimeout) + { + final long timeout; + if (requestReadTimeout != null) { + timeout = requestReadTimeout.getMillis(); + } else if (defaultReadTimeout != null) { + timeout = defaultReadTimeout.getMillis(); + } else { + timeout = 0; + } + + if (timeout > 0 && timer == null) { + log.warn("Cannot time out requests without a timer! Disabling timeout for this request."); + return 0; + } else { + return timeout; + } + } + + private String getHost(URL url) + { + int port = url.getPort(); + + if (port == -1) { + final String protocol = url.getProtocol(); + + if ("http".equalsIgnoreCase(protocol)) { + port = 80; + } else if ("https".equalsIgnoreCase(protocol)) { + port = 443; + } else { + throw new IAE("Cannot figure out default port for protocol[%s], please set Host header.", protocol); + } + } + + return StringUtils.format("%s:%s", url.getHost(), port); + } + + private String getPoolKey(URL url) + { + return StringUtils.format( + "%s://%s:%s", url.getProtocol(), url.getHost(), url.getPort() == -1 ? url.getDefaultPort() : url.getPort() + ); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/Request.java b/java-util/src/main/java/io/druid/java/util/http/client/Request.java new file mode 100644 index 000000000000..f49764101a8c --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/Request.java @@ -0,0 +1,200 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client; + +import com.google.common.base.Charsets; +import com.google.common.base.Supplier; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; +import io.druid.java.util.common.StringUtils; +import org.jboss.netty.buffer.ChannelBuffer; +import org.jboss.netty.buffer.ChannelBufferFactory; +import org.jboss.netty.buffer.HeapChannelBufferFactory; +import org.jboss.netty.handler.codec.base64.Base64; +import org.jboss.netty.handler.codec.http.HttpHeaders; +import org.jboss.netty.handler.codec.http.HttpMethod; + +import java.net.URL; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + */ +public class Request +{ + private static final ChannelBufferFactory factory = HeapChannelBufferFactory.getInstance(); + + private final HttpMethod method; + private final URL url; + private final Multimap headers = Multimaps.newListMultimap( + Maps.>newHashMap(), + new Supplier>() + { + @Override + public List get() + { + return Lists.newArrayList(); + } + } + ); + + private ChannelBuffer content; + + public Request( + HttpMethod method, + URL url + ) + { + this.method = method; + this.url = url; + } + + public HttpMethod getMethod() + { + return method; + } + + public URL getUrl() + { + return url; + } + + public Multimap getHeaders() + { + return headers; + } + + public boolean hasContent() + { + return content != null; + } + + public ChannelBuffer getContent() + { + return content; + } + + public Request copy() + { + Request retVal = new Request(method, url); + retVal.headers.putAll(this.headers); + retVal.content = content == null ? null : content.copy(); + return retVal; + } + + public Request setHeader(String header, String value) + { + headers.replaceValues(header, Arrays.asList(value)); + return this; + } + + public Request setHeaderValues(String header, Iterable value) + { + headers.replaceValues(header, value); + return this; + } + + public Request setHeaderValues(Multimap inHeaders) + { + for (Map.Entry> entry : inHeaders.asMap().entrySet()) { + this.setHeaderValues(entry.getKey(), entry.getValue()); + } + return this; + } + + public Request addHeader(String header, String value) + { + headers.put(header, value); + return this; + } + + public Request addHeaderValues(String header, Iterable value) + { + headers.putAll(header, value); + return this; + } + + public Request addHeaderValues(Multimap inHeaders) + { + for (Map.Entry> entry : inHeaders.asMap().entrySet()) { + this.addHeaderValues(entry.getKey(), entry.getValue()); + } + return this; + } + + public Request setContent(byte[] bytes) + { + return setContent(null, bytes); + } + + public Request setContent(byte[] bytes, int offset, int length) + { + return setContent(null, bytes, offset, length); + } + + public Request setContent(ChannelBuffer content) + { + return setContent(null, content); + } + + public Request setContent(String contentType, byte[] bytes) + { + return setContent(contentType, bytes, 0, bytes.length); + } + + public Request setContent(String contentType, byte[] bytes, int offset, int length) + { + return setContent(contentType, factory.getBuffer(bytes, offset, length)); + } + + public Request setContent(String contentType, ChannelBuffer content) + { + if (contentType != null) { + setHeader(HttpHeaders.Names.CONTENT_TYPE, contentType); + } + + this.content = content; + + setHeader(HttpHeaders.Names.CONTENT_LENGTH, String.valueOf(content.writerIndex())); + + return this; + } + + public Request setBasicAuthentication(String username, String password) + { + final String base64Value = base64Encode(StringUtils.format("%s:%s", username, password)); + setHeader(HttpHeaders.Names.AUTHORIZATION, StringUtils.format("Basic %s", base64Value)); + return this; + } + + private String base64Encode(final String value) + { + final ChannelBufferFactory bufferFactory = HeapChannelBufferFactory.getInstance(); + + return Base64 + .encode(bufferFactory.getBuffer(ByteBuffer.wrap(value.getBytes(Charsets.UTF_8))), false) + .toString(Charsets.UTF_8); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/auth/BasicCredentials.java b/java-util/src/main/java/io/druid/java/util/http/client/auth/BasicCredentials.java new file mode 100644 index 000000000000..29cc85f5f385 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/auth/BasicCredentials.java @@ -0,0 +1,45 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.auth; + +import io.druid.java.util.http.client.Request; + +/** + */ +public class BasicCredentials implements Credentials +{ + private final String username; + private final String password; + + public BasicCredentials( + String username, + String password + ) + { + this.username = username; + this.password = password; + } + + @Override + public Request addCredentials(Request builder) + { + return builder.setBasicAuthentication(username, password); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/auth/Credentials.java b/java-util/src/main/java/io/druid/java/util/http/client/auth/Credentials.java new file mode 100644 index 000000000000..35e204616cf5 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/auth/Credentials.java @@ -0,0 +1,29 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.auth; + +import io.druid.java.util.http.client.Request; + +/** + */ +public interface Credentials +{ + Request addCredentials(Request builder); +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/io/AppendableByteArrayInputStream.java b/java-util/src/main/java/io/druid/java/util/http/client/io/AppendableByteArrayInputStream.java new file mode 100644 index 000000000000..8c3c336cd568 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/io/AppendableByteArrayInputStream.java @@ -0,0 +1,211 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.io; + +import io.druid.java.util.common.logger.Logger; + +import java.io.IOException; +import java.io.InputStream; +import java.util.LinkedList; + +/** + */ +public class AppendableByteArrayInputStream extends InputStream +{ + private static final Logger log = new Logger(AppendableByteArrayInputStream.class); + + private final LinkedList bytes = new LinkedList(); + private final SingleByteReaderDoer singleByteReaderDoer = new SingleByteReaderDoer(); + + private volatile boolean done = false; + private volatile Throwable throwable; + private volatile int available = 0; + + private byte[] curr = new byte[]{}; + private int currIndex = 0; + + public void add(byte[] bytesToAdd) + { + if (bytesToAdd.length == 0) { + return; + } + + synchronized (singleByteReaderDoer) { + bytes.addLast(bytesToAdd); + available += bytesToAdd.length; + singleByteReaderDoer.notify(); + } + } + + public void done() + { + synchronized (singleByteReaderDoer) { + done = true; + singleByteReaderDoer.notify(); + } + } + + public void exceptionCaught(Throwable t) + { + synchronized (singleByteReaderDoer) { + done = true; + throwable = t; + singleByteReaderDoer.notify(); + } + } + + @Override + public int read() throws IOException + { + if (scanThroughBytesAndDoSomething(1, singleByteReaderDoer) == 0) { + return -1; + } + return singleByteReaderDoer.getRetVal(); + } + + @Override + public int read(final byte[] b, final int off, int len) throws IOException + { + if (b == null) { + throw new NullPointerException(); + } else if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException(); + } else if (len == 0) { + return 0; + } + + final long retVal = scanThroughBytesAndDoSomething( + len, + new Doer() + { + int currOff = off; + + @Override + public void doSomethingWithByteArray(int numRead) + { + System.arraycopy(curr, currIndex, b, currOff, numRead); + currOff += numRead; + } + } + ); + return retVal == 0 ? -1 : (int) retVal; + } + + @Override + public long skip(long n) throws IOException + { + return scanThroughBytesAndDoSomething( + n, + new Doer() + { + @Override + public void doSomethingWithByteArray(int numToScan) + { + } + } + ); + } + + private long scanThroughBytesAndDoSomething(long numToScan, Doer doer) throws IOException + { + long numScanned = 0; + long numPulled = 0; + + while (numToScan > numScanned) { + if (currIndex >= curr.length) { + synchronized (singleByteReaderDoer) { + if (bytes.isEmpty()) { + if (done) { + break; + } + try { + available -= numPulled; + numPulled = 0; + singleByteReaderDoer.wait(); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted!"); + } + } + + if (throwable != null) { + throw new IOException(throwable); + } + + if (bytes.isEmpty()) { + if (done) { + break; + } else { + log.debug("bytes was empty, but read thread was awakened without being done. This shouldn't happen."); + continue; + } + } + + curr = bytes.removeFirst(); + currIndex = 0; + } + } + + final long numToPullFromCurr = Math.min(curr.length - currIndex, numToScan - numScanned); + doer.doSomethingWithByteArray((int) numToPullFromCurr); + numScanned += numToPullFromCurr; + currIndex += numToPullFromCurr; + numPulled += numToPullFromCurr; + } + + synchronized (singleByteReaderDoer) { + available -= numPulled; + } + + return numScanned; + } + + @Override + public int available() throws IOException + { + return available; + } + + private interface Doer + { + void doSomethingWithByteArray(int numToScan); + } + + private class SingleByteReaderDoer implements Doer + { + private int retVal; + + public SingleByteReaderDoer() + { + } + + @Override + public void doSomethingWithByteArray(int numToScan) + { + retVal = curr[currIndex]; + } + + public int getRetVal() + { + return retVal; + } + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/netty/HttpClientPipelineFactory.java b/java-util/src/main/java/io/druid/java/util/http/client/netty/HttpClientPipelineFactory.java new file mode 100644 index 000000000000..eddda1a53a82 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/netty/HttpClientPipelineFactory.java @@ -0,0 +1,42 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.netty; + +import org.jboss.netty.channel.ChannelPipeline; +import org.jboss.netty.channel.ChannelPipelineFactory; +import org.jboss.netty.channel.DefaultChannelPipeline; +import org.jboss.netty.handler.codec.http.HttpClientCodec; +import org.jboss.netty.handler.codec.http.HttpContentDecompressor; + +/** + */ +public class HttpClientPipelineFactory implements ChannelPipelineFactory +{ + @Override + public ChannelPipeline getPipeline() throws Exception + { + ChannelPipeline pipeline = new DefaultChannelPipeline(); + + pipeline.addLast("codec", new HttpClientCodec()); + pipeline.addLast("inflater", new HttpContentDecompressor()); + + return pipeline; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/pool/ChannelResourceFactory.java b/java-util/src/main/java/io/druid/java/util/http/client/pool/ChannelResourceFactory.java new file mode 100644 index 000000000000..91dbae44f7d9 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/pool/ChannelResourceFactory.java @@ -0,0 +1,182 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.pool; + +import com.google.common.base.Preconditions; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.logger.Logger; +import org.jboss.netty.bootstrap.ClientBootstrap; +import org.jboss.netty.channel.Channel; +import org.jboss.netty.channel.ChannelException; +import org.jboss.netty.channel.ChannelFuture; +import org.jboss.netty.channel.ChannelFutureListener; +import org.jboss.netty.channel.ChannelPipeline; +import org.jboss.netty.channel.Channels; +import org.jboss.netty.handler.ssl.SslHandler; +import org.jboss.netty.util.Timer; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLParameters; +import java.net.InetSocketAddress; +import java.net.MalformedURLException; +import java.net.URL; + +/** + */ +public class ChannelResourceFactory implements ResourceFactory +{ + private static final Logger log = new Logger(ChannelResourceFactory.class); + + private static final long DEFAULT_SSL_HANDSHAKE_TIMEOUT = 10000L; /* 10 seconds */ + + private final ClientBootstrap bootstrap; + private final SSLContext sslContext; + private final Timer timer; + private final long sslHandshakeTimeout; + + public ChannelResourceFactory( + ClientBootstrap bootstrap, + SSLContext sslContext, + Timer timer, + long sslHandshakeTimeout + ) + { + this.bootstrap = Preconditions.checkNotNull(bootstrap, "bootstrap"); + this.sslContext = sslContext; + this.timer = timer; + this.sslHandshakeTimeout = sslHandshakeTimeout >= 0 ? sslHandshakeTimeout : DEFAULT_SSL_HANDSHAKE_TIMEOUT; + + if (sslContext != null) { + Preconditions.checkNotNull(timer, "timer is required when sslContext is present"); + } + } + + @Override + public ChannelFuture generate(final String hostname) + { + log.info("Generating: %s", hostname); + URL url = null; + try { + url = new URL(hostname); + } + catch (MalformedURLException e) { + throw new RuntimeException(e); + } + + final String host = url.getHost(); + final int port = url.getPort() == -1 ? url.getDefaultPort() : url.getPort(); + final ChannelFuture retVal; + final ChannelFuture connectFuture = bootstrap.connect(new InetSocketAddress(host, port)); + + if ("https".equals(url.getProtocol())) { + if (sslContext == null) { + throw new IllegalStateException("No sslContext set, cannot do https"); + } + + final SSLEngine sslEngine = sslContext.createSSLEngine(host, port); + final SSLParameters sslParameters = new SSLParameters(); + sslParameters.setEndpointIdentificationAlgorithm("HTTPS"); + sslEngine.setSSLParameters(sslParameters); + sslEngine.setUseClientMode(true); + final SslHandler sslHandler = new SslHandler( + sslEngine, + SslHandler.getDefaultBufferPool(), + false, + timer, + sslHandshakeTimeout + ); + + // https://github.com/netty/netty/issues/160 + sslHandler.setCloseOnSSLException(true); + + final ChannelPipeline pipeline = connectFuture.getChannel().getPipeline(); + pipeline.addFirst("ssl", sslHandler); + + final ChannelFuture handshakeFuture = Channels.future(connectFuture.getChannel()); + connectFuture.addListener( + new ChannelFutureListener() + { + @Override + public void operationComplete(ChannelFuture f) throws Exception + { + if (f.isSuccess()) { + sslHandler.handshake().addListener( + new ChannelFutureListener() + { + @Override + public void operationComplete(ChannelFuture f2) throws Exception + { + if (f2.isSuccess()) { + handshakeFuture.setSuccess(); + } else { + handshakeFuture.setFailure( + new ChannelException( + StringUtils.format("Failed to handshake with host[%s]", hostname), + f2.getCause() + ) + ); + } + } + } + ); + } else { + handshakeFuture.setFailure( + new ChannelException( + StringUtils.format("Failed to connect to host[%s]", hostname), + f.getCause() + ) + ); + } + } + } + ); + + retVal = handshakeFuture; + } else { + retVal = connectFuture; + } + + return retVal; + } + + @Override + public boolean isGood(ChannelFuture resource) + { + Channel channel = resource.awaitUninterruptibly().getChannel(); + + boolean isSuccess = resource.isSuccess(); + boolean isConnected = channel.isConnected(); + boolean isOpen = channel.isOpen(); + + if (log.isTraceEnabled()) { + log.trace("isGood = isSucess[%s] && isConnected[%s] && isOpen[%s]", isSuccess, isConnected, isOpen); + } + + return isSuccess && isConnected && isOpen; + } + + @Override + public void close(ChannelFuture resource) + { + log.trace("Closing"); + resource.awaitUninterruptibly().getChannel().close(); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/pool/ResourceContainer.java b/java-util/src/main/java/io/druid/java/util/http/client/pool/ResourceContainer.java new file mode 100644 index 000000000000..dcaac747f3e8 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/pool/ResourceContainer.java @@ -0,0 +1,28 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.pool; + +/** + */ +public interface ResourceContainer +{ + ResourceType get(); + void returnResource(); +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/pool/ResourceFactory.java b/java-util/src/main/java/io/druid/java/util/http/client/pool/ResourceFactory.java new file mode 100644 index 000000000000..324345304631 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/pool/ResourceFactory.java @@ -0,0 +1,29 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.pool; + +/** + */ +public interface ResourceFactory +{ + V generate(K key); + boolean isGood(V resource); + void close(V resource); +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/pool/ResourcePool.java b/java-util/src/main/java/io/druid/java/util/http/client/pool/ResourcePool.java new file mode 100644 index 000000000000..a9254f96de77 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/pool/ResourcePool.java @@ -0,0 +1,301 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.pool; + +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableSet; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.logger.Logger; + +import java.io.Closeable; +import java.util.ArrayDeque; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + */ +public class ResourcePool implements Closeable +{ + private static final Logger log = new Logger(ResourcePool.class); + private final LoadingCache> pool; + private final AtomicBoolean closed = new AtomicBoolean(false); + + public ResourcePool( + final ResourceFactory factory, + final ResourcePoolConfig config + ) + { + this.pool = CacheBuilder.newBuilder().build( + new CacheLoader>() + { + @Override + public ImmediateCreationResourceHolder load(K input) throws Exception + { + return new ImmediateCreationResourceHolder( + config.getMaxPerKey(), + config.getUnusedConnectionTimeoutMillis(), + input, + factory + ); + } + } + ); + } + + public ResourceContainer take(final K key) + { + if (closed.get()) { + log.error(StringUtils.format("take(%s) called even though I'm closed.", key)); + return null; + } + + final ImmediateCreationResourceHolder holder; + try { + holder = pool.get(key); + } + catch (ExecutionException e) { + throw Throwables.propagate(e); + } + final V value = holder.get(); + + return new ResourceContainer() + { + private final AtomicBoolean returned = new AtomicBoolean(false); + + @Override + public V get() + { + Preconditions.checkState(!returned.get(), "Resource for key[%s] has been returned, cannot get().", key); + return value; + } + + @Override + public void returnResource() + { + if (returned.getAndSet(true)) { + log.warn(StringUtils.format("Resource at key[%s] was returned multiple times?", key)); + } else { + holder.giveBack(value); + } + } + + @Override + protected void finalize() throws Throwable + { + if (!returned.get()) { + log.warn( + StringUtils.format( + "Resource[%s] at key[%s] was not returned before Container was finalized, potential resource leak.", + value, + key + ) + ); + returnResource(); + } + super.finalize(); + } + }; + } + + @Override + public void close() + { + closed.set(true); + final Map> mapView = pool.asMap(); + for (K k : ImmutableSet.copyOf(mapView.keySet())) { + mapView.remove(k).close(); + } + } + + private static class ImmediateCreationResourceHolder + { + private final int maxSize; + private final K key; + private final ResourceFactory factory; + private final ArrayDeque> resourceHolderList; + private int deficit = 0; + private boolean closed = false; + private final long unusedResourceTimeoutMillis; + + private ImmediateCreationResourceHolder( + int maxSize, + long unusedResourceTimeoutMillis, + K key, + ResourceFactory factory + ) + { + this.maxSize = maxSize; + this.key = key; + this.factory = factory; + this.unusedResourceTimeoutMillis = unusedResourceTimeoutMillis; + this.resourceHolderList = new ArrayDeque<>(); + + for (int i = 0; i < maxSize; ++i) { + resourceHolderList.add(new ResourceHolder<>( + System.currentTimeMillis(), + Preconditions.checkNotNull( + factory.generate(key), + "factory.generate(key)" + ) + )); + } + } + + V get() + { + // resourceHolderList can't have nulls, so we'll use a null to signal that we need to create a new resource. + final V poolVal; + synchronized (this) { + while (!closed && resourceHolderList.size() == 0 && deficit == 0) { + try { + this.wait(); + } + catch (InterruptedException e) { + Thread.interrupted(); + return null; + } + } + + if (closed) { + log.info(StringUtils.format("get() called even though I'm closed. key[%s]", key)); + return null; + } else if (!resourceHolderList.isEmpty()) { + ResourceHolder holder = resourceHolderList.removeFirst(); + if (System.currentTimeMillis() - holder.getLastAccessedTime() > unusedResourceTimeoutMillis) { + factory.close(holder.getResource()); + poolVal = factory.generate(key); + } else { + poolVal = holder.getResource(); + } + } else if (deficit > 0) { + deficit--; + poolVal = null; + } else { + throw new IllegalStateException("WTF?! No objects left, and no object deficit. This is probably a bug."); + } + } + + // At this point, we must either return a valid resource or increment "deficit". + final V retVal; + try { + if (poolVal != null && factory.isGood(poolVal)) { + retVal = poolVal; + } else { + if (poolVal != null) { + factory.close(poolVal); + } + retVal = factory.generate(key); + } + } + catch (Throwable e) { + synchronized (this) { + deficit++; + this.notifyAll(); + } + throw Throwables.propagate(e); + } + + return retVal; + } + + void giveBack(V object) + { + Preconditions.checkNotNull(object, "object"); + + synchronized (this) { + if (closed) { + log.info(StringUtils.format("giveBack called after being closed. key[%s]", key)); + factory.close(object); + return; + } + + if (resourceHolderList.size() >= maxSize) { + if (holderListContains(object)) { + log.warn( + StringUtils.format( + "Returning object[%s] at key[%s] that has already been returned!? Skipping", + object, + key + ), + new Exception("Exception for stacktrace") + ); + } else { + log.warn( + StringUtils.format( + "Returning object[%s] at key[%s] even though we already have all that we can hold[%s]!? Skipping", + object, + key, + resourceHolderList + ), + new Exception("Exception for stacktrace") + ); + } + return; + } + + resourceHolderList.addLast(new ResourceHolder<>(System.currentTimeMillis(), object)); + this.notifyAll(); + } + } + + private boolean holderListContains(V object) + { + return resourceHolderList.stream().anyMatch(a -> a.getResource().equals(object)); + } + + void close() + { + synchronized (this) { + closed = true; + resourceHolderList.forEach(v -> factory.close(v.getResource())); + resourceHolderList.clear(); + this.notifyAll(); + } + } + } + + private static class ResourceHolder + { + private long lastAccessedTime; + private V resource; + + public ResourceHolder(long lastAccessedTime, V resource) + { + this.resource = resource; + this.lastAccessedTime = lastAccessedTime; + } + + public long getLastAccessedTime() + { + return lastAccessedTime; + } + + public V getResource() + { + return resource; + } + + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/pool/ResourcePoolConfig.java b/java-util/src/main/java/io/druid/java/util/http/client/pool/ResourcePoolConfig.java new file mode 100644 index 000000000000..66217d844341 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/pool/ResourcePoolConfig.java @@ -0,0 +1,69 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.pool; + +/** + */ +public class ResourcePoolConfig +{ + private final int maxPerKey; + private final long unusedConnectionTimeoutMillis; + + public ResourcePoolConfig( + int maxPerKey, + long unusedConnectionTimeoutMillis + ) + { + this.maxPerKey = maxPerKey; + this.unusedConnectionTimeoutMillis = unusedConnectionTimeoutMillis; + } + + @Deprecated + public ResourcePoolConfig( + int maxPerKey, + boolean cleanIdle, + long unusedConnectionTimeoutMillis + ) + { + this(maxPerKey, unusedConnectionTimeoutMillis); + + if (cleanIdle) { + throw new IllegalStateException( + "Cleaning up idle connections is a bad idea. " + + "If your services can't handle the max number then lower the max number." + ); + } + } + + public int getMaxPerKey() + { + return maxPerKey; + } + + public boolean isCleanIdle() + { + return false; + } + + public long getUnusedConnectionTimeoutMillis() + { + return unusedConnectionTimeoutMillis; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/pool/ResourceVerifier.java b/java-util/src/main/java/io/druid/java/util/http/client/pool/ResourceVerifier.java new file mode 100644 index 000000000000..926b4bba0abe --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/pool/ResourceVerifier.java @@ -0,0 +1,26 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.pool; + +/** + */ +public interface ResourceVerifier +{ +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/response/ClientResponse.java b/java-util/src/main/java/io/druid/java/util/http/client/response/ClientResponse.java new file mode 100644 index 000000000000..e5bd2ddb16f7 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/response/ClientResponse.java @@ -0,0 +1,57 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.response; + +/** + */ +public class ClientResponse +{ + private final boolean finished; + private final T obj; + + public static ClientResponse finished(T obj) + { + return new ClientResponse(true, obj); + } + + public static ClientResponse unfinished(T obj) + { + return new ClientResponse(false, obj); + } + + protected ClientResponse( + boolean finished, + T obj + ) + { + this.finished = finished; + this.obj = obj; + } + + public boolean isFinished() + { + return finished; + } + + public T getObj() + { + return obj; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/response/FullResponseHandler.java b/java-util/src/main/java/io/druid/java/util/http/client/response/FullResponseHandler.java new file mode 100644 index 000000000000..2b75b4941432 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/response/FullResponseHandler.java @@ -0,0 +1,80 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.response; + +import org.jboss.netty.handler.codec.http.HttpChunk; +import org.jboss.netty.handler.codec.http.HttpResponse; + +import java.nio.charset.Charset; + +/** + */ +public class FullResponseHandler implements HttpResponseHandler +{ + private final Charset charset; + + public FullResponseHandler(Charset charset) + { + this.charset = charset; + } + + @Override + public ClientResponse handleResponse(HttpResponse response) + { + return ClientResponse.unfinished( + new FullResponseHolder( + response.getStatus(), + response, + new StringBuilder(response.getContent().toString(charset)) + ) + ); + } + + @Override + public ClientResponse handleChunk( + ClientResponse response, + HttpChunk chunk + ) + { + final StringBuilder builder = response.getObj().getBuilder(); + + if (builder == null) { + return ClientResponse.finished(null); + } + + builder.append(chunk.getContent().toString(charset)); + return response; + } + + @Override + public ClientResponse done(ClientResponse response) + { + return ClientResponse.finished(response.getObj()); + } + + @Override + public void exceptionCaught( + ClientResponse clientResponse, Throwable e + ) + { + // Its safe to Ignore as the ClientResponse returned in handleChunk were unfinished + } + +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/response/FullResponseHolder.java b/java-util/src/main/java/io/druid/java/util/http/client/response/FullResponseHolder.java new file mode 100644 index 000000000000..b75610cdae8d --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/response/FullResponseHolder.java @@ -0,0 +1,63 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.response; + +import org.jboss.netty.handler.codec.http.HttpResponse; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; + +/** + */ +public class FullResponseHolder +{ + private final HttpResponseStatus status; + private final HttpResponse response; + private final StringBuilder builder; + + public FullResponseHolder( + HttpResponseStatus status, + HttpResponse response, + StringBuilder builder + ) + { + this.status = status; + this.response = response; + this.builder = builder; + } + + public HttpResponseStatus getStatus() + { + return status; + } + + public HttpResponse getResponse() + { + return response; + } + + public StringBuilder getBuilder() + { + return builder; + } + + public String getContent() + { + return builder.toString(); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/response/HttpResponseHandler.java b/java-util/src/main/java/io/druid/java/util/http/client/response/HttpResponseHandler.java new file mode 100644 index 000000000000..69eb792cb9cd --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/response/HttpResponseHandler.java @@ -0,0 +1,54 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.response; + +import org.jboss.netty.handler.codec.http.HttpChunk; +import org.jboss.netty.handler.codec.http.HttpResponse; + +/** + * A handler for an HTTP request. + * + * The ClientResponse object passed around is used to store state between further chunks and indicate when it is safe + * to hand the object back to the caller. + * + * If the response is chunked, the ClientResponse object returned from handleResponse will be passed in as the + * first argument to handleChunk(). + * + * If the ClientResponse object is marked as finished, that indicates that the object stored is safe to hand + * off to the caller. This is most often done either from the done() method after all content has been processed or + * from the initial handleResponse method to indicate that the object is thread-safe and aware that it might be + * accessed before all chunks come back. + * + * Note: if you return a finished ClientResponse object from anything other than the done() method, IntermediateType + * must be castable to FinalType + */ +public interface HttpResponseHandler +{ + /** + * Handles the initial HttpResponse object that comes back from Netty. + * + * @param response - response from Netty + * @return + */ + ClientResponse handleResponse(HttpResponse response); + ClientResponse handleChunk(ClientResponse clientResponse, HttpChunk chunk); + ClientResponse done(ClientResponse clientResponse); + void exceptionCaught(ClientResponse clientResponse, Throwable e); +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/response/InputStreamResponseHandler.java b/java-util/src/main/java/io/druid/java/util/http/client/response/InputStreamResponseHandler.java new file mode 100644 index 000000000000..e2792842d226 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/response/InputStreamResponseHandler.java @@ -0,0 +1,74 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.response; + +import io.druid.java.util.http.client.io.AppendableByteArrayInputStream; +import org.jboss.netty.buffer.ChannelBuffer; +import org.jboss.netty.handler.codec.http.HttpChunk; +import org.jboss.netty.handler.codec.http.HttpResponse; + +import java.io.InputStream; + +/** + */ +public class InputStreamResponseHandler implements HttpResponseHandler +{ + @Override + public ClientResponse handleResponse(HttpResponse response) + { + AppendableByteArrayInputStream in = new AppendableByteArrayInputStream(); + in.add(getContentBytes(response.getContent())); + return ClientResponse.finished(in); + } + + @Override + public ClientResponse handleChunk( + ClientResponse clientResponse, HttpChunk chunk + ) + { + clientResponse.getObj().add(getContentBytes(chunk.getContent())); + return clientResponse; + } + + @Override + public ClientResponse done(ClientResponse clientResponse) + { + final AppendableByteArrayInputStream obj = clientResponse.getObj(); + obj.done(); + return ClientResponse.finished(obj); + } + + @Override + public void exceptionCaught( + ClientResponse clientResponse, + Throwable e + ) + { + final AppendableByteArrayInputStream obj = clientResponse.getObj(); + obj.exceptionCaught(e); + } + + private byte[] getContentBytes(ChannelBuffer content) + { + byte[] contentBytes = new byte[content.readableBytes()]; + content.readBytes(contentBytes); + return contentBytes; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/response/SequenceInputStreamResponseHandler.java b/java-util/src/main/java/io/druid/java/util/http/client/response/SequenceInputStreamResponseHandler.java new file mode 100644 index 000000000000..ffe0a9a14187 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/response/SequenceInputStreamResponseHandler.java @@ -0,0 +1,184 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.response; + +import com.google.common.base.Throwables; +import com.google.common.io.ByteSource; +import io.druid.java.util.common.logger.Logger; +import org.jboss.netty.buffer.ChannelBuffer; +import org.jboss.netty.buffer.ChannelBufferInputStream; +import org.jboss.netty.handler.codec.http.HttpChunk; +import org.jboss.netty.handler.codec.http.HttpResponse; + +import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.util.Enumeration; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A HTTP response handler which uses sequence input streams to create a final InputStream. + * Any particular instance is encouraged to overwrite a method and call the super if they need extra handling of the + * method parameters. + * + * This implementation uses a blocking queue to feed a SequenceInputStream that is terminated whenever the handler's Done + * method is called or a throwable is detected. + * + * The resulting InputStream will attempt to terminate normally, but on exception in HttpResponseHandler + * may end with an IOException upon read() + */ +public class SequenceInputStreamResponseHandler implements HttpResponseHandler +{ + private static final Logger log = new Logger(SequenceInputStreamResponseHandler.class); + private final AtomicLong byteCount = new AtomicLong(0); + private final BlockingQueue queue = new LinkedBlockingQueue<>(); + private final AtomicBoolean done = new AtomicBoolean(false); + + @Override + public ClientResponse handleResponse(HttpResponse response) + { + try { + queue.put(new ChannelBufferInputStream(response.getContent())); + } + catch (InterruptedException e) { + log.error(e, "Queue appending interrupted"); + Thread.currentThread().interrupt(); + throw Throwables.propagate(e); + } + byteCount.addAndGet(response.getContent().readableBytes()); + return ClientResponse.finished( + new SequenceInputStream( + new Enumeration() + { + @Override + public boolean hasMoreElements() + { + // Done is always true until the last stream has be put in the queue. + // Then the stream should be spouting good InputStreams. + synchronized (done) { + return !done.get() || !queue.isEmpty(); + } + } + + @Override + public InputStream nextElement() + { + try { + return queue.take(); + } + catch (InterruptedException e) { + log.warn(e, "Thread interrupted while taking from queue"); + Thread.currentThread().interrupt(); + throw Throwables.propagate(e); + } + } + } + ) + ); + } + + @Override + public ClientResponse handleChunk( + ClientResponse clientResponse, HttpChunk chunk + ) + { + final ChannelBuffer channelBuffer = chunk.getContent(); + final int bytes = channelBuffer.readableBytes(); + if (bytes > 0) { + try { + queue.put(new ChannelBufferInputStream(channelBuffer)); + // Queue.size() can be expensive in some implementations, but LinkedBlockingQueue.size is just an AtomicLong + log.debug("Added stream. Queue length %d", queue.size()); + } + catch (InterruptedException e) { + log.warn(e, "Thread interrupted while adding to queue"); + Thread.currentThread().interrupt(); + throw Throwables.propagate(e); + } + byteCount.addAndGet(bytes); + } else { + log.debug("Skipping zero length chunk"); + } + return clientResponse; + } + + @Override + public ClientResponse done(ClientResponse clientResponse) + { + synchronized (done) { + try { + // An empty byte array is put at the end to give the SequenceInputStream.close() as something to close out + // after done is set to true, regardless of the rest of the stream's state. + queue.put(ByteSource.empty().openStream()); + log.debug("Added terminal empty stream"); + } + catch (InterruptedException e) { + log.warn(e, "Thread interrupted while adding to queue"); + Thread.currentThread().interrupt(); + throw Throwables.propagate(e); + } + catch (IOException e) { + // This should never happen + log.wtf(e, "The empty stream threw an IOException"); + throw Throwables.propagate(e); + } + finally { + log.debug("Done after adding %d bytes of streams", byteCount.get()); + done.set(true); + } + } + return ClientResponse.finished(clientResponse.getObj()); + } + + @Override + public void exceptionCaught(final ClientResponse clientResponse, final Throwable e) + { + // Don't wait for lock in case the lock had something to do with the error + synchronized (done) { + done.set(true); + // Make a best effort to put a zero length buffer into the queue in case something is waiting on the take() + // If nothing is waiting on take(), this will be closed out anyways. + final boolean accepted = queue.offer( + new InputStream() + { + @Override + public int read() throws IOException + { + throw new IOException(e); + } + } + ); + if (!accepted) { + log.warn("Unable to place final IOException offer in queue"); + } else { + log.debug("Placed IOException in queue"); + } + log.debug(e, "Exception with queue length of %d and %d bytes available", queue.size(), byteCount.get()); + } + } + + public final long getByteCount() + { + return byteCount.get(); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/response/StatusResponseHandler.java b/java-util/src/main/java/io/druid/java/util/http/client/response/StatusResponseHandler.java new file mode 100644 index 000000000000..c0b55e902e99 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/response/StatusResponseHandler.java @@ -0,0 +1,79 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.response; + +import org.jboss.netty.handler.codec.http.HttpChunk; +import org.jboss.netty.handler.codec.http.HttpResponse; + +import java.nio.charset.Charset; + +/** + */ +public class StatusResponseHandler implements HttpResponseHandler +{ + private final Charset charset; + + public StatusResponseHandler(Charset charset) + { + this.charset = charset; + } + + @Override + public ClientResponse handleResponse(HttpResponse response) + { + return ClientResponse.unfinished( + new StatusResponseHolder( + response.getStatus(), + new StringBuilder(response.getContent().toString(charset)) + ) + ); + } + + @Override + public ClientResponse handleChunk( + ClientResponse response, + HttpChunk chunk + ) + { + final StringBuilder builder = response.getObj().getBuilder(); + + if (builder == null) { + return ClientResponse.finished(null); + } + + builder.append(chunk.getContent().toString(charset)); + return response; + } + + @Override + public ClientResponse done(ClientResponse response) + { + return ClientResponse.finished(response.getObj()); + } + + @Override + public void exceptionCaught( + ClientResponse clientResponse, Throwable e + ) + { + // Its safe to Ignore as the ClientResponse returned in handleChunk were unfinished + } + +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/response/StatusResponseHolder.java b/java-util/src/main/java/io/druid/java/util/http/client/response/StatusResponseHolder.java new file mode 100644 index 000000000000..58142e1a2c2b --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/response/StatusResponseHolder.java @@ -0,0 +1,54 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.response; + +import org.jboss.netty.handler.codec.http.HttpResponseStatus; + +/** + */ +public class StatusResponseHolder +{ + private final HttpResponseStatus status; + private final StringBuilder builder; + + public StatusResponseHolder( + HttpResponseStatus status, + StringBuilder builder + ) + { + this.status = status; + this.builder = builder; + } + + public HttpResponseStatus getStatus() + { + return status; + } + + public StringBuilder getBuilder() + { + return builder; + } + + public String getContent() + { + return builder.toString(); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/http/client/response/ToStringResponseHandler.java b/java-util/src/main/java/io/druid/java/util/http/client/response/ToStringResponseHandler.java new file mode 100644 index 000000000000..2533b6b90801 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/http/client/response/ToStringResponseHandler.java @@ -0,0 +1,78 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.response; + +import org.jboss.netty.handler.codec.http.HttpChunk; +import org.jboss.netty.handler.codec.http.HttpResponse; + +import java.nio.charset.Charset; + +/** + */ +public class ToStringResponseHandler implements HttpResponseHandler +{ + private final Charset charset; + + public ToStringResponseHandler(Charset charset) + { + this.charset = charset; + } + + @Override + public ClientResponse handleResponse(HttpResponse response) + { + return ClientResponse.unfinished(new StringBuilder(response.getContent().toString(charset))); + } + + @Override + public ClientResponse handleChunk( + ClientResponse response, + HttpChunk chunk + ) + { + final StringBuilder builder = response.getObj(); + if (builder == null) { + return ClientResponse.finished(null); + } + + builder.append(chunk.getContent().toString(charset)); + return response; + } + + @Override + public ClientResponse done(ClientResponse response) + { + final StringBuilder builder = response.getObj(); + if (builder == null) { + return ClientResponse.finished(null); + } + + return ClientResponse.finished(builder.toString()); + } + + @Override + public void exceptionCaught( + ClientResponse clientResponse, Throwable e + ) + { + // Its safe to Ignore as the ClientResponse returned in handleChunk were unfinished + } + +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/AbstractMonitor.java b/java-util/src/main/java/io/druid/java/util/metrics/AbstractMonitor.java new file mode 100644 index 000000000000..e3f757d619f1 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/AbstractMonitor.java @@ -0,0 +1,54 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + + +import io.druid.java.util.emitter.service.ServiceEmitter; + +/** + */ +public abstract class AbstractMonitor implements Monitor +{ + private volatile boolean started = false; + + @Override + public void start() + { + started = true; + } + + @Override + public void stop() + { + started = false; + } + + @Override + public boolean monitor(ServiceEmitter emitter) + { + if (started) { + return doMonitor(emitter); + } + + return false; + } + + public abstract boolean doMonitor(ServiceEmitter emitter); +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/CgroupUtil.java b/java-util/src/main/java/io/druid/java/util/metrics/CgroupUtil.java new file mode 100644 index 000000000000..a476b4f3159f --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/CgroupUtil.java @@ -0,0 +1,29 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import java.util.regex.Pattern; + +public class CgroupUtil +{ + public static final String SPACE_MATCH = Pattern.quote(" "); + public static final String COMMA_MATCH = Pattern.quote(","); + public static final String COLON_MATCH = Pattern.quote(":"); +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/CompoundMonitor.java b/java-util/src/main/java/io/druid/java/util/metrics/CompoundMonitor.java new file mode 100644 index 000000000000..1e1ca4634a05 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/CompoundMonitor.java @@ -0,0 +1,75 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import com.google.common.base.Function; +import com.google.common.collect.Lists; +import io.druid.java.util.emitter.service.ServiceEmitter; + +import java.util.Arrays; +import java.util.List; + +public abstract class CompoundMonitor implements Monitor +{ + private final List monitors; + + public CompoundMonitor(List monitors) + { + this.monitors = monitors; + } + + public CompoundMonitor(Monitor... monitors) + { + this(Arrays.asList(monitors)); + } + + @Override + public void start() + { + for (Monitor monitor : monitors) { + monitor.start(); + } + } + + @Override + public void stop() + { + for (Monitor monitor : monitors) { + monitor.stop(); + } + } + + @Override + public boolean monitor(final ServiceEmitter emitter) + { + return shouldReschedule(Lists.transform(monitors, + new Function() + { + @Override + public Boolean apply(Monitor monitor) + { + return monitor.monitor(emitter); + } + } + )); + } + + public abstract boolean shouldReschedule(List reschedules); +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/CpuAcctDeltaMonitor.java b/java-util/src/main/java/io/druid/java/util/metrics/CpuAcctDeltaMonitor.java new file mode 100644 index 000000000000..e3a21f1d7203 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/CpuAcctDeltaMonitor.java @@ -0,0 +1,134 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import io.druid.java.util.common.DateTimes; +import io.druid.java.util.common.logger.Logger; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; +import io.druid.java.util.metrics.cgroups.CgroupDiscoverer; +import io.druid.java.util.metrics.cgroups.CpuAcct; +import io.druid.java.util.metrics.cgroups.ProcSelfCgroupDiscoverer; +import org.joda.time.DateTime; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +public class CpuAcctDeltaMonitor extends FeedDefiningMonitor +{ + private static final Logger log = new Logger(CpuAcctDeltaMonitor.class); + private final AtomicReference priorSnapshot = new AtomicReference<>(null); + private final Map dimensions; + + private final CgroupDiscoverer cgroupDiscoverer; + + public CpuAcctDeltaMonitor() + { + this(ImmutableMap.of()); + } + + public CpuAcctDeltaMonitor(final Map dimensions) + { + this(dimensions, DEFAULT_METRICS_FEED); + } + + public CpuAcctDeltaMonitor(final Map dimensions, final String feed) + { + this(feed, dimensions, new ProcSelfCgroupDiscoverer()); + } + + public CpuAcctDeltaMonitor( + String feed, + Map dimensions, + CgroupDiscoverer cgroupDiscoverer + ) + { + super(feed); + Preconditions.checkNotNull(dimensions); + this.dimensions = ImmutableMap.copyOf(dimensions); + this.cgroupDiscoverer = Preconditions.checkNotNull(cgroupDiscoverer, "cgroupDiscoverer required"); + } + + @Override + public boolean doMonitor(ServiceEmitter emitter) + { + final CpuAcct cpuAcct = new CpuAcct(cgroupDiscoverer); + final CpuAcct.CpuAcctMetric snapshot = cpuAcct.snapshot(); + final long nanoTime = System.nanoTime(); // Approx time... may be influenced by an unlucky GC + final DateTime dateTime = DateTimes.nowUtc(); + final SnapshotHolder priorSnapshotHolder = this.priorSnapshot.get(); + if (!priorSnapshot.compareAndSet(priorSnapshotHolder, new SnapshotHolder(snapshot, nanoTime))) { + log.debug("Pre-empted by another monitor run"); + return false; + } + if (priorSnapshotHolder == null) { + log.info("Detected first run, storing result for next run"); + return false; + } + final long elapsedNs = nanoTime - priorSnapshotHolder.timestamp; + if (snapshot.cpuCount() != priorSnapshotHolder.metric.cpuCount()) { + log.warn( + "Prior CPU count [%d] does not match current cpu count [%d]. Skipping metrics emission", + priorSnapshotHolder.metric.cpuCount(), + snapshot.cpuCount() + ); + return false; + } + for (int i = 0; i < snapshot.cpuCount(); ++i) { + final ServiceMetricEvent.Builder builderUsr = builder() + .setDimension("cpuName", Integer.toString(i)) + .setDimension("cpuTime", "usr"); + final ServiceMetricEvent.Builder builderSys = builder() + .setDimension("cpuName", Integer.toString(i)) + .setDimension("cpuTime", "sys"); + MonitorUtils.addDimensionsToBuilder(builderUsr, dimensions); + MonitorUtils.addDimensionsToBuilder(builderSys, dimensions); + emitter.emit(builderUsr.build( + dateTime, + "cgroup/cpu_time_delta_ns", + snapshot.usrTime(i) - priorSnapshotHolder.metric.usrTime(i) + )); + emitter.emit(builderSys.build( + dateTime, + "cgroup/cpu_time_delta_ns", + snapshot.sysTime(i) - priorSnapshotHolder.metric.sysTime(i) + )); + } + if (snapshot.cpuCount() > 0) { + // Don't bother emitting metrics if there aren't actually any cpus (usually from error) + emitter.emit(builder().build(dateTime, "cgroup/cpu_time_delta_ns_elapsed", elapsedNs)); + } + return true; + } + + static class SnapshotHolder + { + private final CpuAcct.CpuAcctMetric metric; + private final long timestamp; + + SnapshotHolder(CpuAcct.CpuAcctMetric metric, long timestamp) + { + this.metric = metric; + this.timestamp = timestamp; + } + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/FeedDefiningMonitor.java b/java-util/src/main/java/io/druid/java/util/metrics/FeedDefiningMonitor.java new file mode 100644 index 000000000000..aa3d2863d5b1 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/FeedDefiningMonitor.java @@ -0,0 +1,40 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import com.google.common.base.Preconditions; +import io.druid.java.util.emitter.service.ServiceMetricEvent; + +public abstract class FeedDefiningMonitor extends AbstractMonitor +{ + public static final String DEFAULT_METRICS_FEED = "metrics"; + protected final String feed; + + public FeedDefiningMonitor(String feed) + { + Preconditions.checkNotNull(feed); + this.feed = feed; + } + + protected ServiceMetricEvent.Builder builder() + { + return ServiceMetricEvent.builder().setFeed(feed); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/HttpPostEmitterMonitor.java b/java-util/src/main/java/io/druid/java/util/metrics/HttpPostEmitterMonitor.java new file mode 100644 index 000000000000..590eabda17d6 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/HttpPostEmitterMonitor.java @@ -0,0 +1,91 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import com.google.common.collect.ImmutableMap; +import io.druid.java.util.emitter.core.ConcurrentTimeCounter; +import io.druid.java.util.emitter.core.HttpPostEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; + +public class HttpPostEmitterMonitor extends FeedDefiningMonitor +{ + private final HttpPostEmitter httpPostEmitter; + private final ImmutableMap extraDimensions; + private final ServiceMetricEvent.Builder builder; + private long lastTotalEmittedEvents = 0; + private int lastDroppedBuffers = 0; + + public HttpPostEmitterMonitor( + String feed, + HttpPostEmitter httpPostEmitter, + ImmutableMap extraDimensions + ) + { + super(feed); + this.httpPostEmitter = httpPostEmitter; + this.extraDimensions = extraDimensions; + this.builder = builder(); + } + + @Override + public boolean doMonitor(ServiceEmitter emitter) + { + long newTotalEmittedEvents = httpPostEmitter.getTotalEmittedEvents(); + long totalEmittedEventsDiff = newTotalEmittedEvents - lastTotalEmittedEvents; + emitter.emit(builder.build("emitter/events/emitted", totalEmittedEventsDiff)); + lastTotalEmittedEvents = newTotalEmittedEvents; + + int newDroppedBuffers = httpPostEmitter.getDroppedBuffers(); + int droppedBuffersDiff = newDroppedBuffers - lastDroppedBuffers; + emitter.emit(builder.build("emitter/buffers/dropped", droppedBuffersDiff)); + lastDroppedBuffers = newDroppedBuffers; + + emitTimeCounterMetrics(emitter, httpPostEmitter.getBatchFillingTimeCounter(), "emitter/batchFilling/"); + emitTimeCounterMetrics(emitter, httpPostEmitter.getSuccessfulSendingTimeCounter(), "emitter/successfulSending/"); + emitTimeCounterMetrics(emitter, httpPostEmitter.getFailedSendingTimeCounter(), "emitter/failedSending/"); + + emitter.emit(builder.build("emitter/events/emitQueue", httpPostEmitter.getEventsToEmit())); + emitter.emit(builder.build("emitter/events/large/emitQueue", httpPostEmitter.getLargeEventsToEmit())); + emitter.emit(builder.build("emitter/buffers/totalAllocated", httpPostEmitter.getTotalAllocatedBuffers())); + emitter.emit(builder.build("emitter/buffers/emitQueue", httpPostEmitter.getBuffersToEmit())); + emitter.emit(builder.build("emitter/buffers/failed", httpPostEmitter.getFailedBuffers())); + emitter.emit(builder.build("emitter/buffers/reuseQueue", httpPostEmitter.getBuffersToReuse())); + + return true; + } + + private void emitTimeCounterMetrics(ServiceEmitter emitter, ConcurrentTimeCounter timeCounter, String metricNameBase) + { + long timeSumAndCount = timeCounter.getTimeSumAndCountAndReset(); + emitter.emit(builder.build(metricNameBase + "timeMsSum", ConcurrentTimeCounter.timeSum(timeSumAndCount))); + emitter.emit(builder.build(metricNameBase + "count", ConcurrentTimeCounter.count(timeSumAndCount))); + emitter.emit(builder.build(metricNameBase + "maxTimeMs", timeCounter.getAndResetMaxTime())); + emitter.emit(builder.build(metricNameBase + "minTimeMs", timeCounter.getAndResetMinTime())); + } + + @Override + protected ServiceMetricEvent.Builder builder() + { + ServiceMetricEvent.Builder builder = super.builder(); + extraDimensions.forEach(builder::setDimension); + return builder; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/JvmCpuMonitor.java b/java-util/src/main/java/io/druid/java/util/metrics/JvmCpuMonitor.java new file mode 100644 index 000000000000..fcefa3d9bfd3 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/JvmCpuMonitor.java @@ -0,0 +1,89 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import io.druid.java.util.common.logger.Logger; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; +import org.hyperic.sigar.ProcCpu; +import org.hyperic.sigar.Sigar; +import org.hyperic.sigar.SigarException; + +import java.util.Map; + +public class JvmCpuMonitor extends FeedDefiningMonitor +{ + private static final Logger log = new Logger(JvmCpuMonitor.class); + + private final Sigar sigar = SigarUtil.getSigar(); + private final long currentProcessId = sigar.getPid(); + + private final KeyedDiff diff = new KeyedDiff(); + + private Map dimensions; + + public JvmCpuMonitor() + { + this(ImmutableMap.of()); + } + + public JvmCpuMonitor(Map dimensions) + { + this(dimensions, DEFAULT_METRICS_FEED); + } + + public JvmCpuMonitor(Map dimensions, String feed) + { + super(feed); + Preconditions.checkNotNull(dimensions); + this.dimensions = ImmutableMap.copyOf(dimensions); + } + + @Override + public boolean doMonitor(ServiceEmitter emitter) + { + // process CPU + try { + ProcCpu procCpu = sigar.getProcCpu(currentProcessId); + final ServiceMetricEvent.Builder builder = builder(); + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + // delta for total, sys, user + Map procDiff = diff.to( + "proc/cpu", ImmutableMap.of( + "jvm/cpu/total", procCpu.getTotal(), + "jvm/cpu/sys", procCpu.getSys(), + "jvm/cpu/user", procCpu.getUser() + ) + ); + if (procDiff != null) { + for (Map.Entry entry : procDiff.entrySet()) { + emitter.emit(builder.build(entry.getKey(), entry.getValue())); + } + } + emitter.emit(builder.build("jvm/cpu/percent", procCpu.getPercent())); + } + catch (SigarException e) { + log.error(e, "Failed to get ProcCpu"); + } + return true; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/JvmMonitor.java b/java-util/src/main/java/io/druid/java/util/metrics/JvmMonitor.java new file mode 100644 index 000000000000..6f696be05be0 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/JvmMonitor.java @@ -0,0 +1,315 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; +import org.gridkit.lab.jvm.perfdata.JStatData; +import org.gridkit.lab.jvm.perfdata.JStatData.LongCounter; +import org.gridkit.lab.jvm.perfdata.JStatData.StringCounter; +import org.gridkit.lab.jvm.perfdata.JStatData.TickCounter; + +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryType; +import java.lang.management.MemoryUsage; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class JvmMonitor extends FeedDefiningMonitor +{ + private final Map dimensions; + private final long pid; + + private final GcCounters gcCounters = new GcCounters(); + + public JvmMonitor() + { + this(ImmutableMap.of()); + } + + public JvmMonitor(Map dimensions) + { + this(dimensions, DEFAULT_METRICS_FEED); + } + + public JvmMonitor(Map dimensions, String feed) + { + this(dimensions, feed, JvmPidDiscoverer.instance()); + } + + public JvmMonitor(Map dimensions, String feed, PidDiscoverer pidDiscoverer) + { + super(feed); + Preconditions.checkNotNull(dimensions); + this.dimensions = ImmutableMap.copyOf(dimensions); + this.pid = Preconditions.checkNotNull(pidDiscoverer).getPid(); + } + + @Override + public boolean doMonitor(ServiceEmitter emitter) + { + emitJvmMemMetrics(emitter); + emitDirectMemMetrics(emitter); + emitGcMetrics(emitter); + + return true; + } + + /** + * These metrics are going to be replaced by new jvm/gc/mem/* metrics + */ + @Deprecated + private void emitJvmMemMetrics(ServiceEmitter emitter) + { + // I have no idea why, but jvm/mem is slightly more than the sum of jvm/pool. Let's just include + // them both. + final Map usages = ImmutableMap.of( + "heap", ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(), + "nonheap", ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage() + ); + for (Map.Entry entry : usages.entrySet()) { + final String kind = entry.getKey(); + final MemoryUsage usage = entry.getValue(); + final ServiceMetricEvent.Builder builder = builder().setDimension("memKind", kind); + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + + emitter.emit(builder.build("jvm/mem/max", usage.getMax())); + emitter.emit(builder.build("jvm/mem/committed", usage.getCommitted())); + emitter.emit(builder.build("jvm/mem/used", usage.getUsed())); + emitter.emit(builder.build("jvm/mem/init", usage.getInit())); + } + + // jvm/pool + for (MemoryPoolMXBean pool : ManagementFactory.getMemoryPoolMXBeans()) { + final String kind = pool.getType() == MemoryType.HEAP ? "heap" : "nonheap"; + final MemoryUsage usage = pool.getUsage(); + final ServiceMetricEvent.Builder builder = builder() + .setDimension("poolKind", kind) + .setDimension("poolName", pool.getName()); + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + + emitter.emit(builder.build("jvm/pool/max", usage.getMax())); + emitter.emit(builder.build("jvm/pool/committed", usage.getCommitted())); + emitter.emit(builder.build("jvm/pool/used", usage.getUsed())); + emitter.emit(builder.build("jvm/pool/init", usage.getInit())); + } + } + + private void emitDirectMemMetrics(ServiceEmitter emitter) + { + for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) { + final ServiceMetricEvent.Builder builder = builder().setDimension("bufferpoolName", pool.getName()); + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + + emitter.emit(builder.build("jvm/bufferpool/capacity", pool.getTotalCapacity())); + emitter.emit(builder.build("jvm/bufferpool/used", pool.getMemoryUsed())); + emitter.emit(builder.build("jvm/bufferpool/count", pool.getCount())); + } + } + + private void emitGcMetrics(ServiceEmitter emitter) + { + gcCounters.emit(emitter, dimensions); + } + + /** + * The following GC-related code is partially based on + * https://github.com/aragozin/jvm-tools/blob/e0e37692648951440aa1a4ea5046261cb360df70/ + * sjk-core/src/main/java/org/gridkit/jvmtool/PerfCounterGcCpuUsageMonitor.java + */ + private class GcCounters + { + private final List generations = new ArrayList<>(); + + GcCounters() + { + // connect to itself + final JStatData jStatData = JStatData.connect(pid); + final Map> jStatCounters = jStatData.getAllCounters(); + + generations.add(new GcGeneration(jStatCounters, 0, "young")); + generations.add(new GcGeneration(jStatCounters, 1, "old")); + // Removed in Java 8 but still actual for previous Java versions + if (jStatCounters.containsKey("sun.gc.generation.2.name")) { + generations.add(new GcGeneration(jStatCounters, 2, "perm")); + } + } + + void emit(ServiceEmitter emitter, Map dimensions) + { + for (GcGeneration generation : generations) { + generation.emit(emitter, dimensions); + } + } + } + + private class GcGeneration + { + private final String name; + private final GcGenerationCollector collector; + private final List spaces = new ArrayList<>(); + + GcGeneration(Map> jStatCounters, long genIndex, String name) + { + this.name = StringUtils.toLowerCase(name); + + long spacesCount = ((JStatData.LongCounter) jStatCounters.get( + StringUtils.format("sun.gc.generation.%d.spaces", genIndex) + )).getLong(); + for (long spaceIndex = 0; spaceIndex < spacesCount; spaceIndex++) { + spaces.add(new GcGenerationSpace(jStatCounters, genIndex, spaceIndex)); + } + + if (jStatCounters.containsKey(StringUtils.format("sun.gc.collector.%d.name", genIndex))) { + collector = new GcGenerationCollector(jStatCounters, genIndex); + } else { + collector = null; + } + } + + void emit(ServiceEmitter emitter, Map dimensions) + { + ImmutableMap.Builder dimensionsCopyBuilder = ImmutableMap + .builder() + .putAll(dimensions) + .put("gcGen", new String[]{name}); + + if (collector != null) { + dimensionsCopyBuilder.put("gcName", new String[]{collector.name}); + } + + Map dimensionsCopy = dimensionsCopyBuilder.build(); + + if (collector != null) { + collector.emit(emitter, dimensionsCopy); + } + + for (GcGenerationSpace space : spaces) { + space.emit(emitter, dimensionsCopy); + } + } + } + + private class GcGenerationCollector + { + private final String name; + private final LongCounter invocationsCounter; + private final TickCounter cpuCounter; + private long lastInvocations = 0; + private long lastCpuNanos = 0; + + GcGenerationCollector(Map> jStatCounters, long genIndex) + { + String collectorKeyPrefix = StringUtils.format("sun.gc.collector.%d", genIndex); + + String nameKey = StringUtils.format("%s.name", collectorKeyPrefix); + StringCounter nameCounter = (StringCounter) jStatCounters.get(nameKey); + name = getReadableName(nameCounter.getString()); + + invocationsCounter = (LongCounter) jStatCounters.get(StringUtils.format("%s.invocations", collectorKeyPrefix)); + cpuCounter = (TickCounter) jStatCounters.get(StringUtils.format("%s.time", collectorKeyPrefix)); + } + + void emit(ServiceEmitter emitter, Map dimensions) + { + final ServiceMetricEvent.Builder builder = builder(); + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + + long newInvocations = invocationsCounter.getLong(); + emitter.emit(builder.build("jvm/gc/count", newInvocations - lastInvocations)); + lastInvocations = newInvocations; + + long newCpuNanos = cpuCounter.getNanos(); + emitter.emit(builder.build("jvm/gc/cpu", newCpuNanos - lastCpuNanos)); + lastCpuNanos = newCpuNanos; + } + + private String getReadableName(String name) + { + switch (name) { + // Young gen + case "Copy": + return "serial"; + case "PSScavenge": + return "parallel"; + case "PCopy": + return "cms"; + case "G1 incremental collections": + return "g1"; + + // Old gen + case "MCS": + return "serial"; + case "PSParallelCompact": + return "parallel"; + case "CMS": + return "cms"; + case "G1 stop-the-world full collections": + return "g1"; + + default: + return name; + } + } + } + + private class GcGenerationSpace + { + private final String name; + + private final LongCounter maxCounter; + private final LongCounter capacityCounter; + private final LongCounter usedCounter; + private final LongCounter initCounter; + + GcGenerationSpace(Map> jStatCounters, long genIndex, long spaceIndex) + { + String spaceKeyPrefix = StringUtils.format("sun.gc.generation.%d.space.%d", genIndex, spaceIndex); + + String nameKey = StringUtils.format("%s.name", spaceKeyPrefix); + StringCounter nameCounter = (StringCounter) jStatCounters.get(nameKey); + name = StringUtils.toLowerCase(nameCounter.toString()); + + maxCounter = (LongCounter) jStatCounters.get(StringUtils.format("%s.maxCapacity", spaceKeyPrefix)); + capacityCounter = (LongCounter) jStatCounters.get(StringUtils.format("%s.capacity", spaceKeyPrefix)); + usedCounter = (LongCounter) jStatCounters.get(StringUtils.format("%s.used", spaceKeyPrefix)); + initCounter = (LongCounter) jStatCounters.get(StringUtils.format("%s.initCapacity", spaceKeyPrefix)); + } + + void emit(ServiceEmitter emitter, Map dimensions) + { + final ServiceMetricEvent.Builder builder = builder(); + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + + builder.setDimension("gcGenSpaceName", name); + + emitter.emit(builder.build("jvm/gc/mem/max", maxCounter.getLong())); + emitter.emit(builder.build("jvm/gc/mem/capacity", capacityCounter.getLong())); + emitter.emit(builder.build("jvm/gc/mem/used", usedCounter.getLong())); + emitter.emit(builder.build("jvm/gc/mem/init", initCounter.getLong())); + } + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/JvmPidDiscoverer.java b/java-util/src/main/java/io/druid/java/util/metrics/JvmPidDiscoverer.java new file mode 100644 index 000000000000..b279f43e18a4 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/JvmPidDiscoverer.java @@ -0,0 +1,80 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import io.druid.java.util.common.RE; + +import java.lang.management.ManagementFactory; +import java.util.regex.Pattern; + +/** + * For systems that for whatever reason cannot use Sigar (through io.druid.java.util.metrics.SigarPidDiscoverer ), + * this attempts to get the PID from the JVM "name". + */ +public class JvmPidDiscoverer implements PidDiscoverer +{ + private static final JvmPidDiscoverer INSTANCE = new JvmPidDiscoverer(); + + public static JvmPidDiscoverer instance() + { + return INSTANCE; + } + + /** + * use {JvmPidDiscoverer.instance()} + */ + private JvmPidDiscoverer() + { + } + + /** + * Returns the PID as a best guess. This uses methods that are not guaranteed to actually be the PID. + *

+ * TODO: switch to ProcessHandle.current().getPid() for java9 potentially + * + * @return the PID of the current jvm if available + * + * @throws RuntimeException if the pid cannot be determined + */ + @Override + public long getPid() + { + return Inner.PID; + } + + private static class Inner + { + private static final long PID; + + static { + final String jvmName = ManagementFactory.getRuntimeMXBean().getName(); + final String[] nameSplits = jvmName.split(Pattern.quote("@")); + if (nameSplits.length != 2) { + throw new RE("Unable to determine pid from [%s]", jvmName); + } + try { + PID = Long.parseLong(nameSplits[0]); + } + catch (NumberFormatException nfe) { + throw new RE(nfe, "Unable to determine pid from [%s]", jvmName); + } + } + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/JvmThreadsMonitor.java b/java-util/src/main/java/io/druid/java/util/metrics/JvmThreadsMonitor.java new file mode 100644 index 000000000000..60efeb004472 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/JvmThreadsMonitor.java @@ -0,0 +1,83 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; + +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadMXBean; +import java.util.Map; + +public class JvmThreadsMonitor extends FeedDefiningMonitor +{ + private final Map dimensions; + + private int lastLiveThreads = 0; + private long lastStartedThreads = 0; + + public JvmThreadsMonitor() + { + this(ImmutableMap.of()); + } + + public JvmThreadsMonitor(Map dimensions) + { + this(dimensions, DEFAULT_METRICS_FEED); + } + + public JvmThreadsMonitor(Map dimensions, String feed) + { + super(feed); + Preconditions.checkNotNull(dimensions); + this.dimensions = ImmutableMap.copyOf(dimensions); + } + + @Override + public boolean doMonitor(ServiceEmitter emitter) + { + ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); + + final ServiceMetricEvent.Builder builder = builder(); + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + + // Because between next two calls on ThreadMXBean new threads can be started we can observe some inconsistency + // in counters values and finished counter could be even negative + int newLiveThreads = threadBean.getThreadCount(); + long newStartedThreads = threadBean.getTotalStartedThreadCount(); + + long startedThreadsDiff = newStartedThreads - lastStartedThreads; + + emitter.emit(builder.build("jvm/threads/started", startedThreadsDiff)); + emitter.emit(builder.build("jvm/threads/finished", lastLiveThreads + startedThreadsDiff - newLiveThreads)); + emitter.emit(builder.build("jvm/threads/live", newLiveThreads)); + emitter.emit(builder.build("jvm/threads/liveDaemon", threadBean.getDaemonThreadCount())); + + emitter.emit(builder.build("jvm/threads/livePeak", threadBean.getPeakThreadCount())); + threadBean.resetPeakThreadCount(); + + lastStartedThreads = newStartedThreads; + lastLiveThreads = newLiveThreads; + + return true; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/KeyedDiff.java b/java-util/src/main/java/io/druid/java/util/metrics/KeyedDiff.java new file mode 100644 index 000000000000..7378071f7191 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/KeyedDiff.java @@ -0,0 +1,53 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import io.druid.java.util.common.logger.Logger; + +import java.util.HashMap; +import java.util.Map; + +public class KeyedDiff +{ + private static final Logger log = new Logger(KeyedDiff.class); + + private final Map> prevs = new HashMap>(); + + public Map to(String key, Map curr) + { + final Map prev = prevs.put(key, curr); + if (prev != null) { + return subtract(curr, prev); + } else { + log.debug("No previous data for key[%s]", key); + return null; + } + } + + public static Map subtract(Map xs, Map ys) + { + assert xs.keySet().equals(ys.keySet()); + final Map zs = new HashMap(); + for (String k : xs.keySet()) { + zs.put(k, xs.get(k) - ys.get(k)); + } + return zs; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/Monitor.java b/java-util/src/main/java/io/druid/java/util/metrics/Monitor.java new file mode 100644 index 000000000000..666477b108b6 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/Monitor.java @@ -0,0 +1,33 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import io.druid.java.util.emitter.service.ServiceEmitter; + +/** + */ +public interface Monitor +{ + void start(); + + void stop(); + + boolean monitor(ServiceEmitter emitter); +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/MonitorOfTheMonitors.java b/java-util/src/main/java/io/druid/java/util/metrics/MonitorOfTheMonitors.java new file mode 100644 index 000000000000..edfd94dd32d8 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/MonitorOfTheMonitors.java @@ -0,0 +1,35 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import io.druid.java.util.common.logger.Logger; +import io.druid.java.util.emitter.service.ServiceEmitter; + +public class MonitorOfTheMonitors extends AbstractMonitor +{ + private static final Logger log = new Logger(MonitorOfTheMonitors.class); + + @Override + public boolean doMonitor(ServiceEmitter emitter) + { + log.info("I am watching..."); + return true; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/MonitorScheduler.java b/java-util/src/main/java/io/druid/java/util/metrics/MonitorScheduler.java new file mode 100644 index 000000000000..46f374609f89 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/MonitorScheduler.java @@ -0,0 +1,142 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import com.google.common.collect.Sets; +import io.druid.java.util.common.ISE; +import io.druid.java.util.common.concurrent.ScheduledExecutors; +import io.druid.java.util.common.lifecycle.LifecycleStart; +import io.druid.java.util.common.lifecycle.LifecycleStop; +import io.druid.java.util.emitter.service.ServiceEmitter; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ScheduledExecutorService; + +/** + */ +public class MonitorScheduler +{ + private final MonitorSchedulerConfig config; + private final ScheduledExecutorService exec; + private final ServiceEmitter emitter; + private final Set monitors; + private final Object lock = new Object(); + + private volatile boolean started = false; + + public MonitorScheduler( + MonitorSchedulerConfig config, + ScheduledExecutorService exec, + ServiceEmitter emitter, + List monitors + ) + { + this.config = config; + this.exec = exec; + this.emitter = emitter; + this.monitors = Sets.newHashSet(monitors); + } + + @LifecycleStart + public void start() + { + synchronized (lock) { + if (started) { + return; + } + started = true; + + for (final Monitor monitor : monitors) { + startMonitor(monitor); + } + } + } + + public void addMonitor(final Monitor monitor) + { + synchronized (lock) { + if (!started) { + throw new ISE("addMonitor must be called after start"); + } + if (hasMonitor(monitor)) { + throw new ISE("Monitor already monitoring: %s", monitor); + } + monitors.add(monitor); + startMonitor(monitor); + } + } + + public void removeMonitor(final Monitor monitor) + { + synchronized (lock) { + monitors.remove(monitor); + monitor.stop(); + } + } + + @LifecycleStop + public void stop() + { + synchronized (lock) { + if (!started) { + return; + } + + started = false; + for (Monitor monitor : monitors) { + monitor.stop(); + } + } + } + + private void startMonitor(final Monitor monitor) + { + synchronized (lock) { + monitor.start(); + ScheduledExecutors.scheduleAtFixedRate( + exec, + config.getEmitterPeriod(), + new Callable() + { + @Override + public ScheduledExecutors.Signal call() throws Exception + { + // Run one more time even if the monitor was removed, in case there's some extra data to flush + if (monitor.monitor(emitter) && hasMonitor(monitor)) { + return ScheduledExecutors.Signal.REPEAT; + } else { + removeMonitor(monitor); + return ScheduledExecutors.Signal.STOP; + } + } + } + ); + } + } + + private boolean hasMonitor(final Monitor monitor) + { + synchronized (lock) { + return monitors.contains(monitor); + } + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/MonitorSchedulerConfig.java b/java-util/src/main/java/io/druid/java/util/metrics/MonitorSchedulerConfig.java new file mode 100644 index 000000000000..e1675d16dd20 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/MonitorSchedulerConfig.java @@ -0,0 +1,33 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import org.joda.time.Duration; +import org.skife.config.Config; +import org.skife.config.Default; + +/** + */ +public abstract class MonitorSchedulerConfig +{ + @Config({"io.druid.java.util.metrics.emitter.period", "com.metamx.druid.emitter.period"}) + @Default("PT60s") + public abstract Duration getEmitterPeriod(); +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/MonitorUtils.java b/java-util/src/main/java/io/druid/java/util/metrics/MonitorUtils.java new file mode 100644 index 000000000000..bd48e0460347 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/MonitorUtils.java @@ -0,0 +1,34 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import io.druid.java.util.emitter.service.ServiceMetricEvent; + +import java.util.Map; + +public class MonitorUtils +{ + public static void addDimensionsToBuilder(ServiceMetricEvent.Builder builder, Map dimensions) + { + for (Map.Entry keyValue : dimensions.entrySet()) { + builder.setDimension(keyValue.getKey(), keyValue.getValue()); + } + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/Monitors.java b/java-util/src/main/java/io/druid/java/util/metrics/Monitors.java new file mode 100644 index 000000000000..24f155fff7cb --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/Monitors.java @@ -0,0 +1,94 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import java.util.List; +import java.util.Map; + +public class Monitors +{ + /** + * Creates a JVM monitor, configured with the given dimensions, that gathers all currently available JVM-wide + * monitors. Emitted events have default feed {@link FeedDefiningMonitor#DEFAULT_METRICS_FEED} + * See: {@link Monitors#createCompoundJvmMonitor(Map, String)} + * + * @param dimensions common dimensions to configure the JVM monitor with + * + * @return a universally useful JVM-wide monitor + */ + public static Monitor createCompoundJvmMonitor(Map dimensions) + { + return createCompoundJvmMonitor(dimensions, FeedDefiningMonitor.DEFAULT_METRICS_FEED); + } + + /** + * Creates a JVM monitor, configured with the given dimensions, that gathers all currently available JVM-wide + * monitors: {@link JvmMonitor}, {@link JvmCpuMonitor} and {@link JvmThreadsMonitor} (this list may + * change in any future release of this library, including a minor release). + * + * @param dimensions common dimensions to configure the JVM monitor with + * @param feed feed for all emitted events + * + * @return a universally useful JVM-wide monitor + */ + public static Monitor createCompoundJvmMonitor(Map dimensions, String feed) + { + // This list doesn't include SysMonitor because it should probably be run only in one JVM, if several JVMs are + // running on the same instance, so most of the time SysMonitor should be configured/set up differently than + // "simple" JVM monitors, created below. + return and(// Could equally be or(), because all member monitors always return true from their monitor() methods. + new JvmMonitor(dimensions, feed), + new JvmCpuMonitor(dimensions, feed), + new JvmThreadsMonitor(dimensions, feed) + ); + } + + public static Monitor and(Monitor... monitors) + { + return new CompoundMonitor(monitors) + { + @Override + public boolean shouldReschedule(List reschedules) + { + boolean b = true; + for (boolean reschedule : reschedules) { + b = b && reschedule; + } + return b; + } + }; + } + + public static Monitor or(Monitor... monitors) + { + return new CompoundMonitor(monitors) + { + @Override + public boolean shouldReschedule(List reschedules) + { + boolean b = false; + for (boolean reschedule : reschedules) { + b = b || reschedule; + } + return b; + } + }; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/ParametrizedUriEmitterMonitor.java b/java-util/src/main/java/io/druid/java/util/metrics/ParametrizedUriEmitterMonitor.java new file mode 100644 index 000000000000..219e82105fc8 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/ParametrizedUriEmitterMonitor.java @@ -0,0 +1,75 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import com.google.common.collect.ImmutableMap; +import io.druid.java.util.emitter.core.ParametrizedUriEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; + +import java.net.URI; +import java.util.HashMap; +import java.util.Map; + +public class ParametrizedUriEmitterMonitor extends FeedDefiningMonitor +{ + private final ParametrizedUriEmitter parametrizedUriEmitter; + private final Map monitors = new HashMap<>(); + + public ParametrizedUriEmitterMonitor(String feed, ParametrizedUriEmitter parametrizedUriEmitter) + { + super(feed); + this.parametrizedUriEmitter = parametrizedUriEmitter; + } + + private void updateMonitors() + { + parametrizedUriEmitter.forEachEmitter( + (uri, emitter) -> { + monitors.computeIfAbsent( + uri, + u -> { + HttpPostEmitterMonitor monitor = new HttpPostEmitterMonitor( + feed, + emitter, + ImmutableMap.of("uri", uri.toString()) + ); + monitor.start(); + return monitor; + } + ); + } + ); + } + + @Override + public void stop() + { + monitors.values().forEach(AbstractMonitor::stop); + super.stop(); + } + + @Override + public boolean doMonitor(ServiceEmitter emitter) + { + updateMonitors(); + monitors.values().forEach(m -> m.doMonitor(emitter)); + return true; + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/PidDiscoverer.java b/java-util/src/main/java/io/druid/java/util/metrics/PidDiscoverer.java new file mode 100644 index 000000000000..3857027395f2 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/PidDiscoverer.java @@ -0,0 +1,25 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +public interface PidDiscoverer +{ + long getPid(); +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/SigarPidDiscoverer.java b/java-util/src/main/java/io/druid/java/util/metrics/SigarPidDiscoverer.java new file mode 100644 index 000000000000..2ae4babe3a32 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/SigarPidDiscoverer.java @@ -0,0 +1,44 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +public class SigarPidDiscoverer implements PidDiscoverer +{ + private static final SigarPidDiscoverer INSTANCE = new SigarPidDiscoverer(); + + public static SigarPidDiscoverer instance() + { + return INSTANCE; + } + + /** + * use {SigarPidDiscoverer.instance()} + */ + private SigarPidDiscoverer() + { + + } + + @Override + public long getPid() + { + return SigarUtil.getCurrentProcessId(); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/SigarUtil.java b/java-util/src/main/java/io/druid/java/util/metrics/SigarUtil.java new file mode 100644 index 000000000000..824826f8df61 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/SigarUtil.java @@ -0,0 +1,85 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import com.google.common.base.Throwables; +import io.druid.java.util.common.StreamUtils; +import io.druid.java.util.common.logger.Logger; +import org.hyperic.jni.ArchLoaderException; +import org.hyperic.jni.ArchNotSupportedException; +import org.hyperic.sigar.Sigar; +import org.hyperic.sigar.SigarLoader; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; + +public class SigarUtil +{ + private static final Logger log = new Logger(SigarUtil.class); + + // Note: this is required to load the sigar native lib. + static { + SigarLoader loader = new SigarLoader(Sigar.class); + try { + String libName = loader.getLibraryName(); + + final URL url = SysMonitor.class.getResource("/" + libName); + if (url != null) { + final File tmpDir = Files.createTempDirectory("sigar").toFile(); + // As per java.io.DeleteOnExitHook.runHooks() deletion order is reversed from registration order + tmpDir.deleteOnExit(); + final File nativeLibTmpFile = new File(tmpDir, libName); + nativeLibTmpFile.deleteOnExit(); + StreamUtils.copyToFileAndClose(url.openStream(), nativeLibTmpFile); + log.info("Loading sigar native lib at tmpPath[%s]", nativeLibTmpFile); + loader.load(nativeLibTmpFile.getParent()); + } else { + log.info("No native libs found in jar, letting the normal load mechanisms figger it out."); + } + } + catch (ArchNotSupportedException | ArchLoaderException | IOException e) { + throw Throwables.propagate(e); + } + } + + public static Sigar getSigar() + { + return new Sigar(); + } + + /** + * CurrentProcessIdHolder class is initialized after SigarUtil, that guarantees that new Sigar() is executed after + * static block (which loads the library) of SigarUtil is executed. This is anyway guaranteed by JLS if the static + * field goes below the static block in textual order, but fragile e. g. if someone applies automatic reformatting and + * the static field is moved above the static block. + */ + private static class CurrentProcessIdHolder + { + private static final long currentProcessId = new Sigar().getPid(); + } + + public static long getCurrentProcessId() + { + return CurrentProcessIdHolder.currentProcessId; + } + +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/SysMonitor.java b/java-util/src/main/java/io/druid/java/util/metrics/SysMonitor.java new file mode 100644 index 000000000000..21ac987e01cb --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/SysMonitor.java @@ -0,0 +1,584 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.druid.java.util.common.logger.Logger; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; +import org.hyperic.sigar.Cpu; +import org.hyperic.sigar.DirUsage; +import org.hyperic.sigar.DiskUsage; +import org.hyperic.sigar.FileSystem; +import org.hyperic.sigar.FileSystemUsage; +import org.hyperic.sigar.Mem; +import org.hyperic.sigar.NetInterfaceConfig; +import org.hyperic.sigar.NetInterfaceStat; +import org.hyperic.sigar.NetStat; +import org.hyperic.sigar.Sigar; +import org.hyperic.sigar.SigarException; +import org.hyperic.sigar.Swap; +import org.hyperic.sigar.Tcp; +import org.hyperic.sigar.Uptime; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public class SysMonitor extends FeedDefiningMonitor +{ + private static final Logger log = new Logger(SysMonitor.class); + + private final Sigar sigar = SigarUtil.getSigar(); + + private final List fsTypeWhitelist = ImmutableList.of("local"); + private final List netAddressBlacklist = ImmutableList.of("0.0.0.0", "127.0.0.1"); + + private final List statsList; + + private Map dimensions; + + public SysMonitor() + { + this(ImmutableMap.of()); + } + + public SysMonitor(Map dimensions) + { + this(dimensions, DEFAULT_METRICS_FEED); + } + + public SysMonitor(Map dimensions, String feed) + { + super(feed); + Preconditions.checkNotNull(dimensions); + this.dimensions = ImmutableMap.copyOf(dimensions); + + sigar.enableLogging(true); + + this.statsList = new ArrayList(); + this.statsList.addAll( + Arrays.asList( + new MemStats(), + new FsStats(), + new DiskStats(), + new NetStats(), + new CpuStats(), + new SwapStats(), + new SysStats(), + new TcpStats() + ) + ); + } + + public void addDirectoriesToMonitor(String[] dirList) + { + for (int i = 0; i < dirList.length; i++) { + dirList[i] = dirList[i].trim(); + } + statsList.add(new DirStats(dirList)); + } + + @Override + public boolean doMonitor(ServiceEmitter emitter) + { + for (Stats stats : statsList) { + stats.emit(emitter); + } + return true; + } + + private interface Stats + { + void emit(ServiceEmitter emitter); + } + + private class MemStats implements Stats + { + @Override + public void emit(ServiceEmitter emitter) + { + Mem mem = null; + try { + mem = sigar.getMem(); + } + catch (SigarException e) { + log.error(e, "Failed to get Mem"); + } + if (mem != null) { + final Map stats = ImmutableMap.of( + "sys/mem/max", mem.getTotal(), + "sys/mem/used", mem.getUsed(), + "sys/mem/actual/used", mem.getActualUsed(), + "sys/mem/actual/free", mem.getActualFree() + ); + final ServiceMetricEvent.Builder builder = builder(); + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + for (Map.Entry entry : stats.entrySet()) { + emitter.emit(builder.build(entry.getKey(), entry.getValue())); + } + } + } + } + + /** + * Gets the swap stats from sigar and emits the periodic pages in & pages out of memory + * along with the max swap and free swap memory. + */ + private class SwapStats implements Stats + { + private long prevPageIn = 0, prevPageOut = 0; + + private SwapStats() + { + try { + Swap swap = sigar.getSwap(); + this.prevPageIn = swap.getPageIn(); + this.prevPageOut = swap.getPageOut(); + } + catch (SigarException e) { + log.error(e, "Failed to get Swap"); + } + } + + @Override + public void emit(ServiceEmitter emitter) + { + Swap swap = null; + try { + swap = sigar.getSwap(); + } + catch (SigarException e) { + log.error(e, "Failed to get Swap"); + } + if (swap != null) { + long currPageIn = swap.getPageIn(); + long currPageOut = swap.getPageOut(); + + final Map stats = ImmutableMap.of( + "sys/swap/pageIn", (currPageIn - prevPageIn), + "sys/swap/pageOut", (currPageOut - prevPageOut), + "sys/swap/max", swap.getTotal(), + "sys/swap/free", swap.getFree() + ); + + final ServiceMetricEvent.Builder builder = builder(); + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + for (Map.Entry entry : stats.entrySet()) { + emitter.emit(builder.build(entry.getKey(), entry.getValue())); + } + + this.prevPageIn = currPageIn; + this.prevPageOut = currPageOut; + } + } + } + + /** + * Gets the disk usage of a particular directory. + */ + private class DirStats implements Stats + { + private final String[] dirList; + + private DirStats(String[] dirList) + { + this.dirList = dirList; + } + + @Override + public void emit(ServiceEmitter emitter) + { + for (String dir : dirList) { + DirUsage du = null; + try { + du = sigar.getDirUsage(dir); + } + catch (SigarException e) { + log.error("Failed to get DiskUsage for [%s] due to [%s]", dir, e.getMessage()); + } + if (du != null) { + final Map stats = ImmutableMap.of( + "sys/storage/used", du.getDiskUsage() + ); + final ServiceMetricEvent.Builder builder = builder() + .setDimension("fsDirName", dir); // fsDirName because FsStats uses fsDirName + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + for (Map.Entry entry : stats.entrySet()) { + emitter.emit(builder.build(entry.getKey(), entry.getValue())); + } + } + } + } + } + + private class FsStats implements Stats + { + @Override + public void emit(ServiceEmitter emitter) + { + FileSystem[] fss = null; + try { + fss = sigar.getFileSystemList(); + } + catch (SigarException e) { + log.error(e, "Failed to get FileSystem list"); + } + if (fss != null) { + log.debug("Found FileSystem list: [%s]", Joiner.on(", ").join(fss)); + for (FileSystem fs : fss) { + final String name = fs.getDirName(); // (fs.getDevName() does something wonky here!) + if (fsTypeWhitelist.contains(fs.getTypeName())) { + FileSystemUsage fsu = null; + try { + fsu = sigar.getFileSystemUsage(name); + } + catch (SigarException e) { + log.error(e, "Failed to get FileSystemUsage[%s]", name); + } + if (fsu != null) { + final Map stats = ImmutableMap.builder() + .put("sys/fs/max", fsu.getTotal() * 1024) + .put("sys/fs/used", fsu.getUsed() * 1024) + .put("sys/fs/files/count", fsu.getFiles()) + .put("sys/fs/files/free", fsu.getFreeFiles()) + .build(); + final ServiceMetricEvent.Builder builder = builder() + .setDimension("fsDevName", fs.getDevName()) + .setDimension("fsDirName", fs.getDirName()) + .setDimension("fsTypeName", fs.getTypeName()) + .setDimension("fsSysTypeName", fs.getSysTypeName()) + .setDimension("fsOptions", fs.getOptions().split(",")); + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + for (Map.Entry entry : stats.entrySet()) { + emitter.emit(builder.build(entry.getKey(), entry.getValue())); + } + } + } else { + log.debug("Not monitoring fs stats for name[%s] with typeName[%s]", name, fs.getTypeName()); + } + } + } + } + } + + private class DiskStats implements Stats + { + private final KeyedDiff diff = new KeyedDiff(); + + @Override + public void emit(ServiceEmitter emitter) + { + FileSystem[] fss = null; + try { + fss = sigar.getFileSystemList(); + } + catch (SigarException e) { + log.error(e, "Failed to get FileSystem list"); + } + if (fss != null) { + log.debug("Found FileSystem list: [%s]", Joiner.on(", ").join(fss)); + for (FileSystem fs : fss) { + // fs.getDevName() appears to give the same results here, but on some nodes results for one disc were substituted by another + // LOG: Sigar - /proc/diskstats /dev/xvdj -> /dev/xvdb [202,16] + final String name = fs.getDirName(); + if (fsTypeWhitelist.contains(fs.getTypeName())) { + DiskUsage du = null; + try { + du = sigar.getDiskUsage(name); + } + catch (SigarException e) { + log.error(e, "Failed to get DiskUsage[%s]", name); + } + if (du != null) { + final Map stats = diff.to( + name, ImmutableMap.builder() + .put("sys/disk/read/size", du.getReadBytes()) + .put("sys/disk/read/count", du.getReads()) + .put("sys/disk/write/size", du.getWriteBytes()) + .put("sys/disk/write/count", du.getWrites()) + .put("sys/disk/queue", Double.valueOf(du.getQueue()).longValue()) + .put("sys/disk/serviceTime", Double.valueOf(du.getServiceTime()).longValue()) + .build() + ); + log.debug("DiskUsage diff for [%s]: %s", name, stats); + if (stats != null) { + final ServiceMetricEvent.Builder builder = builder() + .setDimension("fsDevName", fs.getDevName()) + .setDimension("fsDirName", fs.getDirName()) + .setDimension("fsTypeName", fs.getTypeName()) + .setDimension("fsSysTypeName", fs.getSysTypeName()) + .setDimension("fsOptions", fs.getOptions().split(",")); + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + for (Map.Entry entry : stats.entrySet()) { + emitter.emit(builder.build(entry.getKey(), entry.getValue())); + } + } + } + } else { + log.debug("Not monitoring disk stats for name[%s] with typeName[%s]", name, fs.getTypeName()); + } + } + } + } + } + + private class NetStats implements Stats + { + private final KeyedDiff diff = new KeyedDiff(); + + @Override + public void emit(ServiceEmitter emitter) + { + String[] ifaces = null; + try { + ifaces = sigar.getNetInterfaceList(); + } + catch (SigarException e) { + log.error(e, "Failed to get NetInterface list"); + } + if (ifaces != null) { + log.debug("Found NetInterface list: [%s]", Joiner.on(", ").join(ifaces)); + for (String name : ifaces) { + NetInterfaceConfig netconf = null; + try { + netconf = sigar.getNetInterfaceConfig(name); + } + catch (SigarException e) { + log.error(e, "Failed to get NetInterfaceConfig[%s]", name); + } + if (netconf != null) { + if (!(netAddressBlacklist.contains(netconf.getAddress()))) { + NetInterfaceStat netstat = null; + try { + netstat = sigar.getNetInterfaceStat(name); + } + catch (SigarException e) { + log.error(e, "Failed to get NetInterfaceStat[%s]", name); + } + if (netstat != null) { + final Map stats = diff.to( + name, ImmutableMap.builder() + .put("sys/net/read/size", netstat.getRxBytes()) + .put("sys/net/read/packets", netstat.getRxPackets()) + .put("sys/net/read/errors", netstat.getRxErrors()) + .put("sys/net/read/dropped", netstat.getRxDropped()) + .put("sys/net/read/overruns", netstat.getRxOverruns()) + .put("sys/net/read/frame", netstat.getRxFrame()) + .put("sys/net/write/size", netstat.getTxBytes()) + .put("sys/net/write/packets", netstat.getTxPackets()) + .put("sys/net/write/errors", netstat.getTxErrors()) + .put("sys/net/write/dropped", netstat.getTxDropped()) + .put("sys/net/write/collisions", netstat.getTxCollisions()) + .put("sys/net/write/overruns", netstat.getTxOverruns()) + .build() + ); + if (stats != null) { + final ServiceMetricEvent.Builder builder = builder() + .setDimension("netName", netconf.getName()) + .setDimension("netAddress", netconf.getAddress()) + .setDimension("netHwaddr", netconf.getHwaddr()); + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + for (Map.Entry entry : stats.entrySet()) { + emitter.emit(builder.build(entry.getKey(), entry.getValue())); + } + } + } + } else { + log.debug("Not monitoring net stats for name[%s] with address[%s]", name, netconf.getAddress()); + } + } + } + } + } + } + + private class CpuStats implements Stats + { + private final KeyedDiff diff = new KeyedDiff(); + + @Override + public void emit(ServiceEmitter emitter) + { + Cpu[] cpus = null; + try { + cpus = sigar.getCpuList(); + } + catch (SigarException e) { + log.error(e, "Failed to get Cpu list"); + } + if (cpus != null) { + log.debug("Found Cpu list: [%s]", Joiner.on(", ").join(cpus)); + for (int i = 0; i < cpus.length; ++i) { + final Cpu cpu = cpus[i]; + final String name = Integer.toString(i); + final Map stats = diff.to( + name, ImmutableMap.builder() + .put("user", cpu.getUser()) // user = Δuser / Δtotal + .put("sys", cpu.getSys()) // sys = Δsys / Δtotal + .put("nice", cpu.getNice()) // nice = Δnice / Δtotal + .put("wait", cpu.getWait()) // wait = Δwait / Δtotal + .put("irq", cpu.getIrq()) // irq = Δirq / Δtotal + .put("softIrq", cpu.getSoftIrq()) // softIrq = ΔsoftIrq / Δtotal + .put("stolen", cpu.getStolen()) // stolen = Δstolen / Δtotal + .put("_total", cpu.getTotal()) // (not reported) + .build() + ); + if (stats != null) { + final long total = stats.remove("_total"); + for (Map.Entry entry : stats.entrySet()) { + final ServiceMetricEvent.Builder builder = builder() + .setDimension("cpuName", name) + .setDimension("cpuTime", entry.getKey()); + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + emitter.emit(builder.build("sys/cpu", entry.getValue() * 100 / total)); // [0,100] + } + } + } + } + } + } + + private class SysStats implements Stats + { + @Override + public void emit(ServiceEmitter emitter) + { + final ServiceMetricEvent.Builder builder = builder(); + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + + Uptime uptime = null; + try { + uptime = sigar.getUptime(); + } + catch (SigarException e) { + log.error(e, "Failed to get Uptime"); + } + + double[] la = null; + try { + la = sigar.getLoadAverage(); + } + catch (SigarException e) { + log.error(e, "Failed to get Load Average"); + } + + if (uptime != null) { + final Map stats = ImmutableMap.of( + "sys/uptime", Double.valueOf(uptime.getUptime()).longValue() + ); + for (Map.Entry entry : stats.entrySet()) { + emitter.emit(builder.build(entry.getKey(), entry.getValue())); + } + } + + if (la != null) { + final Map stats = ImmutableMap.of( + "sys/la/1", la[0], + "sys/la/5", la[1], + "sys/la/15", la[2] + ); + for (Map.Entry entry : stats.entrySet()) { + emitter.emit(builder.build(entry.getKey(), entry.getValue())); + } + } + } + } + + private class TcpStats implements Stats + { + private final KeyedDiff diff = new KeyedDiff(); + + @Override + public void emit(ServiceEmitter emitter) + { + final ServiceMetricEvent.Builder builder = builder(); + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + + Tcp tcp = null; + try { + tcp = sigar.getTcp(); + } + catch (SigarException e) { + log.error(e, "Failed to get Tcp"); + } + + if (tcp != null) { + final Map stats = diff.to( + "tcp", ImmutableMap.builder() + .put("sys/tcp/activeOpens", tcp.getActiveOpens()) + .put("sys/tcp/passiveOpens", tcp.getPassiveOpens()) + .put("sys/tcp/attemptFails", tcp.getAttemptFails()) + .put("sys/tcp/estabResets", tcp.getEstabResets()) + .put("sys/tcp/in/segs", tcp.getInSegs()) + .put("sys/tcp/in/errs", tcp.getInErrs()) + .put("sys/tcp/out/segs", tcp.getOutSegs()) + .put("sys/tcp/out/rsts", tcp.getOutRsts()) + .put("sys/tcp/retrans/segs", tcp.getRetransSegs()) + .build() + ); + if (stats != null) { + for (Map.Entry entry : stats.entrySet()) { + emitter.emit(builder.build(entry.getKey(), entry.getValue())); + } + } + } + + NetStat netStat = null; + try { + netStat = sigar.getNetStat(); + } + catch (SigarException e) { + log.error(e, "Failed to get NetStat"); + } + if (netStat != null) { + final Map stats = ImmutableMap.builder() + .put("sys/net/inbound", (long) netStat.getAllInboundTotal()) + .put("sys/net/outbound", (long) netStat.getAllOutboundTotal()) + .put("sys/tcp/inbound", (long) netStat.getTcpInboundTotal()) + .put("sys/tcp/outbound", (long) netStat.getTcpOutboundTotal()) + .put( + "sys/tcp/state/established", + (long) netStat.getTcpEstablished() + ) + .put("sys/tcp/state/synSent", (long) netStat.getTcpSynSent()) + .put("sys/tcp/state/synRecv", (long) netStat.getTcpSynRecv()) + .put("sys/tcp/state/finWait1", (long) netStat.getTcpFinWait1()) + .put("sys/tcp/state/finWait2", (long) netStat.getTcpFinWait2()) + .put("sys/tcp/state/timeWait", (long) netStat.getTcpTimeWait()) + .put("sys/tcp/state/close", (long) netStat.getTcpClose()) + .put("sys/tcp/state/closeWait", (long) netStat.getTcpCloseWait()) + .put("sys/tcp/state/lastAck", (long) netStat.getTcpLastAck()) + .put("sys/tcp/state/listen", (long) netStat.getTcpListen()) + .put("sys/tcp/state/closing", (long) netStat.getTcpClosing()) + .put("sys/tcp/state/idle", (long) netStat.getTcpIdle()) + .put("sys/tcp/state/bound", (long) netStat.getTcpBound()) + .build(); + for (Map.Entry entry : stats.entrySet()) { + emitter.emit(builder.build(entry.getKey(), entry.getValue())); + } + } + } + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/cgroups/CgroupDiscoverer.java b/java-util/src/main/java/io/druid/java/util/metrics/cgroups/CgroupDiscoverer.java new file mode 100644 index 000000000000..83308947b34d --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/cgroups/CgroupDiscoverer.java @@ -0,0 +1,32 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics.cgroups; + +import java.nio.file.Path; + +public interface CgroupDiscoverer +{ + /** + * Returns a path for a specific cgroup. This path should contain the interesting cgroup files without further traversing needed. + * @param cgroup The cgroup + * @return The path that contains that cgroup's interesting bits. + */ + Path discover(String cgroup); +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/cgroups/CpuAcct.java b/java-util/src/main/java/io/druid/java/util/metrics/cgroups/CpuAcct.java new file mode 100644 index 000000000000..ad0ceb53525f --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/cgroups/CpuAcct.java @@ -0,0 +1,158 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics.cgroups; + +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import io.druid.java.util.common.RE; +import io.druid.java.util.common.logger.Logger; +import io.druid.java.util.metrics.CgroupUtil; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.List; +import java.util.stream.LongStream; + +public class CpuAcct +{ + private static final Logger LOG = new Logger(CpuAcct.class); + private static final String CGROUP = "cpuacct"; + private static final String CGROUP_ACCT_FILE = "cpuacct.usage_all"; + + // Private because it requires a specific format and cant' take a generic list of strings + private static CpuAcctMetric parse(final List lines) + { + // File has a header. We skip it + // See src/test/resources/cpuacct.usage_all for an example + final int ncpus = lines.size() - 1; + final long[] usrTime = new long[ncpus]; + final long[] sysTime = new long[ncpus]; + for (int i = 1; i < lines.size(); i++) { + final String[] splits = lines.get(i).split(CgroupUtil.SPACE_MATCH, 3); + if (splits.length != 3) { + throw new RE("Error parsing [%s]", lines.get(i)); + } + final int cpuNum = Integer.parseInt(splits[0]); + usrTime[cpuNum] = Long.parseLong(splits[1]); + sysTime[cpuNum] = Long.parseLong(splits[2]); + } + return new CpuAcctMetric(usrTime, sysTime); + } + + private final CgroupDiscoverer cgroupDiscoverer; + + public CpuAcct(CgroupDiscoverer cgroupDiscoverer) + { + this.cgroupDiscoverer = cgroupDiscoverer; + } + + /** + * Take a snapshot of the existing data. + * + * @return A snapshot with the data populated or a snapshot with zero-length arrays for data. + */ + public CpuAcctMetric snapshot() + { + final File cpuacct; + try { + cpuacct = new File( + cgroupDiscoverer.discover(CGROUP).toFile(), + CGROUP_ACCT_FILE + ); + } + catch (RuntimeException re) { + LOG.error(re, "Unable to fetch snapshot"); + return new CpuAcctMetric(new long[0], new long[0]); + } + try { + return parse(Files.readAllLines(cpuacct.toPath(), Charsets.UTF_8)); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static class CpuAcctMetric + { + private final long[] usrTimes; + private final long[] sysTimes; + + CpuAcctMetric(long[] usrTimes, long[] sysTimes) + { + Preconditions.checkArgument(usrTimes.length == sysTimes.length, "Lengths must match"); + this.usrTimes = usrTimes; + this.sysTimes = sysTimes; + } + + public final int cpuCount() + { + return usrTimes.length; + } + + public final long[] sysTimes() + { + return sysTimes; + } + + public final long[] usrTimes() + { + return usrTimes; + } + + public final long usrTime(int cpuNum) + { + return usrTimes[cpuNum]; + } + + public final long sysTime(int cpu_Num) + { + return sysTimes[cpu_Num]; + } + + public final long usrTime() + { + return LongStream.of(usrTimes).sum(); + } + + public final long sysTime() + { + return LongStream.of(sysTimes).sum(); + } + + public final long time() + { + return usrTime() + sysTime(); + } + + public final CpuAcctMetric cumulativeSince(CpuAcctMetric other) + { + final int cpuCount = cpuCount(); + Preconditions.checkArgument(cpuCount == other.cpuCount(), "Cpu count missmatch"); + final long[] sysTimes = new long[cpuCount]; + final long[] usrTimes = new long[cpuCount]; + for (int i = 0; i < cpuCount; i++) { + sysTimes[i] = this.sysTimes[i] - other.sysTimes[i]; + usrTimes[i] = this.usrTimes[i] - other.usrTimes[i]; + } + return new CpuAcctMetric(usrTimes, sysTimes); + } + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/cgroups/ProcCgroupDiscoverer.java b/java-util/src/main/java/io/druid/java/util/metrics/cgroups/ProcCgroupDiscoverer.java new file mode 100644 index 000000000000..ca595cb3aa8e --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/cgroups/ProcCgroupDiscoverer.java @@ -0,0 +1,174 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics.cgroups; + +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import com.google.common.io.Files; +import io.druid.java.util.common.RE; +import io.druid.java.util.metrics.CgroupUtil; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; + +public class ProcCgroupDiscoverer implements CgroupDiscoverer +{ + private static final String CGROUP_TYPE = "cgroup"; + + private final File procDir; + + /** + * Create a proc discovery mechanism based on a `/proc` directory. + * + * @param procDir The directory under proc. This is usually `/proc/self` or `/proc/#pid` + */ + public ProcCgroupDiscoverer(Path procDir) + { + this.procDir = Preconditions.checkNotNull(procDir, "procDir").toFile(); + Preconditions.checkArgument(this.procDir.isDirectory(), "Not a directory: [%s]", procDir); + } + + @Override + public Path discover(final String cgroup) + { + Preconditions.checkNotNull(cgroup, "cgroup required"); + final File procMounts = new File(procDir, "mounts"); + final File pidCgroups = new File(procDir, "cgroup"); + final PidCgroupEntry pidCgroupsEntry = getCgroupEntry(pidCgroups, cgroup); + final ProcMountsEntry procMountsEntry = getMountEntry(procMounts, cgroup); + final File cgroupDir = new File( + procMountsEntry.path.toString(), + pidCgroupsEntry.path.toString() + ); + if (cgroupDir.exists() && cgroupDir.isDirectory()) { + return cgroupDir.toPath(); + } + throw new RE("Invalid cgroup directory [%s]", cgroupDir); + } + + private PidCgroupEntry getCgroupEntry(final File procCgroup, final String cgroup) + { + final List lines; + try { + lines = Files.readLines(procCgroup, Charsets.UTF_8); + } + catch (IOException e) { + throw new RuntimeException(e); + } + for (final String line : lines) { + if (line.startsWith("#")) { + continue; + } + final PidCgroupEntry entry = PidCgroupEntry.parse(line); + if (entry.controllers.contains(cgroup)) { + return entry; + } + } + throw new RE("Hierarchy for [%s] not found", cgroup); + } + + private ProcMountsEntry getMountEntry(final File procMounts, final String cgroup) + { + final List lines; + try { + lines = Files.readLines(procMounts, Charsets.UTF_8); + } + catch (IOException e) { + throw new RuntimeException(e); + } + + for (final String line : lines) { + final ProcMountsEntry entry = ProcMountsEntry.parse(line); + if (CGROUP_TYPE.equals(entry.type) && entry.options.contains(cgroup)) { + return entry; + } + } + throw new RE("Cgroup [%s] not found", cgroup); + } + + /** + * Doesn't use the last two mount entries for priority/boot stuff + */ + static class ProcMountsEntry + { + // Example: cgroup /sys/fs/cgroup/cpu,cpuacct cgroup rw,nosuid,nodev,noexec,relatime,cpu,cpuacct 0 0 + static ProcMountsEntry parse(String entry) + { + final String[] splits = entry.split(CgroupUtil.SPACE_MATCH, 6); + Preconditions.checkArgument(splits.length == 6, "Invalid entry: [%s]", entry); + return new ProcMountsEntry( + splits[0], + Paths.get(splits[1]), + splits[2], + ImmutableSet.copyOf(splits[3].split(CgroupUtil.COMMA_MATCH)) + ); + } + + final String dev; + final Path path; + final String type; + final Set options; + + ProcMountsEntry(String dev, Path path, String type, Collection options) + { + this.dev = dev; + this.path = path; + this.type = type; + this.options = ImmutableSet.copyOf(options); + } + } + + // See man CGROUPS(7) + static class PidCgroupEntry + { + static PidCgroupEntry parse(String entry) + { + // For example, entries with a port number will have an extra `:` in it somewhere, or ipv6 addresses. + final String[] parts = entry.split(Pattern.quote(":"), 3); + if (parts.length != 3) { + throw new RE("Bad entry [%s]", entry); + } + final int heirarchyId = Integer.parseInt(parts[0]); + final Set controllers = new HashSet<>(Arrays.asList(parts[1].split(Pattern.quote(",")))); + final Path path = Paths.get(parts[2]); + return new PidCgroupEntry(heirarchyId, controllers, path); + } + + final int heirarchyId; + final Set controllers; + final Path path; + + private PidCgroupEntry(int heirarchyId, Set controllers, Path path) + { + this.heirarchyId = heirarchyId; + this.controllers = controllers; + this.path = path; + } + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/cgroups/ProcPidCgroupDiscoverer.java b/java-util/src/main/java/io/druid/java/util/metrics/cgroups/ProcPidCgroupDiscoverer.java new file mode 100644 index 000000000000..23feb3d2466f --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/cgroups/ProcPidCgroupDiscoverer.java @@ -0,0 +1,41 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics.cgroups; + +import io.druid.java.util.metrics.PidDiscoverer; + +import java.nio.file.Path; +import java.nio.file.Paths; + +public class ProcPidCgroupDiscoverer implements CgroupDiscoverer +{ + private final ProcCgroupDiscoverer delegate; + + public ProcPidCgroupDiscoverer(PidDiscoverer pidDiscoverer) + { + delegate = new ProcCgroupDiscoverer(Paths.get("/proc", Long.toString(pidDiscoverer.getPid()))); + } + + @Override + public Path discover(String cgroup) + { + return delegate.discover(cgroup); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/metrics/cgroups/ProcSelfCgroupDiscoverer.java b/java-util/src/main/java/io/druid/java/util/metrics/cgroups/ProcSelfCgroupDiscoverer.java new file mode 100644 index 000000000000..b695dbba7ce8 --- /dev/null +++ b/java-util/src/main/java/io/druid/java/util/metrics/cgroups/ProcSelfCgroupDiscoverer.java @@ -0,0 +1,39 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics.cgroups; + +import java.nio.file.Path; +import java.nio.file.Paths; + +public class ProcSelfCgroupDiscoverer implements CgroupDiscoverer +{ + private final ProcCgroupDiscoverer delegate; + + public ProcSelfCgroupDiscoverer() + { + delegate = new ProcCgroupDiscoverer(Paths.get("/proc/self")); + } + + @Override + public Path discover(String cgroup) + { + return delegate.discover(cgroup); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/ComposingEmitterTest.java b/java-util/src/test/java/io/druid/java/util/emitter/core/ComposingEmitterTest.java new file mode 100644 index 000000000000..0b9a3fe22d9f --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/ComposingEmitterTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.google.common.collect.ImmutableList; +import org.easymock.EasyMock; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; + +public class ComposingEmitterTest +{ + + private List childEmitters; + private ComposingEmitter composingEmitter; + + @Before + public void setup() + { + this.childEmitters = ImmutableList.of( + EasyMock.createMock(Emitter.class), + EasyMock.createMock(Emitter.class) + ); + this.composingEmitter = new ComposingEmitter(childEmitters); + } + + @Test + public void testStart() + { + for (Emitter emitter : childEmitters) { + emitter.start(); + EasyMock.replay(emitter); + } + + composingEmitter.start(); + } + + @Test + public void testEmit() + { + Event e = EasyMock.createMock(Event.class); + + for (Emitter emitter : childEmitters) { + emitter.emit(e); + EasyMock.replay(emitter); + } + + composingEmitter.emit(e); + } + + @Test + public void testFlush() throws IOException + { + for (Emitter emitter : childEmitters) { + emitter.flush(); + EasyMock.replay(emitter); + } + + composingEmitter.flush(); + } + + @Test + public void testClose() throws IOException + { + for (Emitter emitter : childEmitters) { + emitter.close(); + EasyMock.replay(emitter); + } + + composingEmitter.close(); + } + + @After + public void tearDown() + { + for (Emitter emitter : childEmitters) { + EasyMock.verify(emitter); + } + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/CustomEmitterFactoryTest.java b/java-util/src/test/java/io/druid/java/util/emitter/core/CustomEmitterFactoryTest.java new file mode 100644 index 000000000000..34eb20d64967 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/CustomEmitterFactoryTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.java.util.common.lifecycle.Lifecycle; +import io.druid.java.util.emitter.factory.EmitterFactory; +import org.asynchttpclient.AsyncHttpClient; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Properties; + +public class CustomEmitterFactoryTest +{ + @JsonTypeName("test") + public static class TestEmitterConfig implements EmitterFactory + { + @JsonProperty + private String stringProperty; + @JsonProperty + private int intProperty; + + @Override + public Emitter makeEmitter(ObjectMapper objectMapper, AsyncHttpClient httpClient, Lifecycle lifecycle) + { + return new StubEmitter(stringProperty, intProperty); + } + } + + public static class StubEmitter implements Emitter + { + private String stringProperty; + private int intProperty; + + public StubEmitter(String stringProperty, int intProperty) + { + this.stringProperty = stringProperty; + this.intProperty = intProperty; + } + + public String getStringProperty() + { + return stringProperty; + } + + public int getIntProperty() + { + return intProperty; + } + + @Override + public void start() {} + + @Override + public void emit(Event event) {} + + @Override + public void flush() throws IOException {} + + @Override + public void close() throws IOException {} + } + + @Test + public void testCustomEmitter() + { + final Properties props = new Properties(); + props.put("io.druid.java.util.emitter.stringProperty", "http://example.com/"); + props.put("io.druid.java.util.emitter.intProperty", "1"); + props.put("io.druid.java.util.emitter.type", "test"); + + final ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerSubtypes(TestEmitterConfig.class); + final Lifecycle lifecycle = new Lifecycle(); + final Emitter emitter = Emitters.create(props, null, objectMapper, lifecycle); + + Assert.assertTrue("created emitter should be of class StubEmitter", emitter instanceof StubEmitter); + StubEmitter stubEmitter = (StubEmitter) emitter; + Assert.assertEquals("http://example.com/", stubEmitter.getStringProperty()); + Assert.assertEquals(1, stubEmitter.getIntProperty()); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/EmitterTest.java b/java-util/src/test/java/io/druid/java/util/emitter/core/EmitterTest.java new file mode 100644 index 000000000000..b4eca7ab8279 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/EmitterTest.java @@ -0,0 +1,591 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; +import com.google.common.io.BaseEncoding; +import io.druid.java.util.common.CompressionUtils; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.lifecycle.Lifecycle; +import io.druid.java.util.emitter.service.UnitEvent; +import io.netty.buffer.Unpooled; +import io.netty.handler.codec.http.DefaultHttpResponse; +import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.codec.http.HttpVersion; +import org.asynchttpclient.DefaultAsyncHttpClientConfig; +import org.asynchttpclient.ListenableFuture; +import org.asynchttpclient.Request; +import org.asynchttpclient.Response; +import org.asynchttpclient.netty.EagerResponseBodyPart; +import org.asynchttpclient.netty.NettyResponseStatus; +import org.asynchttpclient.uri.Uri; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; + +/** + */ +public class EmitterTest +{ + private static final ObjectMapper jsonMapper = new ObjectMapper(); + public static String TARGET_URL = "http://metrics.foo.bar/"; + public static final Response OK_RESPONSE = responseBuilder(HttpVersion.HTTP_1_1, HttpResponseStatus.CREATED) + .accumulate(new EagerResponseBodyPart(Unpooled.wrappedBuffer("Yay".getBytes(StandardCharsets.UTF_8)), true)) + .build(); + + private static Response.ResponseBuilder responseBuilder(HttpVersion version, HttpResponseStatus status) + { + return new Response.ResponseBuilder() + .accumulate( + new NettyResponseStatus( + Uri.create(TARGET_URL), + new DefaultAsyncHttpClientConfig.Builder().build(), + new DefaultHttpResponse(version, status), + null + ) + ); + } + + + MockHttpClient httpClient; + HttpPostEmitter emitter; + + public static Response okResponse() + { + return OK_RESPONSE; + } + + @Before + public void setUp() throws Exception + { + httpClient = new MockHttpClient(); + } + + @After + public void tearDown() throws Exception + { + if (emitter != null) { + emitter.close(); + } + } + + private HttpPostEmitter timeBasedEmitter(long timeInMillis) + { + HttpEmitterConfig config = new HttpEmitterConfig.Builder(TARGET_URL) + .setFlushMillis(timeInMillis) + .setFlushCount(Integer.MAX_VALUE) + .build(); + HttpPostEmitter emitter = new HttpPostEmitter( + config, + httpClient, + jsonMapper + ); + emitter.start(); + return emitter; + } + + private HttpPostEmitter sizeBasedEmitter(int size) + { + HttpEmitterConfig config = new HttpEmitterConfig.Builder(TARGET_URL) + .setFlushMillis(Long.MAX_VALUE) + .setFlushCount(size) + .build(); + HttpPostEmitter emitter = new HttpPostEmitter( + config, + httpClient, + jsonMapper + ); + emitter.start(); + return emitter; + } + + private HttpPostEmitter sizeBasedEmitterGeneralizedCreation(int size) + { + Properties props = new Properties(); + props.setProperty("io.druid.java.util.emitter.type", "http"); + props.setProperty("io.druid.java.util.emitter.recipientBaseUrl", TARGET_URL); + props.setProperty("io.druid.java.util.emitter.flushMillis", String.valueOf(Long.MAX_VALUE)); + props.setProperty("io.druid.java.util.emitter.flushCount", String.valueOf(size)); + + Lifecycle lifecycle = new Lifecycle(); + Emitter emitter = Emitters.create(props, httpClient, jsonMapper, lifecycle); + Assert.assertTrue(StringUtils.format( + "HttpPostEmitter emitter should be created, but found %s", + emitter.getClass().getName() + ), emitter instanceof HttpPostEmitter); + emitter.start(); + return (HttpPostEmitter) emitter; + } + + private HttpPostEmitter sizeBasedEmitterWithContentEncoding(int size, ContentEncoding encoding) + { + HttpEmitterConfig config = new HttpEmitterConfig.Builder(TARGET_URL) + .setFlushMillis(Long.MAX_VALUE) + .setFlushCount(size) + .setContentEncoding(encoding) + .build(); + HttpPostEmitter emitter = new HttpPostEmitter( + config, + httpClient, + jsonMapper + ); + emitter.start(); + return emitter; + } + + private HttpPostEmitter manualFlushEmitterWithBasicAuthenticationAndNewlineSeparating(String authentication) + { + HttpEmitterConfig config = new HttpEmitterConfig.Builder(TARGET_URL) + .setFlushMillis(Long.MAX_VALUE) + .setFlushCount(Integer.MAX_VALUE) + .setBasicAuthentication(authentication) + .setBatchingStrategy(BatchingStrategy.NEWLINES) + .setMaxBatchSize(1024 * 1024) + .build(); + HttpPostEmitter emitter = new HttpPostEmitter( + config, + httpClient, + jsonMapper + ); + emitter.start(); + return emitter; + } + + private HttpPostEmitter manualFlushEmitterWithBatchSizeAndBufferSize(int batchSize, long bufferSize) + { + HttpEmitterConfig config = new HttpEmitterConfig.Builder(TARGET_URL) + .setFlushMillis(Long.MAX_VALUE) + .setFlushCount(Integer.MAX_VALUE) + .setMaxBatchSize(batchSize) + .build(); + HttpPostEmitter emitter = new HttpPostEmitter( + config, + httpClient, + jsonMapper + ); + emitter.start(); + return emitter; + } + + @Test + public void testSanity() throws Exception + { + final List events = Arrays.asList( + new UnitEvent("test", 1), + new UnitEvent("test", 2) + ); + emitter = sizeBasedEmitter(2); + + httpClient.setGoHandler( + new GoHandler() + { + @Override + protected ListenableFuture go(Request request) throws JsonProcessingException + { + Assert.assertEquals(TARGET_URL, request.getUrl()); + Assert.assertEquals( + "application/json", + request.getHeaders().get(HttpHeaders.Names.CONTENT_TYPE) + ); + Assert.assertEquals( + StringUtils.format( + "[%s,%s]\n", + jsonMapper.writeValueAsString(events.get(0)), + jsonMapper.writeValueAsString(events.get(1)) + ), + Charsets.UTF_8.decode(request.getByteBufferData().slice()).toString() + ); + + return GoHandlers.immediateFuture(okResponse()); + } + }.times(1) + ); + + for (UnitEvent event : events) { + emitter.emit(event); + } + waitForEmission(emitter, 0); + closeNoFlush(emitter); + Assert.assertTrue(httpClient.succeeded()); + } + + @Test + public void testSanityWithGeneralizedCreation() throws Exception + { + final List events = Arrays.asList( + new UnitEvent("test", 1), + new UnitEvent("test", 2) + ); + emitter = sizeBasedEmitterGeneralizedCreation(2); + + httpClient.setGoHandler( + new GoHandler() + { + @Override + protected ListenableFuture go(Request request) throws JsonProcessingException + { + Assert.assertEquals(TARGET_URL, request.getUrl()); + Assert.assertEquals( + "application/json", + request.getHeaders().get(HttpHeaders.Names.CONTENT_TYPE) + ); + Assert.assertEquals( + StringUtils.format( + "[%s,%s]\n", + jsonMapper.writeValueAsString(events.get(0)), + jsonMapper.writeValueAsString(events.get(1)) + ), + Charsets.UTF_8.decode(request.getByteBufferData().slice()).toString() + ); + + return GoHandlers.immediateFuture(okResponse()); + } + }.times(1) + ); + + for (UnitEvent event : events) { + emitter.emit(event); + } + waitForEmission(emitter, 0); + closeNoFlush(emitter); + Assert.assertTrue(httpClient.succeeded()); + } + + @Test + public void testSizeBasedEmission() throws Exception + { + emitter = sizeBasedEmitter(3); + + httpClient.setGoHandler(GoHandlers.failingHandler()); + emitter.emit(new UnitEvent("test", 1)); + emitter.emit(new UnitEvent("test", 2)); + + httpClient.setGoHandler(GoHandlers.passingHandler(okResponse()).times(1)); + emitter.emit(new UnitEvent("test", 3)); + waitForEmission(emitter, 0); + + httpClient.setGoHandler(GoHandlers.failingHandler()); + emitter.emit(new UnitEvent("test", 4)); + emitter.emit(new UnitEvent("test", 5)); + + closeAndExpectFlush(emitter); + Assert.assertTrue(httpClient.succeeded()); + } + + @Test + public void testTimeBasedEmission() throws Exception + { + final int timeBetweenEmissions = 100; + emitter = timeBasedEmitter(timeBetweenEmissions); + + final CountDownLatch latch = new CountDownLatch(1); + + httpClient.setGoHandler( + new GoHandler() + { + @Override + protected ListenableFuture go(Request request) + { + latch.countDown(); + return GoHandlers.immediateFuture(okResponse()); + } + }.times(1) + ); + + long emitTime = System.currentTimeMillis(); + emitter.emit(new UnitEvent("test", 1)); + + latch.await(); + long timeWaited = System.currentTimeMillis() - emitTime; + Assert.assertTrue( + StringUtils.format("timeWaited[%s] !< %s", timeWaited, timeBetweenEmissions * 2), + timeWaited < timeBetweenEmissions * 2 + ); + + waitForEmission(emitter, 0); + + final CountDownLatch thisLatch = new CountDownLatch(1); + httpClient.setGoHandler( + new GoHandler() + { + @Override + protected ListenableFuture go(Request request) + { + thisLatch.countDown(); + return GoHandlers.immediateFuture(okResponse()); + } + }.times(1) + ); + + emitTime = System.currentTimeMillis(); + emitter.emit(new UnitEvent("test", 2)); + + thisLatch.await(); + timeWaited = System.currentTimeMillis() - emitTime; + Assert.assertTrue( + StringUtils.format("timeWaited[%s] !< %s", timeWaited, timeBetweenEmissions * 2), + timeWaited < timeBetweenEmissions * 2 + ); + + waitForEmission(emitter, 1); + closeNoFlush(emitter); + Assert.assertTrue("httpClient.succeeded()", httpClient.succeeded()); + } + + @Test(timeout = 60_000) + public void testFailedEmission() throws Exception + { + final UnitEvent event1 = new UnitEvent("test", 1); + final UnitEvent event2 = new UnitEvent("test", 2); + emitter = sizeBasedEmitter(1); + Assert.assertEquals(0, emitter.getTotalEmittedEvents()); + + httpClient.setGoHandler( + new GoHandler() + { + @Override + protected ListenableFuture go(Request request) + { + Response response = responseBuilder(HttpVersion.HTTP_1_1, HttpResponseStatus.BAD_REQUEST).build(); + return GoHandlers.immediateFuture(response); + } + } + ); + emitter.emit(event1); + emitter.flush(); + waitForEmission(emitter, 0); + Assert.assertTrue(httpClient.succeeded()); + + // Failed to emit the first event. + Assert.assertEquals(0, emitter.getTotalEmittedEvents()); + + httpClient.setGoHandler( + new GoHandler() + { + @Override + protected ListenableFuture go(Request request) + { + return GoHandlers.immediateFuture(okResponse()); + } + }.times(2) + ); + + emitter.emit(event2); + emitter.flush(); + waitForEmission(emitter, 1); + closeNoFlush(emitter); + // Failed event is emitted inside emitter thread, there is no other way to wait for it other than joining the + // emitterThread + emitter.joinEmitterThread(); + + // Succeed to emit both events. + Assert.assertEquals(2, emitter.getTotalEmittedEvents()); + + Assert.assertTrue(httpClient.succeeded()); + } + + @Test + public void testBasicAuthenticationAndNewlineSeparating() throws Exception + { + final List events = Arrays.asList( + new UnitEvent("test", 1), + new UnitEvent("test", 2) + ); + emitter = manualFlushEmitterWithBasicAuthenticationAndNewlineSeparating("foo:bar"); + + httpClient.setGoHandler( + new GoHandler() + { + @Override + protected ListenableFuture go(Request request) throws JsonProcessingException + { + Assert.assertEquals(TARGET_URL, request.getUrl()); + Assert.assertEquals( + "application/json", + request.getHeaders().get(HttpHeaders.Names.CONTENT_TYPE) + ); + Assert.assertEquals( + "Basic " + BaseEncoding.base64().encode(StringUtils.toUtf8("foo:bar")), + request.getHeaders().get(HttpHeaders.Names.AUTHORIZATION) + ); + Assert.assertEquals( + StringUtils.format( + "%s\n%s\n", + jsonMapper.writeValueAsString(events.get(0)), + jsonMapper.writeValueAsString(events.get(1)) + ), + Charsets.UTF_8.decode(request.getByteBufferData().slice()).toString() + ); + + return GoHandlers.immediateFuture(okResponse()); + } + }.times(1) + ); + + for (UnitEvent event : events) { + emitter.emit(event); + } + emitter.flush(); + waitForEmission(emitter, 0); + closeNoFlush(emitter); + Assert.assertTrue(httpClient.succeeded()); + } + + @Test + public void testBatchSplitting() throws Exception + { + final byte[] big = new byte[500 * 1024]; + for (int i = 0; i < big.length; i++) { + big[i] = 'x'; + } + final String bigString = StringUtils.fromUtf8(big); + final List events = Arrays.asList( + new UnitEvent(bigString, 1), + new UnitEvent(bigString, 2), + new UnitEvent(bigString, 3), + new UnitEvent(bigString, 4) + ); + final AtomicInteger counter = new AtomicInteger(); + emitter = manualFlushEmitterWithBatchSizeAndBufferSize(1024 * 1024, 5 * 1024 * 1024); + Assert.assertEquals(0, emitter.getTotalEmittedEvents()); + + httpClient.setGoHandler( + new GoHandler() + { + @Override + protected ListenableFuture go(Request request) throws JsonProcessingException + { + Assert.assertEquals(TARGET_URL, request.getUrl()); + Assert.assertEquals( + "application/json", + request.getHeaders().get(HttpHeaders.Names.CONTENT_TYPE) + ); + Assert.assertEquals( + StringUtils.format( + "[%s,%s]\n", + jsonMapper.writeValueAsString(events.get(counter.getAndIncrement())), + jsonMapper.writeValueAsString(events.get(counter.getAndIncrement())) + ), + Charsets.UTF_8.decode(request.getByteBufferData().slice()).toString() + ); + + return GoHandlers.immediateFuture(okResponse()); + } + }.times(3) + ); + + for (UnitEvent event : events) { + emitter.emit(event); + } + waitForEmission(emitter, 0); + Assert.assertEquals(2, emitter.getTotalEmittedEvents()); + + emitter.flush(); + waitForEmission(emitter, 1); + Assert.assertEquals(4, emitter.getTotalEmittedEvents()); + closeNoFlush(emitter); + Assert.assertTrue(httpClient.succeeded()); + } + + @Test + public void testGzipContentEncoding() throws Exception + { + final List events = Arrays.asList( + new UnitEvent("plain-text", 1), + new UnitEvent("plain-text", 2) + ); + + emitter = sizeBasedEmitterWithContentEncoding(2, ContentEncoding.GZIP); + + httpClient.setGoHandler( + new GoHandler() + { + @Override + protected ListenableFuture go(Request request) throws IOException + { + Assert.assertEquals(TARGET_URL, request.getUrl()); + Assert.assertEquals( + "application/json", + request.getHeaders().get(HttpHeaders.Names.CONTENT_TYPE) + ); + Assert.assertEquals( + HttpHeaders.Values.GZIP, + request.getHeaders().get(HttpHeaders.Names.CONTENT_ENCODING) + ); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ByteBuffer data = request.getByteBufferData().slice(); + byte[] dataArray = new byte[data.remaining()]; + data.get(dataArray); + CompressionUtils.gunzip(new ByteArrayInputStream(dataArray), baos); + + Assert.assertEquals( + StringUtils.format( + "[%s,%s]\n", + jsonMapper.writeValueAsString(events.get(0)), + jsonMapper.writeValueAsString(events.get(1)) + ), + baos.toString(Charsets.UTF_8.name()) + ); + + return GoHandlers.immediateFuture(okResponse()); + } + }.times(1) + ); + + for (UnitEvent event : events) { + emitter.emit(event); + } + waitForEmission(emitter, 0); + closeNoFlush(emitter); + Assert.assertTrue(httpClient.succeeded()); + } + + private void closeAndExpectFlush(Emitter emitter) throws IOException + { + httpClient.setGoHandler(GoHandlers.passingHandler(okResponse()).times(1)); + emitter.close(); + } + + private void closeNoFlush(Emitter emitter) throws IOException + { + emitter.close(); + } + + private void waitForEmission(HttpPostEmitter emitter, int batchNumber) throws Exception + { + emitter.waitForEmission(batchNumber); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/GoHandler.java b/java-util/src/test/java/io/druid/java/util/emitter/core/GoHandler.java new file mode 100644 index 000000000000..0e2226236131 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/GoHandler.java @@ -0,0 +1,77 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.google.common.base.Throwables; +import io.druid.java.util.common.ISE; +import org.asynchttpclient.ListenableFuture; +import org.asynchttpclient.Request; +import org.asynchttpclient.Response; + +import java.util.concurrent.atomic.AtomicInteger; + +/** +*/ +public abstract class GoHandler +{ + /******* Abstract Methods *********/ + protected abstract ListenableFuture go(Request request) throws X; + + /******* Non Abstract Methods ********/ + private volatile boolean succeeded = false; + + public boolean succeeded() + { + return succeeded; + } + + public ListenableFuture run(Request request) + { + try { + final ListenableFuture retVal = go(request); + succeeded = true; + return retVal; + } + catch (Throwable e) { + succeeded = false; + throw Throwables.propagate(e); + } + } + + public GoHandler times(final int n) + { + final GoHandler myself = this; + + return new GoHandler() + { + AtomicInteger counter = new AtomicInteger(0); + + @Override + public ListenableFuture go(final Request request) + { + if (counter.getAndIncrement() < n) { + return myself.go(request); + } + succeeded = false; + throw new ISE("Called more than %d times", n); + } + }; + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/GoHandlers.java b/java-util/src/test/java/io/druid/java/util/emitter/core/GoHandlers.java new file mode 100644 index 000000000000..4cd7e738b8a2 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/GoHandlers.java @@ -0,0 +1,125 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import io.druid.java.util.common.ISE; +import org.asynchttpclient.ListenableFuture; +import org.asynchttpclient.Request; +import org.asynchttpclient.Response; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + */ +public class GoHandlers +{ + public static GoHandler failingHandler() + { + return new GoHandler() + { + @Override + protected ListenableFuture go(Request request) + { + throw new ISE("Shouldn't be called"); + } + }; + } + + public static GoHandler passingHandler(final Response retVal) + { + return new GoHandler() + { + @Override + protected ListenableFuture go(Request request) + { + return immediateFuture(retVal); + } + }; + } + + static ListenableFuture immediateFuture(T val) + { + CompletableFuture future = CompletableFuture.completedFuture(val); + return new ListenableFuture() + { + @Override + public void done() + { + } + + @Override + public void abort(Throwable t) + { + } + + @Override + public void touch() + { + } + + @Override + public ListenableFuture addListener(Runnable listener, Executor exec) + { + future.thenAcceptAsync(r -> listener.run(), exec); + return this; + } + + @Override + public CompletableFuture toCompletableFuture() + { + return future; + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) + { + return false; + } + + @Override + public boolean isCancelled() + { + return false; + } + + @Override + public boolean isDone() + { + return true; + } + + @Override + public T get() throws InterruptedException, ExecutionException + { + return future.get(); + } + + @Override + public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException + { + return future.get(timeout, unit); + } + }; + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/HttpEmitterConfigTest.java b/java-util/src/test/java/io/druid/java/util/emitter/core/HttpEmitterConfigTest.java new file mode 100644 index 000000000000..a80740f9eb4e --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/HttpEmitterConfigTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Properties; + +public class HttpEmitterConfigTest +{ + @Test + public void testDefaults() + { + final Properties props = new Properties(); + props.put("io.druid.java.util.emitter.recipientBaseUrl", "http://example.com/"); + + final ObjectMapper objectMapper = new ObjectMapper(); + final HttpEmitterConfig config = objectMapper.convertValue( + Emitters.makeCustomFactoryMap(props), + HttpEmitterConfig.class + ); + + Assert.assertEquals(60000, config.getFlushMillis()); + Assert.assertEquals(500, config.getFlushCount()); + Assert.assertEquals("http://example.com/", config.getRecipientBaseUrl()); + Assert.assertEquals(null, config.getBasicAuthentication()); + Assert.assertEquals(BatchingStrategy.ARRAY, config.getBatchingStrategy()); + Assert.assertEquals(5 * 1024 * 1024, config.getMaxBatchSize()); + Assert.assertEquals(Long.MAX_VALUE, config.getFlushTimeOut()); + Assert.assertEquals(50, config.getBatchQueueSizeLimit()); + Assert.assertEquals(2.0f, config.getHttpTimeoutAllowanceFactor(), 0.0f); + Assert.assertEquals(0, config.getMinHttpTimeoutMillis()); + } + + @Test + public void testDefaultsLegacy() + { + final Properties props = new Properties(); + props.put("io.druid.java.util.emitter.http.url", "http://example.com/"); + + final ObjectMapper objectMapper = new ObjectMapper(); + final HttpEmitterConfig config = objectMapper.convertValue(Emitters.makeHttpMap(props), HttpEmitterConfig.class); + + Assert.assertEquals(60000, config.getFlushMillis()); + Assert.assertEquals(300, config.getFlushCount()); + Assert.assertEquals("http://example.com/", config.getRecipientBaseUrl()); + Assert.assertEquals(null, config.getBasicAuthentication()); + Assert.assertEquals(BatchingStrategy.ARRAY, config.getBatchingStrategy()); + Assert.assertEquals(5 * 1024 * 1024, config.getMaxBatchSize()); + Assert.assertEquals(Long.MAX_VALUE, config.getFlushTimeOut()); + Assert.assertEquals(50, config.getBatchQueueSizeLimit()); + Assert.assertEquals(2.0f, config.getHttpTimeoutAllowanceFactor(), 0.0f); + Assert.assertEquals(0, config.getMinHttpTimeoutMillis()); + } + + @Test + public void testSettingEverything() + { + final Properties props = new Properties(); + props.setProperty("io.druid.java.util.emitter.flushMillis", "1"); + props.setProperty("io.druid.java.util.emitter.flushCount", "2"); + props.setProperty("io.druid.java.util.emitter.recipientBaseUrl", "http://example.com/"); + props.setProperty("io.druid.java.util.emitter.basicAuthentication", "a:b"); + props.setProperty("io.druid.java.util.emitter.batchingStrategy", "NEWLINES"); + props.setProperty("io.druid.java.util.emitter.maxBatchSize", "4"); + props.setProperty("io.druid.java.util.emitter.flushTimeOut", "1000"); + props.setProperty("io.druid.java.util.emitter.batchQueueSizeLimit", "2500"); + props.setProperty("io.druid.java.util.emitter.httpTimeoutAllowanceFactor", "3.0"); + props.setProperty("io.druid.java.util.emitter.minHttpTimeoutMillis", "100"); + + final ObjectMapper objectMapper = new ObjectMapper(); + final HttpEmitterConfig config = objectMapper.convertValue( + Emitters.makeCustomFactoryMap(props), + HttpEmitterConfig.class + ); + + Assert.assertEquals(1, config.getFlushMillis()); + Assert.assertEquals(2, config.getFlushCount()); + Assert.assertEquals("http://example.com/", config.getRecipientBaseUrl()); + Assert.assertEquals("a:b", config.getBasicAuthentication()); + Assert.assertEquals(BatchingStrategy.NEWLINES, config.getBatchingStrategy()); + Assert.assertEquals(4, config.getMaxBatchSize()); + Assert.assertEquals(1000, config.getFlushTimeOut()); + Assert.assertEquals(2500, config.getBatchQueueSizeLimit()); + Assert.assertEquals(3.0f, config.getHttpTimeoutAllowanceFactor(), 0.0f); + Assert.assertEquals(100, config.getMinHttpTimeoutMillis()); + } + + @Test + public void testSettingEverythingLegacy() + { + final Properties props = new Properties(); + props.setProperty("io.druid.java.util.emitter.flushMillis", "1"); + props.setProperty("io.druid.java.util.emitter.flushCount", "2"); + props.setProperty("io.druid.java.util.emitter.http.url", "http://example.com/"); + props.setProperty("io.druid.java.util.emitter.http.basicAuthentication", "a:b"); + props.setProperty("io.druid.java.util.emitter.http.batchingStrategy", "newlines"); + props.setProperty("io.druid.java.util.emitter.http.maxBatchSize", "4"); + props.setProperty("io.druid.java.util.emitter.http.flushTimeOut", "1000"); + props.setProperty("io.druid.java.util.emitter.http.batchQueueSizeLimit", "2500"); + props.setProperty("io.druid.java.util.emitter.http.httpTimeoutAllowanceFactor", "3.0"); + props.setProperty("io.druid.java.util.emitter.http.minHttpTimeoutMillis", "100"); + + final ObjectMapper objectMapper = new ObjectMapper(); + final HttpEmitterConfig config = objectMapper.convertValue(Emitters.makeHttpMap(props), HttpEmitterConfig.class); + + Assert.assertEquals(1, config.getFlushMillis()); + Assert.assertEquals(2, config.getFlushCount()); + Assert.assertEquals("http://example.com/", config.getRecipientBaseUrl()); + Assert.assertEquals("a:b", config.getBasicAuthentication()); + Assert.assertEquals(BatchingStrategy.NEWLINES, config.getBatchingStrategy()); + Assert.assertEquals(4, config.getMaxBatchSize()); + Assert.assertEquals(1000, config.getFlushTimeOut()); + Assert.assertEquals(2500, config.getBatchQueueSizeLimit()); + Assert.assertEquals(3.0f, config.getHttpTimeoutAllowanceFactor(), 0.0f); + Assert.assertEquals(100, config.getMinHttpTimeoutMillis()); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/HttpEmitterTest.java b/java-util/src/test/java/io/druid/java/util/emitter/core/HttpEmitterTest.java new file mode 100644 index 000000000000..85e471466e7c --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/HttpEmitterTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.primitives.Ints; +import org.asynchttpclient.ListenableFuture; +import org.asynchttpclient.Request; +import org.asynchttpclient.Response; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicLong; + +public class HttpEmitterTest +{ + private final MockHttpClient httpClient = new MockHttpClient(); + private static final ObjectMapper objectMapper = new ObjectMapper() + { + @Override + public byte[] writeValueAsBytes(Object value) throws JsonProcessingException + { + return Ints.toByteArray(((IntEvent) value).index); + } + }; + + private final AtomicLong timeoutUsed = new AtomicLong(); + + @Before + public void setup() + { + timeoutUsed.set(-1L); + + httpClient.setGoHandler(new GoHandler() + { + @Override + protected ListenableFuture go(Request request) + { + int timeout = request.getRequestTimeout(); + timeoutUsed.set(timeout); + return GoHandlers.immediateFuture(EmitterTest.okResponse()); + } + }); + } + + @Test + public void timeoutEmptyQueue() throws IOException, InterruptedException + { + final HttpEmitterConfig config = new HttpEmitterConfig.Builder("http://foo.bar") + .setBatchingStrategy(BatchingStrategy.ONLY_EVENTS) + .setHttpTimeoutAllowanceFactor(2.0f) + .build(); + final HttpPostEmitter emitter = new HttpPostEmitter(config, httpClient, objectMapper); + + emitter.start(); + emitter.emitAndReturnBatch(new IntEvent()); + emitter.flush(); + Assert.assertTrue(timeoutUsed.get() < 5); + + final Batch batch = emitter.emitAndReturnBatch(new IntEvent()); + Thread.sleep(1000); + batch.seal(); + emitter.flush(); + Assert.assertTrue(timeoutUsed.get() >= 2000 && timeoutUsed.get() < 3000); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/HttpPostEmitterStressTest.java b/java-util/src/test/java/io/druid/java/util/emitter/core/HttpPostEmitterStressTest.java new file mode 100644 index 000000000000..06ad38c9751e --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/HttpPostEmitterStressTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.primitives.Ints; +import com.google.common.util.concurrent.Futures; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; +import org.asynchttpclient.ListenableFuture; +import org.asynchttpclient.Request; +import org.asynchttpclient.Response; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadLocalRandom; + +public class HttpPostEmitterStressTest +{ + private static final int N = 10_000; + private static final Future OK_FUTURE = Futures.immediateFuture(EmitterTest.OK_RESPONSE); + private static final ObjectMapper objectMapper = new ObjectMapper() + { + @Override + public byte[] writeValueAsBytes(Object value) throws JsonProcessingException + { + return Ints.toByteArray(((IntEvent) value).index); + } + }; + + private final MockHttpClient httpClient = new MockHttpClient(); + + @Test + public void eventCountBased() throws InterruptedException, IOException + { + HttpEmitterConfig config = new HttpEmitterConfig.Builder("http://foo.bar") + .setFlushMillis(100) + .setFlushCount(4) + .setBatchingStrategy(BatchingStrategy.ONLY_EVENTS) + .setMaxBatchSize(1024 * 1024) + // For this test, we don't need any batches to be dropped, i. e. "gaps" in data + .setBatchQueueSizeLimit(1000) + .build(); + final HttpPostEmitter emitter = new HttpPostEmitter(config, httpClient, objectMapper); + int nThreads = Runtime.getRuntime().availableProcessors() * 2; + final List eventsPerThread = new ArrayList<>(nThreads); + final List> eventBatchesPerThread = new ArrayList<>(nThreads); + for (int i = 0; i < nThreads; i++) { + eventsPerThread.add(new IntArrayList()); + eventBatchesPerThread.add(new ArrayList()); + } + for (int i = 0; i < N; i++) { + eventsPerThread.get(ThreadLocalRandom.current().nextInt(nThreads)).add(i); + } + final BitSet emittedEvents = new BitSet(N); + httpClient.setGoHandler(new GoHandler() + { + @Override + protected ListenableFuture go(Request request) + { + ByteBuffer batch = request.getByteBufferData().slice(); + while (batch.remaining() > 0) { + emittedEvents.set(batch.getInt()); + } + return GoHandlers.immediateFuture(EmitterTest.okResponse()); + } + }); + emitter.start(); + final CountDownLatch threadsCompleted = new CountDownLatch(nThreads); + for (int i = 0; i < nThreads; i++) { + final int threadIndex = i; + new Thread() { + @Override + public void run() + { + IntList events = eventsPerThread.get(threadIndex); + List eventBatches = eventBatchesPerThread.get(threadIndex); + IntEvent event = new IntEvent(); + for (int i = 0, eventsSize = events.size(); i < eventsSize; i++) { + event.index = events.getInt(i); + eventBatches.add(emitter.emitAndReturnBatch(event)); + if (i % 16 == 0) { + try { + Thread.sleep(10); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + threadsCompleted.countDown(); + } + }.start(); + } + threadsCompleted.await(); + emitter.flush(); + System.out.println("Allocated buffers: " + emitter.getTotalAllocatedBuffers()); + for (int eventIndex = 0; eventIndex < N; eventIndex++) { + if (!emittedEvents.get(eventIndex)) { + for (int threadIndex = 0; threadIndex < eventsPerThread.size(); threadIndex++) { + IntList threadEvents = eventsPerThread.get(threadIndex); + int indexOfEvent = threadEvents.indexOf(eventIndex); + if (indexOfEvent >= 0) { + Batch batch = eventBatchesPerThread.get(threadIndex).get(indexOfEvent); + System.err.println(batch); + int bufferWatermark = batch.getSealedBufferWatermark(); + ByteBuffer batchBuffer = ByteBuffer.wrap(batch.buffer); + batchBuffer.limit(bufferWatermark); + while (batchBuffer.remaining() > 0) { + System.err.println(batchBuffer.getInt()); + } + break; + } + } + throw new AssertionError("event " + eventIndex); + } + } + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/IntEvent.java b/java-util/src/test/java/io/druid/java/util/emitter/core/IntEvent.java new file mode 100644 index 000000000000..e58ae2f5dbca --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/IntEvent.java @@ -0,0 +1,57 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import org.joda.time.DateTime; + +import java.util.Map; + +class IntEvent implements Event +{ + int index; + + IntEvent() + { + } + + @Override + public Map toMap() + { + return null; + } + + @Override + public String getFeed() + { + return null; + } + + @Override + public DateTime getCreatedTime() + { + return null; + } + + @Override + public boolean isSafeToBuffer() + { + return false; + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/LoggingEmitterConfigTest.java b/java-util/src/test/java/io/druid/java/util/emitter/core/LoggingEmitterConfigTest.java new file mode 100644 index 000000000000..2a7898a10529 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/LoggingEmitterConfigTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Properties; + +public class LoggingEmitterConfigTest +{ + @Test + public void testDefaults() + { + final Properties props = new Properties(); + final ObjectMapper objectMapper = new ObjectMapper(); + final LoggingEmitterConfig config = objectMapper.convertValue( + Emitters.makeCustomFactoryMap(props), + LoggingEmitterConfig.class + ); + Assert.assertEquals("getLoggerClass", LoggingEmitter.class.getName(), config.getLoggerClass()); + Assert.assertEquals("getLogLevel", "info", config.getLogLevel()); + } + + @Test + public void testDefaultsLegacy() + { + final Properties props = new Properties(); + final ObjectMapper objectMapper = new ObjectMapper(); + final LoggingEmitterConfig config = objectMapper.convertValue( + Emitters.makeLoggingMap(props), + LoggingEmitterConfig.class + ); + + Assert.assertEquals("getLoggerClass", LoggingEmitter.class.getName(), config.getLoggerClass()); + Assert.assertEquals("getLogLevel", "debug", config.getLogLevel()); + } + + @Test + public void testSettingEverything() + { + final Properties props = new Properties(); + props.setProperty("io.druid.java.util.emitter.loggerClass", "Foo"); + props.setProperty("io.druid.java.util.emitter.logLevel", "INFO"); + + final ObjectMapper objectMapper = new ObjectMapper(); + final LoggingEmitterConfig config = objectMapper.convertValue( + Emitters.makeCustomFactoryMap(props), + LoggingEmitterConfig.class + ); + + Assert.assertEquals("getLoggerClass", "Foo", config.getLoggerClass()); + Assert.assertEquals("getLogLevel", "INFO", config.getLogLevel()); + } + + @Test + public void testSettingEverythingLegacy() + { + final Properties props = new Properties(); + props.setProperty("io.druid.java.util.emitter.logging.class", "Foo"); + props.setProperty("io.druid.java.util.emitter.logging.level", "INFO"); + + final ObjectMapper objectMapper = new ObjectMapper(); + final LoggingEmitterConfig config = objectMapper.convertValue( + Emitters.makeLoggingMap(props), + LoggingEmitterConfig.class + ); + + Assert.assertEquals("getLoggerClass", "Foo", config.getLoggerClass()); + Assert.assertEquals("getLogLevel", "INFO", config.getLogLevel()); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/MockHttpClient.java b/java-util/src/test/java/io/druid/java/util/emitter/core/MockHttpClient.java new file mode 100644 index 000000000000..06a3b4fee290 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/MockHttpClient.java @@ -0,0 +1,57 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import org.asynchttpclient.DefaultAsyncHttpClient; +import org.asynchttpclient.ListenableFuture; +import org.asynchttpclient.Request; +import org.asynchttpclient.Response; + +/** + */ +public class MockHttpClient extends DefaultAsyncHttpClient +{ + private volatile GoHandler goHandler; + + public MockHttpClient() + { + } + + public GoHandler getGoHandler() + { + return goHandler; + } + + public void setGoHandler(GoHandler goHandler) + { + this.goHandler = goHandler; + } + + public boolean succeeded() + { + return goHandler.succeeded(); + } + + @Override + public ListenableFuture executeRequest(Request request) + { + return goHandler.run(request); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/ParametrizedUriEmitterConfigTest.java b/java-util/src/test/java/io/druid/java/util/emitter/core/ParametrizedUriEmitterConfigTest.java new file mode 100644 index 000000000000..63f7c63a19d0 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/ParametrizedUriEmitterConfigTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Properties; + +public class ParametrizedUriEmitterConfigTest +{ + @Test + public void testDefaults() + { + final Properties props = new Properties(); + + final ObjectMapper objectMapper = new ObjectMapper(); + final ParametrizedUriEmitterConfig paramConfig = objectMapper.convertValue(Emitters.makeCustomFactoryMap(props), ParametrizedUriEmitterConfig.class); + final HttpEmitterConfig config = paramConfig.buildHttpEmitterConfig("http://example.com/topic"); + + Assert.assertEquals(60000, config.getFlushMillis()); + Assert.assertEquals(500, config.getFlushCount()); + Assert.assertEquals("http://example.com/topic", config.getRecipientBaseUrl()); + Assert.assertEquals(null, config.getBasicAuthentication()); + Assert.assertEquals(BatchingStrategy.ARRAY, config.getBatchingStrategy()); + Assert.assertEquals(5 * 1024 * 1024, config.getMaxBatchSize()); + Assert.assertEquals(Long.MAX_VALUE, config.getFlushTimeOut()); + } + + @Test + public void testSettingEverything() + { + final Properties props = new Properties(); + props.setProperty("io.druid.java.util.emitter.httpEmitting.flushMillis", "1"); + props.setProperty("io.druid.java.util.emitter.httpEmitting.flushCount", "2"); + props.setProperty("io.druid.java.util.emitter.httpEmitting.basicAuthentication", "a:b"); + props.setProperty("io.druid.java.util.emitter.httpEmitting.batchingStrategy", "NEWLINES"); + props.setProperty("io.druid.java.util.emitter.httpEmitting.maxBatchSize", "4"); + props.setProperty("io.druid.java.util.emitter.httpEmitting.flushTimeOut", "1000"); + + final ObjectMapper objectMapper = new ObjectMapper(); + final ParametrizedUriEmitterConfig paramConfig = objectMapper.convertValue(Emitters.makeCustomFactoryMap(props), ParametrizedUriEmitterConfig.class); + final HttpEmitterConfig config = paramConfig.buildHttpEmitterConfig("http://example.com/topic"); + + Assert.assertEquals(1, config.getFlushMillis()); + Assert.assertEquals(2, config.getFlushCount()); + Assert.assertEquals("http://example.com/topic", config.getRecipientBaseUrl()); + Assert.assertEquals("a:b", config.getBasicAuthentication()); + Assert.assertEquals(BatchingStrategy.NEWLINES, config.getBatchingStrategy()); + Assert.assertEquals(4, config.getMaxBatchSize()); + Assert.assertEquals(1000, config.getFlushTimeOut()); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/ParametrizedUriEmitterTest.java b/java-util/src/test/java/io/druid/java/util/emitter/core/ParametrizedUriEmitterTest.java new file mode 100644 index 000000000000..0f794e15f72f --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/ParametrizedUriEmitterTest.java @@ -0,0 +1,216 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.core; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableMap; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.lifecycle.Lifecycle; +import io.druid.java.util.emitter.service.UnitEvent; +import org.asynchttpclient.ListenableFuture; +import org.asynchttpclient.Request; +import org.asynchttpclient.Response; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static io.druid.java.util.emitter.core.EmitterTest.okResponse; +import static org.junit.Assert.assertEquals; + +public class ParametrizedUriEmitterTest +{ + private static final ObjectMapper jsonMapper = new ObjectMapper(); + + private MockHttpClient httpClient; + private Lifecycle lifecycle; + + @Before + public void setUp() throws Exception + { + httpClient = new MockHttpClient(); + } + + @After + public void tearDown() throws Exception + { + if (lifecycle != null) { + lifecycle.stop(); + } + } + + private Emitter parametrizedEmmiter(String uriPattern) throws Exception + { + final Properties props = new Properties(); + props.setProperty("io.druid.java.util.emitter.type", "parametrized"); + props.setProperty("io.druid.java.util.emitter.recipientBaseUrlPattern", uriPattern); + lifecycle = new Lifecycle(); + Emitter emitter = Emitters.create(props, httpClient, lifecycle); + assertEquals(ParametrizedUriEmitter.class, emitter.getClass()); + lifecycle.start(); + return emitter; + } + + @Test + public void testParametrizedEmitterCreated() throws Exception + { + parametrizedEmmiter("http://example.com/"); + } + + @Test + public void testEmitterWithFeedUriExtractor() throws Exception + { + Emitter emitter = parametrizedEmmiter("http://example.com/{feed}"); + final List events = Arrays.asList( + new UnitEvent("test", 1), + new UnitEvent("test", 2) + ); + + httpClient.setGoHandler( + new GoHandler() + { + @Override + public ListenableFuture go(Request request) throws JsonProcessingException + { + Assert.assertEquals("http://example.com/test", request.getUrl()); + Assert.assertEquals( + StringUtils.format( + "[%s,%s]\n", + jsonMapper.writeValueAsString(events.get(0)), + jsonMapper.writeValueAsString(events.get(1)) + ), + Charsets.UTF_8.decode(request.getByteBufferData().slice()).toString() + ); + + return GoHandlers.immediateFuture(okResponse()); + } + }.times(1) + ); + + for (UnitEvent event : events) { + emitter.emit(event); + } + emitter.flush(); + Assert.assertTrue(httpClient.succeeded()); + } + + @Test + public void testEmitterWithMultipleFeeds() throws Exception + { + Emitter emitter = parametrizedEmmiter("http://example.com/{feed}"); + final List events = Arrays.asList( + new UnitEvent("test1", 1), + new UnitEvent("test2", 2) + ); + + final Map results = new HashMap<>(); + + httpClient.setGoHandler( + new GoHandler() + { + @Override + protected ListenableFuture go(Request request) + { + results.put( + request.getUrl().toString(), + Charsets.UTF_8.decode(request.getByteBufferData().slice()).toString() + ); + return GoHandlers.immediateFuture(okResponse()); + } + }.times(2) + ); + + for (UnitEvent event : events) { + emitter.emit(event); + } + emitter.flush(); + Assert.assertTrue(httpClient.succeeded()); + Map expected = ImmutableMap.of( + "http://example.com/test1", StringUtils.format("[%s]\n", jsonMapper.writeValueAsString(events.get(0))), + "http://example.com/test2", StringUtils.format("[%s]\n", jsonMapper.writeValueAsString(events.get(1)))); + Assert.assertEquals(expected, results); + } + + @Test + public void testEmitterWithParametrizedUriExtractor() throws Exception + { + Emitter emitter = parametrizedEmmiter("http://example.com/{key1}/{key2}"); + final List events = Arrays.asList( + new UnitEvent("test", 1, ImmutableMap.of("key1", "val1", "key2", "val2")), + new UnitEvent("test", 2, ImmutableMap.of("key1", "val1", "key2", "val2")) + ); + + httpClient.setGoHandler( + new GoHandler() + { + @Override + protected ListenableFuture go(Request request) throws JsonProcessingException + { + Assert.assertEquals("http://example.com/val1/val2", request.getUrl()); + Assert.assertEquals( + StringUtils.format( + "[%s,%s]\n", + jsonMapper.writeValueAsString(events.get(0)), + jsonMapper.writeValueAsString(events.get(1)) + ), + Charsets.UTF_8.decode(request.getByteBufferData().slice()).toString() + ); + + return GoHandlers.immediateFuture(okResponse()); + } + }.times(1) + ); + + for (UnitEvent event : events) { + emitter.emit(event); + } + emitter.flush(); + Assert.assertTrue(httpClient.succeeded()); + } + + @Test + public void failEmitMalformedEvent() throws Exception + { + Emitter emitter = parametrizedEmmiter("http://example.com/{keyNotSetInEvents}"); + Event event = new UnitEvent("test", 1); + + httpClient.setGoHandler(GoHandlers.failingHandler()); + + try { + emitter.emit(event); + emitter.flush(); + } + catch (IllegalArgumentException e) { + Assert.assertEquals( + e.getMessage(), + StringUtils.format( + "ParametrizedUriExtractor with pattern http://example.com/{keyNotSetInEvents} requires keyNotSetInEvents to be set in event, but found %s", event.toMap()) + ); + } + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/service/AlertEventTest.java b/java-util/src/test/java/io/druid/java/util/emitter/service/AlertEventTest.java new file mode 100644 index 000000000000..a2c5853ef069 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/emitter/service/AlertEventTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.emitter.service; + +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import io.druid.java.util.emitter.service.AlertEvent.Severity; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Map; + +/** + */ +public class AlertEventTest +{ + @Test + public void testStupid() throws Exception + { + AlertEvent event = AlertBuilder.create("blargy") + .addData("something1", "a") + .addData("something2", "b") + .build("test", "localhost"); + + Assert.assertEquals( + ImmutableMap.builder() + .put("feed", "alerts") + .put("timestamp", event.getCreatedTime().toString()) + .put("service", "test") + .put("host", "localhost") + .put("severity", "component-failure") + .put("description", "blargy") + .put("data", ImmutableMap.of("something1", "a", "something2", "b")) + .build(), + event.toMap() + ); + } + + @Test + public void testAnomaly() throws Exception + { + AlertEvent event = AlertBuilder.create("blargy") + .severity(Severity.ANOMALY) + .addData("something1", "a") + .addData("something2", "b") + .build("test", "localhost"); + + Assert.assertEquals( + ImmutableMap.builder() + .put("feed", "alerts") + .put("timestamp", event.getCreatedTime().toString()) + .put("service", "test") + .put("host", "localhost") + .put("severity", "anomaly") + .put("description", "blargy") + .put("data", ImmutableMap.of("something1", "a", "something2", "b")) + .build(), + event.toMap() + ); + } + + @Test + public void testComponentFailure() throws Exception + { + AlertEvent event = AlertBuilder.create("blargy") + .severity(Severity.COMPONENT_FAILURE) + .addData("something1", "a") + .addData("something2", "b") + .build("test", "localhost"); + + Assert.assertEquals( + ImmutableMap.builder() + .put("feed", "alerts") + .put("timestamp", event.getCreatedTime().toString()) + .put("service", "test") + .put("host", "localhost") + .put("severity", "component-failure") + .put("description", "blargy") + .put("data", ImmutableMap.of("something1", "a", "something2", "b")) + .build(), + event.toMap() + ); + } + + @Test + public void testServiceFailure() throws Exception + { + AlertEvent event = AlertBuilder.create("blargy") + .severity(Severity.SERVICE_FAILURE) + .addData("something1", "a") + .addData("something2", "b") + .build("test", "localhost"); + + Assert.assertEquals( + ImmutableMap.builder() + .put("feed", "alerts") + .put("timestamp", event.getCreatedTime().toString()) + .put("service", "test") + .put("host", "localhost") + .put("severity", "service-failure") + .put("description", "blargy") + .put("data", ImmutableMap.of("something1", "a", "something2", "b")) + .build(), + event.toMap() + ); + } + + @Test + public void testDefaulting() throws Exception + { + final String service = "some service"; + final String host = "some host"; + final String desc = "some description"; + final Map data = ImmutableMap.builder().put("a", "1").put("b", "2").build(); + for (Severity severity : new Severity[]{Severity.ANOMALY, Severity.COMPONENT_FAILURE, Severity.SERVICE_FAILURE}) { + Assert.assertEquals( + contents(new AlertEvent(service, host, desc, data)), + contents(new AlertEvent(service, host, Severity.COMPONENT_FAILURE, desc, data)) + ); + + Assert.assertEquals( + contents(new AlertEvent(service, host, desc)), + contents(new AlertEvent(service, host, Severity.COMPONENT_FAILURE, desc, ImmutableMap.of())) + ); + + Assert.assertEquals( + contents(AlertBuilder.create(desc).addData("a", "1").addData("b", "2").build(service, host)), + contents(new AlertEvent(service, host, Severity.COMPONENT_FAILURE, desc, data)) + ); + + Assert.assertEquals( + contents(AlertBuilder.create(desc).addData(data).build(service, host)), + contents(new AlertEvent(service, host, Severity.COMPONENT_FAILURE, desc, data)) + ); + + Assert.assertEquals( + contents(AlertBuilder.create(desc) + .severity(severity) + .addData("a", "1") + .addData("b", "2") + .build(service, host)), + contents(new AlertEvent(service, host, severity, desc, data)) + ); + + Assert.assertEquals( + contents(AlertBuilder.create(desc).severity(severity).addData(data).build(service, host)), + contents(new AlertEvent(service, host, severity, desc, data)) + ); + } + } + + public Map contents(AlertEvent a) + { + return Maps.filterKeys(a.toMap(), new Predicate() + { + @Override + public boolean apply(String k) + { + return !k.equals("timestamp"); + } + }); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/service/ServiceMetricEventTest.java b/java-util/src/test/java/io/druid/java/util/emitter/service/ServiceMetricEventTest.java new file mode 100644 index 000000000000..ba64100ee719 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/emitter/service/ServiceMetricEventTest.java @@ -0,0 +1,292 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package io.druid.java.util.emitter.service; + +import com.google.common.collect.ImmutableMap; +import io.druid.java.util.common.DateTimes; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; + +/** + */ +public class ServiceMetricEventTest +{ + @Test + public void testStupidTest() throws Exception + { + ServiceMetricEvent builderEvent = new ServiceMetricEvent.Builder() + .setDimension("user1", "a") + .setDimension("user2", "b") + .setDimension("user3", "c") + .setDimension("user4", "d") + .setDimension("user5", "e") + .setDimension("user6", "f") + .setDimension("user7", "g") + .setDimension("user8", "h") + .setDimension("user9", "i") + .setDimension("user10", "j") + .build("test-metric", 1234) + .build("test", "localhost"); + Assert.assertEquals( + ImmutableMap.builder() + .put("feed", "metrics") + .put("timestamp", builderEvent.getCreatedTime().toString()) + .put("service", "test") + .put("host", "localhost") + .put("metric", "test-metric") + .put("user1", "a") + .put("user2", "b") + .put("user3", "c") + .put("user4", "d") + .put("user5", "e") + .put("user6", "f") + .put("user7", "g") + .put("user8", "h") + .put("user9", "i") + .put("user10", "j") + .put("value", 1234) + .build(), + builderEvent.toMap() + ); + + ServiceMetricEvent constructorEvent = ServiceMetricEvent + .builder() + .setDimension("user1", "a") + .setDimension("user2", "b") + .setDimension("user3", "c") + .setDimension("user4", "d") + .setDimension("user5", "e") + .setDimension("user6", "f") + .setDimension("user7", "g") + .setDimension("user8", "h") + .setDimension("user9", "i") + .setDimension("user10", "j") + .build("test-metric", 1234) + .build("test", "localhost"); + + Assert.assertEquals( + ImmutableMap.builder() + .put("feed", "metrics") + .put("timestamp", constructorEvent.getCreatedTime().toString()) + .put("service", "test") + .put("host", "localhost") + .put("metric", "test-metric") + .put("user1", "a") + .put("user2", "b") + .put("user3", "c") + .put("user4", "d") + .put("user5", "e") + .put("user6", "f") + .put("user7", "g") + .put("user8", "h") + .put("user9", "i") + .put("user10", "j") + .put("value", 1234) + .build(), constructorEvent.toMap() + ); + + ServiceMetricEvent arrayConstructorEvent = ServiceMetricEvent + .builder() + .setDimension("user1", new String[]{"a"}) + .setDimension("user2", new String[]{"b"}) + .setDimension("user3", new String[]{"c"}) + .setDimension("user4", new String[]{"d"}) + .setDimension("user5", new String[]{"e"}) + .setDimension("user6", new String[]{"f"}) + .setDimension("user7", new String[]{"g"}) + .setDimension("user8", new String[]{"h"}) + .setDimension("user9", new String[]{"i"}) + .setDimension("user10", new String[]{"j"}) + .build("test-metric", 1234) + .build("test", "localhost"); + + Assert.assertEquals( + ImmutableMap.builder() + .put("feed", "metrics") + .put("timestamp", arrayConstructorEvent.getCreatedTime().toString()) + .put("service", "test") + .put("host", "localhost") + .put("metric", "test-metric") + .put("user1", Arrays.asList("a")) + .put("user2", Arrays.asList("b")) + .put("user3", Arrays.asList("c")) + .put("user4", Arrays.asList("d")) + .put("user5", Arrays.asList("e")) + .put("user6", Arrays.asList("f")) + .put("user7", Arrays.asList("g")) + .put("user8", Arrays.asList("h")) + .put("user9", Arrays.asList("i")) + .put("user10", Arrays.asList("j")) + .put("value", 1234) + .build(), arrayConstructorEvent.toMap() + ); + + Assert.assertNotNull( + new ServiceMetricEvent.Builder() + .setDimension("user1", "a") + .setDimension("user2", "b") + .setDimension("user3", "c") + .setDimension("user4", "d") + .setDimension("user5", "e") + .setDimension("user6", "f") + .setDimension("user7", "g") + .setDimension("user8", "h") + .setDimension("user9", "i") + .setDimension("user10", "j") + .build(null, "test-metric", 1234) + .build("test", "localhost") + .getCreatedTime() + ); + + Assert.assertNotNull( + ServiceMetricEvent.builder() + .setDimension("user1", new String[]{"a"}) + .setDimension("user2", new String[]{"b"}) + .setDimension("user3", new String[]{"c"}) + .setDimension("user4", new String[]{"d"}) + .setDimension("user5", new String[]{"e"}) + .setDimension("user6", new String[]{"f"}) + .setDimension("user7", new String[]{"g"}) + .setDimension("user8", new String[]{"h"}) + .setDimension("user9", new String[]{"i"}) + .setDimension("user10", new String[]{"j"}) + .build("test-metric", 1234) + .build("test", "localhost") + .getCreatedTime() + ); + + Assert.assertEquals( + ImmutableMap.builder() + .put("feed", "metrics") + .put("timestamp", DateTimes.utc(42).toString()) + .put("service", "test") + .put("host", "localhost") + .put("metric", "test-metric") + .put("user1", "a") + .put("user2", "b") + .put("user3", "c") + .put("user4", "d") + .put("user5", "e") + .put("user6", "f") + .put("user7", "g") + .put("user8", "h") + .put("user9", "i") + .put("user10", "j") + .put("value", 1234) + .build(), + new ServiceMetricEvent.Builder() + .setDimension("user1", "a") + .setDimension("user2", "b") + .setDimension("user3", "c") + .setDimension("user4", "d") + .setDimension("user5", "e") + .setDimension("user6", "f") + .setDimension("user7", "g") + .setDimension("user8", "h") + .setDimension("user9", "i") + .setDimension("user10", "j") + .build(DateTimes.utc(42), "test-metric", 1234) + .build("test", "localhost") + .toMap() + ); + + Assert.assertEquals( + ImmutableMap.builder() + .put("feed", "metrics") + .put("timestamp", DateTimes.utc(42).toString()) + .put("service", "test") + .put("host", "localhost") + .put("metric", "test-metric") + .put("user1", Arrays.asList("a")) + .put("user2", Arrays.asList("b")) + .put("user3", Arrays.asList("c")) + .put("user4", Arrays.asList("d")) + .put("user5", Arrays.asList("e")) + .put("user6", Arrays.asList("f")) + .put("user7", Arrays.asList("g")) + .put("user8", Arrays.asList("h")) + .put("user9", Arrays.asList("i")) + .put("user10", Arrays.asList("j")) + .put("value", 1234) + .build(), + ServiceMetricEvent.builder() + .setDimension("user1", new String[]{"a"}) + .setDimension("user2", new String[]{"b"}) + .setDimension("user3", new String[]{"c"}) + .setDimension("user4", new String[]{"d"}) + .setDimension("user5", new String[]{"e"}) + .setDimension("user6", new String[]{"f"}) + .setDimension("user7", new String[]{"g"}) + .setDimension("user8", new String[]{"h"}) + .setDimension("user9", new String[]{"i"}) + .setDimension("user10", new String[]{"j"}) + .build(DateTimes.utc(42), "test-metric", 1234) + .build("test", "localhost") + .toMap() + ); + + Assert.assertEquals( + ImmutableMap.builder() + .put("feed", "metrics") + .put("timestamp", DateTimes.utc(42).toString()) + .put("service", "test") + .put("host", "localhost") + .put("metric", "test-metric") + .put("foo", "bar") + .put("baz", Arrays.asList("foo", "qux")) + .put("value", 1234) + .build(), + ServiceMetricEvent.builder() + .setDimension("foo", "bar") + .setDimension("baz", new String[]{"foo", "qux"}) + .build(DateTimes.utc(42), "test-metric", 1234) + .build("test", "localhost") + .toMap() + ); + } + + @Test(expected = IllegalStateException.class) + public void testInfinite() throws Exception + { + ServiceMetricEvent.builder().build("foo", 1 / 0d); + } + + @Test(expected = IllegalStateException.class) + public void testInfinite2() throws Exception + { + ServiceMetricEvent.builder().build("foo", 1 / 0f); + } + + + @Test(expected = IllegalStateException.class) + public void testNaN() throws Exception + { + ServiceMetricEvent.builder().build("foo", 0 / 0d); + } + + @Test(expected = IllegalStateException.class) + public void testNaN2() throws Exception + { + ServiceMetricEvent.builder().build("foo", 0 / 0f); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/service/UnitEvent.java b/java-util/src/test/java/io/druid/java/util/emitter/service/UnitEvent.java new file mode 100644 index 000000000000..91385961309c --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/emitter/service/UnitEvent.java @@ -0,0 +1,84 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package io.druid.java.util.emitter.service; + +import com.fasterxml.jackson.annotation.JsonValue; +import com.google.common.collect.ImmutableMap; +import io.druid.java.util.common.DateTimes; +import io.druid.java.util.emitter.core.Event; +import org.joda.time.DateTime; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + */ +public class UnitEvent implements Event +{ + private final String feed; + private final Number value; + private final Map dimensions; + private final DateTime createdTime; + + public UnitEvent(String feed, Number value) + { + this(feed, value, Collections.emptyMap()); + } + + public UnitEvent(String feed, Number value, Map dimensions) + { + this.feed = feed; + this.value = value; + this.dimensions = dimensions; + + createdTime = DateTimes.nowUtc(); + } + + @Override + @JsonValue + public Map toMap() + { + Map result = new HashMap<>(); + result.putAll(dimensions); + result.put("feed", feed); + result.put("metrics", ImmutableMap.of("value", value)); + return ImmutableMap.copyOf(result); + } + + @Override + public DateTime getCreatedTime() + { + return createdTime; + } + + @Override + public String getFeed() + { + return feed; + } + + @Override + public boolean isSafeToBuffer() + { + return true; + } +} diff --git a/java-util/src/test/java/io/druid/java/util/http/client/AsyncHttpClientTest.java b/java-util/src/test/java/io/druid/java/util/http/client/AsyncHttpClientTest.java new file mode 100644 index 000000000000..5650fc61976c --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/http/client/AsyncHttpClientTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client; + +import com.google.common.base.Charsets; +import io.druid.java.util.common.StringUtils; +import org.asynchttpclient.DefaultAsyncHttpClient; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public class AsyncHttpClientTest +{ + + @Test + public void testRequestTimeout() throws Exception + { + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final ServerSocket serverSocket = new ServerSocket(0); + exec.submit( + new Runnable() + { + @Override + public void run() + { + while (!Thread.currentThread().isInterrupted()) { + try ( + Socket clientSocket = serverSocket.accept(); + BufferedReader in = new BufferedReader( + new InputStreamReader(clientSocket.getInputStream(), StandardCharsets.UTF_8) + ); + OutputStream out = clientSocket.getOutputStream() + ) { + while (!in.readLine().equals("")) { + // skip lines + } + Thread.sleep(5000); // times out + out.write("HTTP/1.1 200 OK\r\nContent-Length: 6\r\n\r\nhello!".getBytes(Charsets.UTF_8)); + } + catch (Exception e) { + // Suppress + } + } + } + } + ); + + long requestStart = 0; + DefaultAsyncHttpClient client = new DefaultAsyncHttpClient(); + // Creation of connection for the first time takes long time, probably because of DNS cache or something like that + warmUp(serverSocket, client); + try { + requestStart = System.currentTimeMillis(); + Future future = client + .prepareGet(StringUtils.format("http://localhost:%d/", serverSocket.getLocalPort())) + .setRequestTimeout(2000) + .execute(); + System.out.println("created future in: " + (System.currentTimeMillis() - requestStart)); + future.get(3000, TimeUnit.MILLISECONDS); + Assert.fail("Expected timeout"); + } + catch (ExecutionException | TimeoutException e) { + long elapsed = System.currentTimeMillis() - requestStart; + // Within 10% of timeout + Assert.assertTrue("elapsed: " + elapsed, elapsed < 2200); + } + finally { + exec.shutdownNow(); + serverSocket.close(); + } + } + + private void warmUp(ServerSocket serverSocket, DefaultAsyncHttpClient client) + { + try { + Future future = client + .prepareGet(StringUtils.format("http://localhost:%d/", serverSocket.getLocalPort())) + .setRequestTimeout(100) + .execute(); + future.get(); + } + catch (Exception e) { + // ignore + } + } +} diff --git a/java-util/src/test/java/io/druid/java/util/http/client/FriendlyServersTest.java b/java-util/src/test/java/io/druid/java/util/http/client/FriendlyServersTest.java new file mode 100644 index 000000000000..1496c904bd9c --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/http/client/FriendlyServersTest.java @@ -0,0 +1,309 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client; + +import com.google.common.base.Charsets; +import com.google.common.util.concurrent.ListenableFuture; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.lifecycle.Lifecycle; +import io.druid.java.util.http.client.response.StatusResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHolder; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.SecureRequestCustomizer; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.util.ssl.SslContextFactory; +import org.jboss.netty.channel.ChannelException; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLHandshakeException; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Tests with servers that are at least moderately well-behaving. + */ +public class FriendlyServersTest +{ + @Test + public void testFriendlyHttpServer() throws Exception + { + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final ServerSocket serverSocket = new ServerSocket(0); + exec.submit( + new Runnable() + { + @Override + public void run() + { + while (!Thread.currentThread().isInterrupted()) { + try ( + Socket clientSocket = serverSocket.accept(); + BufferedReader in = new BufferedReader( + new InputStreamReader(clientSocket.getInputStream(), StandardCharsets.UTF_8) + ); + OutputStream out = clientSocket.getOutputStream() + ) { + while (!in.readLine().equals("")) { + // skip lines + } + out.write("HTTP/1.1 200 OK\r\nContent-Length: 6\r\n\r\nhello!".getBytes(Charsets.UTF_8)); + } + catch (Exception e) { + // Suppress + } + } + } + } + ); + + final Lifecycle lifecycle = new Lifecycle(); + try { + final HttpClientConfig config = HttpClientConfig.builder().build(); + final HttpClient client = HttpClientInit.createClient(config, lifecycle); + final StatusResponseHolder response = client + .go( + new Request(HttpMethod.GET, new URL(StringUtils.format("http://localhost:%d/", serverSocket.getLocalPort()))), + new StatusResponseHandler(Charsets.UTF_8) + ).get(); + + Assert.assertEquals(200, response.getStatus().getCode()); + Assert.assertEquals("hello!", response.getContent()); + } + finally { + exec.shutdownNow(); + serverSocket.close(); + lifecycle.stop(); + } + } + + @Test + public void testCompressionCodecConfig() throws Exception + { + final ExecutorService exec = Executors.newSingleThreadExecutor(); + final ServerSocket serverSocket = new ServerSocket(0); + final AtomicBoolean foundAcceptEncoding = new AtomicBoolean(); + exec.submit( + new Runnable() + { + @Override + public void run() + { + while (!Thread.currentThread().isInterrupted()) { + try ( + Socket clientSocket = serverSocket.accept(); + BufferedReader in = new BufferedReader( + new InputStreamReader(clientSocket.getInputStream(), StandardCharsets.UTF_8) + ); + OutputStream out = clientSocket.getOutputStream() + ) { + // Read headers + String header; + while (!(header = in.readLine()).equals("")) { + if (header.equals("Accept-Encoding: identity")) { + foundAcceptEncoding.set(true); + } + } + out.write("HTTP/1.1 200 OK\r\nContent-Length: 6\r\n\r\nhello!".getBytes(Charsets.UTF_8)); + } + catch (Exception e) { + // Suppress + } + } + } + } + ); + + final Lifecycle lifecycle = new Lifecycle(); + try { + final HttpClientConfig config = HttpClientConfig.builder() + .withCompressionCodec(HttpClientConfig.CompressionCodec.IDENTITY) + .build(); + final HttpClient client = HttpClientInit.createClient(config, lifecycle); + final StatusResponseHolder response = client + .go( + new Request(HttpMethod.GET, new URL(StringUtils.format("http://localhost:%d/", serverSocket.getLocalPort()))), + new StatusResponseHandler(Charsets.UTF_8) + ).get(); + + Assert.assertEquals(200, response.getStatus().getCode()); + Assert.assertEquals("hello!", response.getContent()); + Assert.assertTrue(foundAcceptEncoding.get()); + } + finally { + exec.shutdownNow(); + serverSocket.close(); + lifecycle.stop(); + } + } + + @Test + public void testFriendlySelfSignedHttpsServer() throws Exception + { + final Lifecycle lifecycle = new Lifecycle(); + final String keyStorePath = getClass().getClassLoader().getResource("keystore.jks").getFile(); + Server server = new Server(); + + HttpConfiguration https = new HttpConfiguration(); + https.addCustomizer(new SecureRequestCustomizer()); + + SslContextFactory sslContextFactory = new SslContextFactory(); + sslContextFactory.setKeyStorePath(keyStorePath); + sslContextFactory.setKeyStorePassword("abc123"); + sslContextFactory.setKeyManagerPassword("abc123"); + + ServerConnector sslConnector = new ServerConnector( + server, + new SslConnectionFactory(sslContextFactory, "http/1.1"), + new HttpConnectionFactory(https) + ); + + sslConnector.setPort(0); + server.setConnectors(new Connector[]{sslConnector}); + server.start(); + + try { + final SSLContext mySsl = HttpClientInit.sslContextWithTrustedKeyStore(keyStorePath, "abc123"); + final HttpClientConfig trustingConfig = HttpClientConfig.builder().withSslContext(mySsl).build(); + final HttpClient trustingClient = HttpClientInit.createClient(trustingConfig, lifecycle); + + final HttpClientConfig skepticalConfig = HttpClientConfig.builder() + .withSslContext(SSLContext.getDefault()) + .build(); + final HttpClient skepticalClient = HttpClientInit.createClient(skepticalConfig, lifecycle); + + // Correct name ("localhost") + { + final HttpResponseStatus status = trustingClient + .go( + new Request( + HttpMethod.GET, + new URL(StringUtils.format("https://localhost:%d/", sslConnector.getLocalPort())) + ), + new StatusResponseHandler(Charsets.UTF_8) + ).get().getStatus(); + Assert.assertEquals(404, status.getCode()); + } + + // Incorrect name ("127.0.0.1") + { + final ListenableFuture response1 = trustingClient + .go( + new Request( + HttpMethod.GET, + new URL(StringUtils.format("https://127.0.0.1:%d/", sslConnector.getLocalPort())) + ), + new StatusResponseHandler(Charsets.UTF_8) + ); + + Throwable ea = null; + try { + response1.get(); + } + catch (ExecutionException e) { + ea = e.getCause(); + } + + Assert.assertTrue("ChannelException thrown by 'get'", ea instanceof ChannelException); + Assert.assertTrue("Expected error message", ea.getCause().getMessage().matches(".*Failed to handshake.*")); + } + + { + // Untrusting client + final ListenableFuture response2 = skepticalClient + .go( + new Request( + HttpMethod.GET, new URL(StringUtils.format("https://localhost:%d/", sslConnector.getLocalPort())) + ), + new StatusResponseHandler(Charsets.UTF_8) + ); + + Throwable eb = null; + try { + response2.get(); + } + catch (ExecutionException e) { + eb = e.getCause(); + } + Assert.assertNotNull("ChannelException thrown by 'get'", eb); + Assert.assertTrue( + "Root cause is SSLHandshakeException", + eb.getCause().getCause() instanceof SSLHandshakeException + ); + } + } + finally { + lifecycle.stop(); + server.stop(); + } + } + + @Test + @Ignore + public void testHttpBin() throws Throwable + { + final Lifecycle lifecycle = new Lifecycle(); + try { + final HttpClientConfig config = HttpClientConfig.builder().withSslContext(SSLContext.getDefault()).build(); + final HttpClient client = HttpClientInit.createClient(config, lifecycle); + + { + final HttpResponseStatus status = client + .go( + new Request(HttpMethod.GET, new URL("https://httpbin.org/get")), + new StatusResponseHandler(Charsets.UTF_8) + ).get().getStatus(); + + Assert.assertEquals(200, status.getCode()); + } + + { + final HttpResponseStatus status = client + .go( + new Request(HttpMethod.POST, new URL("https://httpbin.org/post")) + .setContent(new byte[]{'a', 'b', 'c', 1, 2, 3}), + new StatusResponseHandler(Charsets.UTF_8) + ).get().getStatus(); + + Assert.assertEquals(200, status.getCode()); + } + } + finally { + lifecycle.stop(); + } + } +} diff --git a/java-util/src/test/java/io/druid/java/util/http/client/JankyServersTest.java b/java-util/src/test/java/io/druid/java/util/http/client/JankyServersTest.java new file mode 100644 index 000000000000..1923ba9fb6bd --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/http/client/JankyServersTest.java @@ -0,0 +1,356 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client; + +import com.google.common.base.Charsets; +import com.google.common.util.concurrent.ListenableFuture; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.lifecycle.Lifecycle; +import io.druid.java.util.http.client.response.StatusResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHolder; +import org.jboss.netty.channel.ChannelException; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.timeout.ReadTimeoutException; +import org.joda.time.Duration; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import javax.net.ssl.SSLContext; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.URL; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Tests with a bunch of goofy not-actually-http servers. + */ +public class JankyServersTest +{ + static ExecutorService exec; + static ServerSocket silentServerSocket; + static ServerSocket echoServerSocket; + static ServerSocket closingServerSocket; + + @BeforeClass + public static void setUp() throws Exception + { + exec = Executors.newCachedThreadPool(); + + silentServerSocket = new ServerSocket(0); + echoServerSocket = new ServerSocket(0); + closingServerSocket = new ServerSocket(0); + + exec.submit( + new Runnable() + { + @Override + public void run() + { + while (!Thread.currentThread().isInterrupted()) { + try ( + Socket clientSocket = silentServerSocket.accept(); + InputStream in = clientSocket.getInputStream() + ) { + while (in.read() != -1) { + } + } + catch (Exception e) { + // Suppress + } + } + } + } + ); + + exec.submit( + new Runnable() + { + @Override + public void run() + { + while (!Thread.currentThread().isInterrupted()) { + try ( + Socket clientSocket = closingServerSocket.accept(); + InputStream in = clientSocket.getInputStream() + ) { + in.read(); + clientSocket.close(); + } + catch (Exception e) { + // Suppress + } + } + } + } + ); + + exec.submit( + new Runnable() + { + @Override + public void run() + { + while (!Thread.currentThread().isInterrupted()) { + try ( + Socket clientSocket = echoServerSocket.accept(); + OutputStream out = clientSocket.getOutputStream(); + InputStream in = clientSocket.getInputStream() + ) { + int b; + while ((b = in.read()) != -1) { + out.write(b); + } + } + catch (Exception e) { + // Suppress + } + } + } + } + ); + } + + @AfterClass + public static void tearDown() throws Exception + { + exec.shutdownNow(); + silentServerSocket.close(); + echoServerSocket.close(); + closingServerSocket.close(); + } + + @Test + public void testHttpSilentServerWithGlobalTimeout() throws Throwable + { + final Lifecycle lifecycle = new Lifecycle(); + try { + final HttpClientConfig config = HttpClientConfig.builder().withReadTimeout(new Duration(100)).build(); + final HttpClient client = HttpClientInit.createClient(config, lifecycle); + final ListenableFuture future = client + .go( + new Request(HttpMethod.GET, new URL(StringUtils.format("http://localhost:%d/", silentServerSocket.getLocalPort()))), + new StatusResponseHandler(Charsets.UTF_8) + ); + + Throwable e = null; + try { + future.get(); + } + catch (ExecutionException e1) { + e = e1.getCause(); + } + + Assert.assertTrue("ReadTimeoutException thrown by 'get'", e instanceof ReadTimeoutException); + } + finally { + lifecycle.stop(); + } + } + + @Test + public void testHttpSilentServerWithRequestTimeout() throws Throwable + { + final Lifecycle lifecycle = new Lifecycle(); + try { + final HttpClientConfig config = HttpClientConfig.builder().withReadTimeout(new Duration(86400L * 365)).build(); + final HttpClient client = HttpClientInit.createClient(config, lifecycle); + final ListenableFuture future = client + .go( + new Request(HttpMethod.GET, new URL(StringUtils.format("http://localhost:%d/", silentServerSocket.getLocalPort()))), + new StatusResponseHandler(Charsets.UTF_8), + new Duration(100L) + ); + + Throwable e = null; + try { + future.get(); + } + catch (ExecutionException e1) { + e = e1.getCause(); + } + + Assert.assertTrue("ReadTimeoutException thrown by 'get'", e instanceof ReadTimeoutException); + } + finally { + lifecycle.stop(); + } + } + + @Test + public void testHttpsSilentServer() throws Throwable + { + final Lifecycle lifecycle = new Lifecycle(); + try { + final HttpClientConfig config = HttpClientConfig.builder() + .withSslContext(SSLContext.getDefault()) + .withSslHandshakeTimeout(new Duration(100)) + .build(); + final HttpClient client = HttpClientInit.createClient(config, lifecycle); + + final ListenableFuture response = client + .go( + new Request(HttpMethod.GET, new URL(StringUtils.format("https://localhost:%d/", silentServerSocket.getLocalPort()))), + new StatusResponseHandler(Charsets.UTF_8) + ); + + Throwable e = null; + try { + response.get(); + } + catch (ExecutionException e1) { + e = e1.getCause(); + } + + Assert.assertTrue("ChannelException thrown by 'get'", e instanceof ChannelException); + } + finally { + lifecycle.stop(); + } + } + + @Test + public void testHttpConnectionClosingServer() throws Throwable + { + final Lifecycle lifecycle = new Lifecycle(); + try { + final HttpClientConfig config = HttpClientConfig.builder().build(); + final HttpClient client = HttpClientInit.createClient(config, lifecycle); + final ListenableFuture response = client + .go( + new Request(HttpMethod.GET, new URL(StringUtils.format("http://localhost:%d/", closingServerSocket.getLocalPort()))), + new StatusResponseHandler(Charsets.UTF_8) + ); + Throwable e = null; + try { + response.get(); + } + catch (ExecutionException e1) { + e = e1.getCause(); + e1.printStackTrace(); + } + + Assert.assertTrue("ChannelException thrown by 'get'", isChannelClosedException(e)); + } + finally { + lifecycle.stop(); + } + } + + @Test + public void testHttpsConnectionClosingServer() throws Throwable + { + final Lifecycle lifecycle = new Lifecycle(); + try { + final HttpClientConfig config = HttpClientConfig.builder().withSslContext(SSLContext.getDefault()).build(); + final HttpClient client = HttpClientInit.createClient(config, lifecycle); + + final ListenableFuture response = client + .go( + new Request(HttpMethod.GET, new URL(StringUtils.format("https://localhost:%d/", closingServerSocket.getLocalPort()))), + new StatusResponseHandler(Charsets.UTF_8) + ); + + Throwable e = null; + try { + response.get(); + } + catch (ExecutionException e1) { + e = e1.getCause(); + e1.printStackTrace(); + } + + Assert.assertTrue("ChannelException thrown by 'get'", isChannelClosedException(e)); + } + finally { + lifecycle.stop(); + } + } + + public boolean isChannelClosedException(Throwable e) + { + return e instanceof ChannelException || + (e instanceof IOException && e.getMessage().matches(".*Connection reset by peer.*")); + } + + @Test + public void testHttpEchoServer() throws Throwable + { + final Lifecycle lifecycle = new Lifecycle(); + try { + final HttpClientConfig config = HttpClientConfig.builder().build(); + final HttpClient client = HttpClientInit.createClient(config, lifecycle); + final ListenableFuture response = client + .go( + new Request(HttpMethod.GET, new URL(StringUtils.format("http://localhost:%d/", echoServerSocket.getLocalPort()))), + new StatusResponseHandler(Charsets.UTF_8) + ); + + Throwable e = null; + try { + response.get(); + } + catch (ExecutionException e1) { + e = e1.getCause(); + } + + Assert.assertTrue("IllegalArgumentException thrown by 'get'", e instanceof IllegalArgumentException); + Assert.assertTrue("Expected error message", e.getMessage().matches(".*invalid version format:.*")); + } + finally { + lifecycle.stop(); + } + } + + @Test + public void testHttpsEchoServer() throws Throwable + { + final Lifecycle lifecycle = new Lifecycle(); + try { + final HttpClientConfig config = HttpClientConfig.builder().withSslContext(SSLContext.getDefault()).build(); + final HttpClient client = HttpClientInit.createClient(config, lifecycle); + + final ListenableFuture response = client + .go( + new Request(HttpMethod.GET, new URL(StringUtils.format("https://localhost:%d/", echoServerSocket.getLocalPort()))), + new StatusResponseHandler(Charsets.UTF_8) + ); + + Throwable e = null; + try { + response.get(); + } + catch (ExecutionException e1) { + e = e1.getCause(); + } + + Assert.assertNotNull("ChannelException thrown by 'get'", e); + } + finally { + lifecycle.stop(); + } + } +} diff --git a/java-util/src/test/java/io/druid/java/util/http/client/io/AppendableByteArrayInputStreamTest.java b/java-util/src/test/java/io/druid/java/util/http/client/io/AppendableByteArrayInputStreamTest.java new file mode 100644 index 000000000000..4289e8104686 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/http/client/io/AppendableByteArrayInputStreamTest.java @@ -0,0 +1,261 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.io; + +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.logger.Logger; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicReference; + +/** + */ +public class AppendableByteArrayInputStreamTest +{ + private static final Logger log = new Logger(AppendableByteArrayInputStreamTest.class); + + @Test + public void testSingleByteArray() throws Exception + { + byte[][] bytesToWrite = new byte[][]{{0, 1, 2, 3, 4, 5, 6}}; + + testAll(bytesToWrite, bytesToWrite[0]); + } + + @Test + public void testMultiByteArray() throws Exception + { + byte[] expectedBytes = new byte[]{0, 1, 2, 3, 4, 5, 6}; + + testAll(new byte[][]{{0, 1, 2, 3}, {4, 5, 6}}, expectedBytes); + testAll(new byte[][]{{0, 1}, {2, 3}, {4, 5, 6}}, expectedBytes); + testAll(new byte[][]{{0}, {1}, {2}, {3}, {4}, {5}, {6}}, expectedBytes); + } + + public void testAll(byte[][] writtenBytes, byte[] expectedBytes) throws Exception + { + testFullRead(writtenBytes, expectedBytes); + testIndividualRead(writtenBytes, expectedBytes); + } + + public void testIndividualRead(byte[][] writtenBytes, byte[] expectedBytes) throws IOException + { + AppendableByteArrayInputStream in = new AppendableByteArrayInputStream(); + + for (byte[] writtenByte : writtenBytes) { + in.add(writtenByte); + } + + for (int i = 0; i < expectedBytes.length; i++) { + final int readByte = in.read(); + if (expectedBytes[i] != (byte) readByte) { + Assert.assertEquals(StringUtils.format("%s[%d]", Arrays.toString(expectedBytes), i), expectedBytes[i], readByte); + } + } + } + + public void testFullRead(byte[][] writtenBytes, byte[] expectedBytes) throws IOException + { + AppendableByteArrayInputStream in = new AppendableByteArrayInputStream(); + byte[] readBytes = new byte[expectedBytes.length]; + + for (byte[] writtenByte : writtenBytes) { + in.add(writtenByte); + } + Assert.assertEquals(readBytes.length, in.read(readBytes)); + Assert.assertArrayEquals(expectedBytes, readBytes); + } + + @Test + public void testReadsAndWritesInterspersed() throws Exception + { + AppendableByteArrayInputStream in = new AppendableByteArrayInputStream(); + + in.add(new byte[]{0, 1, 2}); + + byte[] readBytes = new byte[3]; + Assert.assertEquals(3, in.read(readBytes)); + Assert.assertArrayEquals(new byte[]{0, 1, 2}, readBytes); + + in.add(new byte[]{3, 4}); + in.add(new byte[]{5, 6, 7}); + + readBytes = new byte[5]; + Assert.assertEquals(5, in.read(readBytes)); + Assert.assertArrayEquals(new byte[]{3, 4, 5, 6, 7}, readBytes); + } + + @Test + public void testReadLessThanWritten() throws Exception + { + AppendableByteArrayInputStream in = new AppendableByteArrayInputStream(); + + in.add(new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + + byte[] readBytes = new byte[4]; + + Assert.assertEquals(4, in.read(readBytes)); + Assert.assertArrayEquals(new byte[]{0, 1, 2, 3}, readBytes); + + Assert.assertEquals(4, in.read(readBytes)); + Assert.assertArrayEquals(new byte[]{4, 5, 6, 7}, readBytes); + + Assert.assertEquals(2, in.read(readBytes, 0, 2)); + Assert.assertArrayEquals(new byte[]{8, 9, 6, 7}, readBytes); + } + + @Test + public void testReadLessThanWrittenMultiple() throws Exception + { + AppendableByteArrayInputStream in = new AppendableByteArrayInputStream(); + + in.add(new byte[]{0, 1, 2}); + in.add(new byte[]{3, 4, 5}); + in.add(new byte[]{6, 7}); + in.add(new byte[]{8, 9}); + + byte[] readBytes = new byte[4]; + + Assert.assertEquals(4, in.read(readBytes)); + Assert.assertArrayEquals(new byte[]{0, 1, 2, 3}, readBytes); + + Assert.assertEquals(4, in.read(readBytes)); + Assert.assertArrayEquals(new byte[]{4, 5, 6, 7}, readBytes); + + Assert.assertEquals(2, in.read(readBytes, 0, 2)); + Assert.assertArrayEquals(new byte[]{8, 9, 6, 7}, readBytes); + } + + @Test + public void testBlockingRead() throws Exception + { + final AppendableByteArrayInputStream in = new AppendableByteArrayInputStream(); + + in.add(new byte[]{0, 1, 2, 3, 4}); + + Assert.assertEquals(5, in.available()); + + Future bytesFuture = Executors.newSingleThreadExecutor().submit( + new Callable() + { + @Override + public byte[] call() throws Exception + { + byte[] readBytes = new byte[10]; + in.read(readBytes); + return readBytes; + } + } + ); + + int count = 0; + while (in.available() != 0) { + if (count >= 100) { + Assert.fail("available didn't become 0 fast enough."); + } + count++; + Thread.sleep(10); + } + + in.add(new byte[]{5, 6, 7, 8, 9, 10}); + + count = 0; + while (in.available() != 1) { + if (count >= 100) { + Assert.fail("available didn't become 1 fast enough."); + } + count++; + Thread.sleep(10); + } + + Assert.assertArrayEquals(new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, bytesFuture.get()); + Assert.assertEquals(10, in.read()); + Assert.assertEquals(0, in.available()); + } + + @Test + public void testAddEmptyByteArray() throws Exception + { + AppendableByteArrayInputStream in = new AppendableByteArrayInputStream(); + + in.add(new byte[]{}); + in.add(new byte[]{1}); + in.add(new byte[]{}); + in.done(); + + Assert.assertEquals(1, in.available()); + Assert.assertEquals(1, in.read()); + Assert.assertEquals(0, in.available()); + Assert.assertEquals(-1, in.read()); + } + + @Test + public void testExceptionUnblocks() throws InterruptedException + { + final AppendableByteArrayInputStream in = new AppendableByteArrayInputStream(); + in.add(new byte[]{}); + in.add(new byte[]{1}); + in.add(new byte[]{}); + final AtomicReference exceptionThrown = new AtomicReference(); + final CountDownLatch latch = new CountDownLatch(1); + Executors.newSingleThreadExecutor().submit( + new Callable() + { + @Override + public byte[] call() throws Exception + { + try { + byte[] readBytes = new byte[10]; + while (in.read(readBytes) != -1) { + } + return readBytes; + } + catch (IOException e) { + exceptionThrown.set(e); + latch.countDown(); + } + return null; + } + } + ); + + Exception expected = new Exception(); + in.exceptionCaught(expected); + + latch.await(); + Assert.assertEquals(expected, exceptionThrown.get().getCause()); + + try { + in.read(); + Assert.fail(); + } + catch (IOException thrown) { + Assert.assertEquals(expected, thrown.getCause()); + } + + } +} diff --git a/java-util/src/test/java/io/druid/java/util/http/client/pool/ResourcePoolTest.java b/java-util/src/test/java/io/druid/java/util/http/client/pool/ResourcePoolTest.java new file mode 100644 index 000000000000..b6167dcd831e --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/http/client/pool/ResourcePoolTest.java @@ -0,0 +1,328 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.pool; + +import io.druid.java.util.common.ISE; +import org.easymock.EasyMock; +import org.easymock.IAnswer; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + */ +public class ResourcePoolTest +{ + ResourceFactory resourceFactory; + ResourcePool pool; + + @Before + public void setUp() throws Exception + { + resourceFactory = (ResourceFactory) EasyMock.createMock(ResourceFactory.class); + + EasyMock.replay(resourceFactory); + pool = new ResourcePool( + resourceFactory, + new ResourcePoolConfig(2, TimeUnit.MINUTES.toMillis(4)) + ); + + EasyMock.verify(resourceFactory); + EasyMock.reset(resourceFactory); + } + + @Test + public void testSanity() throws Exception + { + primePool(); + EasyMock.replay(resourceFactory); + } + + private void primePool() + { + EasyMock.expect(resourceFactory.generate("billy")).andAnswer(new StringIncrementingAnswer("billy")).times(2); + EasyMock.expect(resourceFactory.generate("sally")).andAnswer(new StringIncrementingAnswer("sally")).times(2); + EasyMock.expect(resourceFactory.isGood("billy0")).andReturn(true).times(1); + EasyMock.expect(resourceFactory.isGood("sally0")).andReturn(true).times(1); + EasyMock.replay(resourceFactory); + + ResourceContainer billyString = pool.take("billy"); + ResourceContainer sallyString = pool.take("sally"); + Assert.assertEquals("billy0", billyString.get()); + Assert.assertEquals("sally0", sallyString.get()); + + EasyMock.verify(resourceFactory); + EasyMock.reset(resourceFactory); + + billyString.returnResource(); + sallyString.returnResource(); + } + + @Test + public void testFailedResource() throws Exception + { + primePool(); + + EasyMock.expect(resourceFactory.isGood("billy1")).andReturn(false).times(1); + resourceFactory.close("billy1"); + EasyMock.expectLastCall(); + EasyMock.expect(resourceFactory.generate("billy")).andReturn("billy2").times(1); + EasyMock.replay(resourceFactory); + + ResourceContainer billy = pool.take("billy"); + Assert.assertEquals("billy2", billy.get()); + billy.returnResource(); + + EasyMock.verify(resourceFactory); + EasyMock.reset(resourceFactory); + } + + @Test + public void testFaultyFailedResourceReplacement() throws Exception + { + primePool(); + + EasyMock.expect(resourceFactory.isGood("billy1")).andReturn(false).times(1); + resourceFactory.close("billy1"); + EasyMock.expectLastCall(); + EasyMock.expect(resourceFactory.generate("billy")).andThrow(new ISE("where's billy?")).times(1); + EasyMock.expect(resourceFactory.isGood("billy0")).andReturn(false).times(1); + resourceFactory.close("billy0"); + EasyMock.expectLastCall(); + EasyMock.expect(resourceFactory.generate("billy")).andThrow(new ISE("where's billy?")).times(1); + EasyMock.expect(resourceFactory.generate("billy")).andReturn("billy2").times(1); + EasyMock.replay(resourceFactory); + + IllegalStateException e1 = null; + try { + pool.take("billy"); + } + catch (IllegalStateException e) { + e1 = e; + } + Assert.assertNotNull("exception", e1); + Assert.assertEquals("where's billy?", e1.getMessage()); + + IllegalStateException e2 = null; + try { + pool.take("billy"); + } + catch (IllegalStateException e) { + e2 = e; + } + Assert.assertNotNull("exception", e2); + Assert.assertEquals("where's billy?", e2.getMessage()); + + ResourceContainer billy = pool.take("billy"); + Assert.assertEquals("billy2", billy.get()); + billy.returnResource(); + + EasyMock.verify(resourceFactory); + EasyMock.reset(resourceFactory); + } + + @Test + public void testTakeMoreThanAllowed() throws Exception + { + primePool(); + EasyMock.expect(resourceFactory.isGood("billy1")).andReturn(true).times(1); + EasyMock.expect(resourceFactory.isGood("billy0")).andReturn(true).times(1); + EasyMock.replay(resourceFactory); + + CountDownLatch latch1 = new CountDownLatch(1); + CountDownLatch latch2 = new CountDownLatch(1); + CountDownLatch latch3 = new CountDownLatch(1); + + MyThread billy1Thread = new MyThread(latch1, "billy"); + billy1Thread.start(); + billy1Thread.waitForValueToBeGotten(1, TimeUnit.SECONDS); + MyThread billy0Thread = new MyThread(latch2, "billy"); + billy0Thread.start(); + billy0Thread.waitForValueToBeGotten(1, TimeUnit.SECONDS); + + MyThread blockedThread = new MyThread(latch3, "billy"); + blockedThread.start(); + + EasyMock.verify(resourceFactory); + EasyMock.reset(resourceFactory); + EasyMock.expect(resourceFactory.isGood("billy0")).andReturn(true).times(1); + EasyMock.replay(resourceFactory); + + latch2.countDown(); + blockedThread.waitForValueToBeGotten(1, TimeUnit.SECONDS); + + EasyMock.verify(resourceFactory); + EasyMock.reset(resourceFactory); + + latch1.countDown(); + latch3.countDown(); + + Assert.assertEquals("billy1", billy1Thread.getValue()); + Assert.assertEquals("billy0", billy0Thread.getValue()); + Assert.assertEquals("billy0", blockedThread.getValue()); + } + + @Test + public void testCloseUnblocks() throws InterruptedException + { + primePool(); + EasyMock.expect(resourceFactory.isGood("billy1")).andReturn(true).times(1); + EasyMock.expect(resourceFactory.isGood("billy0")).andReturn(true).times(1); + resourceFactory.close("sally1"); + EasyMock.expectLastCall().times(1); + resourceFactory.close("sally0"); + EasyMock.expectLastCall().times(1); + EasyMock.replay(resourceFactory); + CountDownLatch latch1 = new CountDownLatch(1); + CountDownLatch latch2 = new CountDownLatch(1); + CountDownLatch latch3 = new CountDownLatch(1); + + MyThread billy1Thread = new MyThread(latch1, "billy"); + billy1Thread.start(); + billy1Thread.waitForValueToBeGotten(1, TimeUnit.SECONDS); + MyThread billy0Thread = new MyThread(latch2, "billy"); + billy0Thread.start(); + billy0Thread.waitForValueToBeGotten(1, TimeUnit.SECONDS); + + MyThread blockedThread = new MyThread(latch3, "billy"); + blockedThread.start(); + blockedThread.waitForValueToBeGotten(1, TimeUnit.SECONDS); + pool.close(); + + + EasyMock.verify(resourceFactory); + EasyMock.reset(resourceFactory); + EasyMock.replay(resourceFactory); + + latch2.countDown(); + blockedThread.waitForValueToBeGotten(1, TimeUnit.SECONDS); + + EasyMock.verify(resourceFactory); + EasyMock.reset(resourceFactory); + + latch1.countDown(); + latch3.countDown(); + + Assert.assertEquals("billy1", billy1Thread.getValue()); + Assert.assertEquals("billy0", billy0Thread.getValue()); + blockedThread.join(); + // pool returns null after close + Assert.assertEquals(null, blockedThread.getValue()); + } + + @Test + public void testTimedOutResource() throws Exception + { + resourceFactory = (ResourceFactory) EasyMock.createMock(ResourceFactory.class); + + pool = new ResourcePool( + resourceFactory, + new ResourcePoolConfig(2, TimeUnit.MILLISECONDS.toMillis(10)) + ); + + EasyMock.expect(resourceFactory.generate("billy")).andAnswer(new StringIncrementingAnswer("billy")).times(2); + EasyMock.expect(resourceFactory.isGood("billy0")).andReturn(true).times(1); + EasyMock.replay(resourceFactory); + + ResourceContainer billyString = pool.take("billy"); + Assert.assertEquals("billy0", billyString.get()); + + EasyMock.verify(resourceFactory); + EasyMock.reset(resourceFactory); + + billyString.returnResource(); + + //make sure resources have been timed out. + Thread.sleep(100); + + EasyMock.expect(resourceFactory.generate("billy")).andReturn("billy1").times(1); + resourceFactory.close("billy1"); + EasyMock.expect(resourceFactory.isGood("billy1")).andReturn(true).times(1); + EasyMock.replay(resourceFactory); + + ResourceContainer billy = pool.take("billy"); + Assert.assertEquals("billy1", billy.get()); + billy.returnResource(); + + EasyMock.verify(resourceFactory); + EasyMock.reset(resourceFactory); + } + + private static class StringIncrementingAnswer implements IAnswer + { + int count = 0; + private String string; + + public StringIncrementingAnswer(String string) + { + this.string = string; + } + + @Override + public String answer() throws Throwable + { + return string + count++; + } + } + + private class MyThread extends Thread + { + private final CountDownLatch gotValueLatch = new CountDownLatch(1); + + private final CountDownLatch latch1; + private String resourceName; + + volatile String value = null; + + public MyThread(CountDownLatch latch1, String resourceName) + { + this.latch1 = latch1; + this.resourceName = resourceName; + } + + @Override + public void run() + { + ResourceContainer resourceContainer = pool.take(resourceName); + value = resourceContainer.get(); + gotValueLatch.countDown(); + try { + latch1.await(); + } + catch (InterruptedException e) { + + } + resourceContainer.returnResource(); + } + + public String getValue() + { + return value; + } + + public void waitForValueToBeGotten(long length, TimeUnit timeUnit) throws InterruptedException + { + gotValueLatch.await(length, timeUnit); + } + } +} diff --git a/java-util/src/test/java/io/druid/java/util/http/client/response/SequenceInputStreamResponseHandlerTest.java b/java-util/src/test/java/io/druid/java/util/http/client/response/SequenceInputStreamResponseHandlerTest.java new file mode 100644 index 000000000000..22554743d229 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/http/client/response/SequenceInputStreamResponseHandlerTest.java @@ -0,0 +1,226 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.http.client.response; + +import org.jboss.netty.buffer.BigEndianHeapChannelBuffer; +import org.jboss.netty.handler.codec.http.DefaultHttpChunk; +import org.jboss.netty.handler.codec.http.DefaultHttpResponse; +import org.jboss.netty.handler.codec.http.HttpResponse; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.jboss.netty.handler.codec.http.HttpVersion; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.Random; + +public class SequenceInputStreamResponseHandlerTest +{ + private static final int TOTAL_BYTES = 1 << 10; + private static final ArrayList BYTE_LIST = new ArrayList<>(); + private static final Random RANDOM = new Random(378134789L); + private static byte[] allBytes = new byte[TOTAL_BYTES]; + + @BeforeClass + public static void setUp() + { + final ByteBuffer buffer = ByteBuffer.wrap(allBytes); + while (buffer.hasRemaining()) { + final byte[] bytes = new byte[Math.min(Math.abs(RANDOM.nextInt()) % 128, buffer.remaining())]; + RANDOM.nextBytes(bytes); + buffer.put(bytes); + BYTE_LIST.add(bytes); + } + } + + @AfterClass + public static void tearDown() + { + BYTE_LIST.clear(); + allBytes = null; + } + + private static void fillBuff(InputStream inputStream, byte[] buff) throws IOException + { + int off = 0; + while (off < buff.length) { + final int read = inputStream.read(buff, off, buff.length - off); + if (read < 0) { + throw new IOException("Unexpected end of stream"); + } + off += read; + } + } + + @Test(expected = TesterException.class) + public void testExceptionalChunkedStream() throws IOException + { + Iterator it = BYTE_LIST.iterator(); + + SequenceInputStreamResponseHandler responseHandler = new SequenceInputStreamResponseHandler(); + final HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK); + response.setChunked(true); + ClientResponse clientResponse = responseHandler.handleResponse(response); + final int failAt = Math.abs(RANDOM.nextInt()) % allBytes.length; + while (it.hasNext()) { + final DefaultHttpChunk chunk = new DefaultHttpChunk( + new BigEndianHeapChannelBuffer(it.next()) + { + @Override + public void getBytes(int index, byte[] dst, int dstIndex, int length) + { + if (dstIndex + length >= failAt) { + throw new TesterException(); + } + super.getBytes(index, dst, dstIndex, length); + } + } + ); + clientResponse = responseHandler.handleChunk(clientResponse, chunk); + } + clientResponse = responseHandler.done(clientResponse); + + final InputStream stream = clientResponse.getObj(); + final byte[] buff = new byte[allBytes.length]; + fillBuff(stream, buff); + } + + public static class TesterException extends RuntimeException + { + } + + @Test(expected = TesterException.class) + public void testExceptionalSingleStream() throws IOException + { + SequenceInputStreamResponseHandler responseHandler = new SequenceInputStreamResponseHandler(); + final HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK); + response.setChunked(false); + response.setContent( + new BigEndianHeapChannelBuffer(allBytes) + { + @Override + public void getBytes(int index, byte[] dst, int dstIndex, int length) + { + if (dstIndex + length >= allBytes.length) { + throw new TesterException(); + } + super.getBytes(index, dst, dstIndex, length); + } + } + ); + ClientResponse clientResponse = responseHandler.handleResponse(response); + clientResponse = responseHandler.done(clientResponse); + + final InputStream stream = clientResponse.getObj(); + final byte[] buff = new byte[allBytes.length]; + fillBuff(stream, buff); + } + + @Test + public void simpleMultiStreamTest() throws IOException + { + Iterator it = BYTE_LIST.iterator(); + + SequenceInputStreamResponseHandler responseHandler = new SequenceInputStreamResponseHandler(); + final HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK); + response.setChunked(true); + ClientResponse clientResponse = responseHandler.handleResponse(response); + while (it.hasNext()) { + final DefaultHttpChunk chunk = new DefaultHttpChunk(new BigEndianHeapChannelBuffer(it.next())); + clientResponse = responseHandler.handleChunk(clientResponse, chunk); + } + clientResponse = responseHandler.done(clientResponse); + + final InputStream stream = clientResponse.getObj(); + final InputStream expectedStream = new ByteArrayInputStream(allBytes); + int read = 0; + while (read < allBytes.length) { + final byte[] expectedBytes = new byte[Math.min(Math.abs(RANDOM.nextInt()) % 128, allBytes.length - read)]; + final byte[] actualBytes = new byte[expectedBytes.length]; + fillBuff(stream, actualBytes); + fillBuff(expectedStream, expectedBytes); + Assert.assertArrayEquals(expectedBytes, actualBytes); + read += expectedBytes.length; + } + Assert.assertEquals(allBytes.length, responseHandler.getByteCount()); + } + + + @Test + public void alignedMultiStreamTest() throws IOException + { + Iterator it = BYTE_LIST.iterator(); + + SequenceInputStreamResponseHandler responseHandler = new SequenceInputStreamResponseHandler(); + final HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK); + response.setChunked(true); + ClientResponse clientResponse = responseHandler.handleResponse(response); + while (it.hasNext()) { + final DefaultHttpChunk chunk = new DefaultHttpChunk(new BigEndianHeapChannelBuffer(it.next())); + clientResponse = responseHandler.handleChunk(clientResponse, chunk); + } + clientResponse = responseHandler.done(clientResponse); + + final InputStream stream = clientResponse.getObj(); + final InputStream expectedStream = new ByteArrayInputStream(allBytes); + + for (byte[] bytes : BYTE_LIST) { + final byte[] expectedBytes = new byte[bytes.length]; + final byte[] actualBytes = new byte[expectedBytes.length]; + fillBuff(stream, actualBytes); + fillBuff(expectedStream, expectedBytes); + Assert.assertArrayEquals(expectedBytes, actualBytes); + Assert.assertArrayEquals(expectedBytes, bytes); + } + Assert.assertEquals(allBytes.length, responseHandler.getByteCount()); + } + + @Test + public void simpleSingleStreamTest() throws IOException + { + SequenceInputStreamResponseHandler responseHandler = new SequenceInputStreamResponseHandler(); + final HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK); + response.setChunked(false); + response.setContent(new BigEndianHeapChannelBuffer(allBytes)); + ClientResponse clientResponse = responseHandler.handleResponse(response); + clientResponse = responseHandler.done(clientResponse); + + final InputStream stream = clientResponse.getObj(); + final InputStream expectedStream = new ByteArrayInputStream(allBytes); + int read = 0; + while (read < allBytes.length) { + final byte[] expectedBytes = new byte[Math.min(Math.abs(RANDOM.nextInt()) % 128, allBytes.length - read)]; + final byte[] actualBytes = new byte[expectedBytes.length]; + fillBuff(stream, actualBytes); + fillBuff(expectedStream, expectedBytes); + Assert.assertArrayEquals(expectedBytes, actualBytes); + read += expectedBytes.length; + } + Assert.assertEquals(allBytes.length, responseHandler.getByteCount()); + } + +} diff --git a/java-util/src/test/java/io/druid/java/util/metrics/CpuAcctDeltaMonitorTest.java b/java-util/src/test/java/io/druid/java/util/metrics/CpuAcctDeltaMonitorTest.java new file mode 100644 index 000000000000..81273ccfec7f --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/metrics/CpuAcctDeltaMonitorTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import com.google.common.collect.ImmutableMap; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.metrics.cgroups.CgroupDiscoverer; +import io.druid.java.util.metrics.cgroups.ProcCgroupDiscoverer; +import io.druid.java.util.metrics.cgroups.TestUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +public class CpuAcctDeltaMonitorTest +{ + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + private File procDir; + private File cgroupDir; + private File cpuacctDir; + private CgroupDiscoverer discoverer; + + @Before + public void setUp() throws IOException + { + cgroupDir = temporaryFolder.newFolder(); + procDir = temporaryFolder.newFolder(); + discoverer = new ProcCgroupDiscoverer(procDir.toPath()); + TestUtils.setUpCgroups(procDir, cgroupDir); + cpuacctDir = new File( + cgroupDir, + "cpu,cpuacct/system.slice/mesos-agent-druid.service/f12ba7e0-fa16-462e-bb9d-652ccc27f0ee" + ); + Assert.assertTrue((cpuacctDir.isDirectory() && cpuacctDir.exists()) || cpuacctDir.mkdirs()); + TestUtils.copyResource("/cpuacct.usage_all", new File(cpuacctDir, "cpuacct.usage_all")); + } + + @Test + public void testMonitorWontCrash() throws Exception + { + final CpuAcctDeltaMonitor monitor = new CpuAcctDeltaMonitor( + "some_feed", + ImmutableMap.of(), + cgroup -> { + throw new RuntimeException("Should continue"); + } + ); + final StubServiceEmitter emitter = new StubServiceEmitter("service", "host"); + monitor.doMonitor(emitter); + monitor.doMonitor(emitter); + monitor.doMonitor(emitter); + Assert.assertTrue(emitter.getEvents().isEmpty()); + } + + @Test + public void testSimpleMonitor() throws Exception + { + final File cpuacct = new File(cpuacctDir, "cpuacct.usage_all"); + try (final FileOutputStream fos = new FileOutputStream(cpuacct)) { + fos.write(StringUtils.toUtf8("cpu user system\n")); + for (int i = 0; i < 128; ++i) { + fos.write(StringUtils.toUtf8(StringUtils.format("%d 0 0\n", i))); + } + } + final CpuAcctDeltaMonitor monitor = new CpuAcctDeltaMonitor( + "some_feed", + ImmutableMap.of(), + (cgroup) -> cpuacctDir.toPath() + ); + final StubServiceEmitter emitter = new StubServiceEmitter("service", "host"); + Assert.assertFalse(monitor.doMonitor(emitter)); + // First should just cache + Assert.assertEquals(0, emitter.getEvents().size()); + Assert.assertTrue(cpuacct.delete()); + TestUtils.copyResource("/cpuacct.usage_all", cpuacct); + Assert.assertTrue(monitor.doMonitor(emitter)); + Assert.assertEquals(2 * 128 + 1, emitter.getEvents().size()); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/metrics/GcNameTest.java b/java-util/src/test/java/io/druid/java/util/metrics/GcNameTest.java new file mode 100644 index 000000000000..21a84277718d --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/metrics/GcNameTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import org.gridkit.lab.jvm.perfdata.JStatData; + +public class GcNameTest +{ + + public static void main(String[] args) + { + JStatData jStatData = JStatData.connect(SigarUtil.getCurrentProcessId()); + System.out.println(jStatData.getAllCounters().get("sun.gc.collector.0.name").getValue()); + System.out.println(jStatData.getAllCounters().get("sun.gc.collector.1.name").getValue()); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/metrics/JvmMonitorTest.java b/java-util/src/test/java/io/druid/java/util/metrics/JvmMonitorTest.java new file mode 100644 index 000000000000..ac04f9d2f06c --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/metrics/JvmMonitorTest.java @@ -0,0 +1,148 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.Event; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.List; + +public class JvmMonitorTest +{ + + @Test(timeout = 60000) + public void testGcCounts() throws InterruptedException + { + GcTrackingEmitter emitter = new GcTrackingEmitter(); + + final ServiceEmitter serviceEmitter = new ServiceEmitter("test", "localhost", emitter); + serviceEmitter.start(); + final JvmMonitor jvmMonitor = new JvmMonitor(); + + while (true) { + // generate some garbage to see gc counters incremented + @SuppressWarnings("unused") + byte[] b = new byte[1024 * 1024 * 50]; + emitter.reset(); + jvmMonitor.doMonitor(serviceEmitter); + if (emitter.gcSeen()) { + return; + } + Thread.sleep(10); + } + } + + private static class GcTrackingEmitter implements Emitter + { + private Number oldGcCount; + private Number oldGcCpu; + private Number youngGcCount; + private Number youngGcCpu; + + @Override + public void start() + { + + } + + void reset() + { + oldGcCount = null; + oldGcCpu = null; + youngGcCount = null; + youngGcCpu = null; + } + + @Override + public void emit(Event e) + { + ServiceMetricEvent event = (ServiceMetricEvent) e; + String gcGen = null; + if (event.toMap().get("gcGen") != null) { + gcGen = ((List) event.toMap().get("gcGen")).get(0).toString(); + } + + switch (event.getMetric() + "/" + gcGen) { + case "jvm/gc/count/old": + oldGcCount = event.getValue(); + break; + case "jvm/gc/cpu/old": + oldGcCpu = event.getValue(); + break; + case "jvm/gc/count/young": + youngGcCount = event.getValue(); + break; + case "jvm/gc/cpu/young": + youngGcCpu = event.getValue(); + break; + } + } + + boolean gcSeen() + { + return oldGcSeen() || youngGcSeen(); + } + + private boolean oldGcSeen() + { + boolean oldGcCountSeen = oldGcCount != null && oldGcCount.longValue() > 0; + boolean oldGcCpuSeen = oldGcCpu != null && oldGcCpu.longValue() > 0; + if (oldGcCountSeen || oldGcCpuSeen) { + System.out.println("old count: " + oldGcCount + ", cpu: " + oldGcCpu); + } + Assert.assertFalse( + "expected to see old gc count and cpu both zero or non-existent or both positive", + oldGcCountSeen ^ oldGcCpuSeen + ); + return oldGcCountSeen; + } + + private boolean youngGcSeen() + { + boolean youngGcCountSeen = youngGcCount != null && youngGcCount.longValue() > 0; + boolean youngGcCpuSeen = youngGcCpu != null && youngGcCpu.longValue() > 0; + if (youngGcCountSeen || youngGcCpuSeen) { + System.out.println("young count: " + youngGcCount + ", cpu: " + youngGcCpu); + } + Assert.assertFalse( + "expected to see young gc count and cpu both zero/non-existent or both positive", + youngGcCountSeen ^ youngGcCpuSeen + ); + return youngGcCountSeen; + } + + @Override + public void flush() throws IOException + { + + } + + @Override + public void close() throws IOException + { + + } + } +} diff --git a/java-util/src/test/java/io/druid/java/util/metrics/JvmPidDiscovererTest.java b/java-util/src/test/java/io/druid/java/util/metrics/JvmPidDiscovererTest.java new file mode 100644 index 000000000000..98ea6ca261b0 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/metrics/JvmPidDiscovererTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import org.junit.Assert; +import org.junit.Test; + +public class JvmPidDiscovererTest +{ + @Test + public void getPid() throws Exception + { + Assert.assertNotNull(JvmPidDiscoverer.instance().getPid()); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/metrics/MonitorUtilsTest.java b/java-util/src/test/java/io/druid/java/util/metrics/MonitorUtilsTest.java new file mode 100644 index 000000000000..adecb0145e0a --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/metrics/MonitorUtilsTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.druid.java.util.emitter.service.ServiceMetricEvent; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Map; + +public class MonitorUtilsTest +{ + @Test + public void testAddDimensionsToBuilder() + { + ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder(); + Map dimensions = ImmutableMap.of( + "dim1", new String[]{"value1"}, + "dim2", new String[]{"value2.1", "value2.2"} + ); + + MonitorUtils.addDimensionsToBuilder(builder, dimensions); + + Assert.assertEquals(builder.getDimension("dim1"), ImmutableList.of("value1")); + Assert.assertEquals(builder.getDimension("dim2"), ImmutableList.of("value2.1", "value2.2")); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/metrics/MonitorsTest.java b/java-util/src/test/java/io/druid/java/util/metrics/MonitorsTest.java new file mode 100644 index 000000000000..885118b96703 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/metrics/MonitorsTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import com.google.common.collect.ImmutableMap; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.emitter.core.Event; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class MonitorsTest +{ + + @Test + public void testSetFeed() + { + String feed = "testFeed"; + StubServiceEmitter emitter = new StubServiceEmitter("dev/monitor-test", "localhost:0000"); + Monitor m = Monitors.createCompoundJvmMonitor(ImmutableMap.of(), feed); + m.start(); + m.monitor(emitter); + m.stop(); + checkEvents(emitter.getEvents(), feed); + } + + @Test + public void testDefaultFeed() + { + StubServiceEmitter emitter = new StubServiceEmitter("dev/monitor-test", "localhost:0000"); + Monitor m = Monitors.createCompoundJvmMonitor(ImmutableMap.of()); + m.start(); + m.monitor(emitter); + m.stop(); + checkEvents(emitter.getEvents(), "metrics"); + } + + private void checkEvents(List events, String expectedFeed) + { + Assert.assertFalse("no events emitted", events.isEmpty()); + for (Event e : events) { + if (!expectedFeed.equals(e.getFeed())) { + String message = StringUtils.format("\"feed\" in event: %s", e.toMap().toString()); + Assert.assertEquals(message, expectedFeed, e.getFeed()); + } + } + } +} diff --git a/java-util/src/test/java/io/druid/java/util/metrics/SigarLoadTest.java b/java-util/src/test/java/io/druid/java/util/metrics/SigarLoadTest.java new file mode 100644 index 000000000000..032097579fa0 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/metrics/SigarLoadTest.java @@ -0,0 +1,35 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import junit.framework.Assert; +import org.hyperic.sigar.Sigar; +import org.hyperic.sigar.SigarException; +import org.junit.Test; + +public class SigarLoadTest +{ + @Test + public void testSigarLoad() throws SigarException + { + Sigar sigar = SigarUtil.getSigar(); + Assert.assertTrue(sigar.getPid() > 0); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/metrics/SigarPidDiscovererTest.java b/java-util/src/test/java/io/druid/java/util/metrics/SigarPidDiscovererTest.java new file mode 100644 index 000000000000..96219b5fa6d0 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/metrics/SigarPidDiscovererTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import org.junit.Test; + +public class SigarPidDiscovererTest +{ + @Test + public void simpleTest() + { + // Just make sure we don't crash + SigarPidDiscoverer.instance().getPid(); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/metrics/StubServiceEmitter.java b/java-util/src/test/java/io/druid/java/util/metrics/StubServiceEmitter.java new file mode 100644 index 000000000000..ed9aa18b7acc --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/metrics/StubServiceEmitter.java @@ -0,0 +1,63 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics; + +import io.druid.java.util.emitter.core.Event; +import io.druid.java.util.emitter.service.ServiceEmitter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +class StubServiceEmitter extends ServiceEmitter +{ + private List events = new ArrayList<>(); + + public StubServiceEmitter(String service, String host) + { + super(service, host, null); + } + + @Override + public void emit(Event event) + { + events.add(event); + } + + public List getEvents() + { + return events; + } + + @Override + public void start() + { + } + + @Override + public void flush() throws IOException + { + } + + @Override + public void close() throws IOException + { + } +} diff --git a/java-util/src/test/java/io/druid/java/util/metrics/cgroups/CpuAcctTest.java b/java-util/src/test/java/io/druid/java/util/metrics/cgroups/CpuAcctTest.java new file mode 100644 index 000000000000..22b8973fbe8c --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/metrics/cgroups/CpuAcctTest.java @@ -0,0 +1,390 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics.cgroups; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Random; +import java.util.stream.LongStream; + + +public class CpuAcctTest +{ + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + private File procDir; + private File cgroupDir; + private CgroupDiscoverer discoverer; + + @Before + public void setUp() throws IOException + { + cgroupDir = temporaryFolder.newFolder(); + procDir = temporaryFolder.newFolder(); + discoverer = new ProcCgroupDiscoverer(procDir.toPath()); + TestUtils.setUpCgroups(procDir, cgroupDir); + final File cpuacctDir = new File( + cgroupDir, + "cpu,cpuacct/system.slice/mesos-agent-druid.service/f12ba7e0-fa16-462e-bb9d-652ccc27f0ee" + ); + Assert.assertTrue((cpuacctDir.isDirectory() && cpuacctDir.exists()) || cpuacctDir.mkdirs()); + TestUtils.copyResource("/cpuacct.usage_all", new File(cpuacctDir, "cpuacct.usage_all")); + } + + @Test + public void testWontCrash() + { + final CpuAcct cpuAcct = new CpuAcct(cgroup -> { + throw new RuntimeException("Should still continue"); + }); + final CpuAcct.CpuAcctMetric metric = cpuAcct.snapshot(); + Assert.assertEquals(0L, metric.cpuCount()); + Assert.assertEquals(0L, metric.usrTime()); + Assert.assertEquals(0L, metric.sysTime()); + } + + @Test + public void testSimpleLoad() + { + final CpuAcct cpuAcct = new CpuAcct(discoverer); + final CpuAcct.CpuAcctMetric snapshot = cpuAcct.snapshot(); + Assert.assertEquals(128, snapshot.cpuCount()); + Assert.assertArrayEquals(new long[]{ + 7344294132655L, + 28183572804378L, + 29552215219002L, + 29478124053329L, + 29829248571038L, + 30290864470719L, + 30561719193413L, + 30638606697446L, + 39251561450889L, + 39082643428276L, + 38829852195583L, + 39158341842449L, + 39490263697181L, + 39363774325162L, + 39569806302164L, + 39410558504372L, + 44907796060505L, + 42522297123640L, + 41920625622542L, + 40593391967420L, + 40350585953295L, + 40139554930678L, + 40019783380923L, + 40182686097717L, + 39778858132385L, + 40252938541440L, + 40476150948365L, + 40277874584618L, + 39938509407084L, + 39914644718371L, + 40010393213659L, + 39938252119551L, + 44958993952996L, + 42967015146867L, + 41742610896758L, + 40751067975683L, + 40390633464986L, + 40143331504478L, + 40486014164571L, + 40565630824649L, + 39976774290845L, + 39942348143441L, + 40149234675554L, + 39895306827546L, + 40062736204343L, + 39208930836306L, + 40098687814379L, + 39803234124100L, + 44894501101599L, + 43470418903266L, + 41844924510711L, + 41137017142223L, + 40958534485692L, + 40996749346830L, + 40722256755299L, + 40715123538100L, + 40756697196452L, + 40388351638364L, + 40607150623932L, + 40799783862688L, + 41085552637672L, + 40406189914954L, + 40723714534227L, + 40594766265305L, + 47966186930606L, + 40950398764685L, + 39773685629470L, + 39799299693868L, + 39962809136735L, + 39621597321912L, + 39576312003193L, + 39306677714061L, + 37450385749152L, + 37262591956707L, + 37867848418162L, + 37583170923549L, + 37565074790371L, + 37490674520644L, + 37627356285158L, + 37841963931932L, + 36467248910690L, + 37168392893625L, + 37299551044970L, + 37765703017416L, + 37799573327332L, + 38049895238765L, + 37985869086888L, + 37696241330128L, + 38292683839783L, + 38120890685615L, + 38045845683675L, + 38182343881607L, + 37729375994055L, + 38074905443126L, + 37912923241296L, + 37937307782462L, + 36325058440018L, + 37157290185847L, + 37403692187351L, + 38153199365119L, + 37880374831086L, + 37651504251556L, + 37739944955714L, + 37627835848111L, + 37903369827551L, + 37981555620129L, + 37848203152449L, + 37990323769817L, + 38347560243684L, + 37887959856632L, + 37937702600487L, + 38221455324656L, + 37035158753494L, + 37519359498531L, + 38185495941617L, + 38947633192125L, + 38497334926906L, + 38621231881393L, + 38817038222494L, + 38911615674430L, + 38384669525324L, + 38597980524270L, + 38477107776771L, + 38483564156449L, + 38471547310020L, + 38827188957783L, + 38554167083817L, + 38870461161179L + }, snapshot.usrTimes()); + Assert.assertArrayEquals(new long[]{ + 4583688852335L, + 385888457233L, + 370465239852L, + 363572894675L, + 334329517808L, + 327800567541L, + 289395249935L, + 302853990791L, + 255558344564L, + 274043522998L, + 256014012773L, + 253276920707L, + 257971375228L, + 244926573383L, + 240692265222L, + 241820386298L, + 74203786299L, + 79965589957L, + 73735621808L, + 74851270413L, + 84689908189L, + 69618323977L, + 76316055513L, + 87434816528L, + 80680588218L, + 70083776283L, + 59931280896L, + 67678277638L, + 85713335794L, + 74594785949L, + 69367790895L, + 68817875729L, + 83346318662L, + 80872459027L, + 69882000641L, + 59230207177L, + 80985355290L, + 88305162767L, + 79610055475L, + 77097429500L, + 72748340407L, + 77647202034L, + 61982641775L, + 63292828955L, + 71501429739L, + 101050648913L, + 67603152691L, + 85242844849L, + 53320735254L, + 59480233446L, + 53738738094L, + 50064771695L, + 49322497528L, + 60437383202L, + 58974386647L, + 57254872107L, + 59214245666L, + 60135823463L, + 53295222550L, + 54850380995L, + 55260978656L, + 58478426264L, + 54870256138L, + 57541909382L, + 563254758724L, + 310547129324L, + 297956013630L, + 314333221636L, + 301425507083L, + 307849203392L, + 302818464346L, + 300708730620L, + 201706674463L, + 237643132530L, + 203202644049L, + 200889116684L, + 211272183842L, + 211346947120L, + 204440036165L, + 214268740497L, + 79712665639L, + 75090730135L, + 78826984017L, + 80223883420L, + 74309961837L, + 73501632933L, + 84726122364L, + 89432904192L, + 86169574674L, + 61631022854L, + 63064271269L, + 71350572902L, + 79603442903L, + 80632248259L, + 73143984225L, + 68368595815L, + 93212419757L, + 79875769874L, + 71939175528L, + 61279926839L, + 80531076242L, + 97919090064L, + 62379356509L, + 73687439569L, + 65237344029L, + 88874846168L, + 73032242506L, + 66715221936L, + 75213471783L, + 85339287415L, + 70516346051L, + 80265423715L, + 55020848133L, + 68103451501L, + 56282008328L, + 57420205172L, + 51517654341L, + 53668755335L, + 58390679981L, + 59254129440L, + 60781018005L, + 60445939750L, + 62771597837L, + 57294449683L, + 57716404007L, + 55643587481L, + 53593212339L, + 52866253864L + }, snapshot.sysTimes()); + Assert.assertEquals(LongStream.of(snapshot.sysTimes()).sum(), snapshot.sysTime()); + Assert.assertEquals(LongStream.of(snapshot.usrTimes()).sum(), snapshot.usrTime()); + Assert.assertEquals( + LongStream.of(snapshot.sysTimes()).sum() + LongStream.of(snapshot.usrTimes()).sum(), + snapshot.time() + ); + } + + @Test + public void testSimpleMetricFunctions() + { + final long[] usrTime = new long[]{1, 2, 3}; + final long[] sysTime = new long[]{4, 5, 6}; + final CpuAcct.CpuAcctMetric metric = new CpuAcct.CpuAcctMetric(usrTime, sysTime); + Assert.assertEquals(6, metric.usrTime()); + Assert.assertEquals(15, metric.sysTime()); + Assert.assertArrayEquals(usrTime, metric.usrTimes()); + Assert.assertArrayEquals(sysTime, metric.sysTimes()); + for (int i = 0; i < usrTime.length; ++i) { + Assert.assertEquals(usrTime[i], metric.usrTime(i)); + } + for (int i = 0; i < sysTime.length; ++i) { + Assert.assertEquals(sysTime[i], metric.sysTime(i)); + } + } + + @Test + public void testDiff() + { + final Random random = new Random(364781L); + final long[] zeroes = new long[32]; + Arrays.fill(zeroes, 0); + final long[] usr = new long[zeroes.length]; + final long[] sys = new long[zeroes.length]; + long total = 0L; + for (int i = 0; i < usr.length; ++i) { + int add = random.nextInt(Integer.MAX_VALUE >>> 2); + usr[i] = add; + sys[i] = add << 1; + total += add; + } + final CpuAcct.CpuAcctMetric metric0 = new CpuAcct.CpuAcctMetric(zeroes, zeroes); + final CpuAcct.CpuAcctMetric metric1 = new CpuAcct.CpuAcctMetric(usr, sys); + final CpuAcct.CpuAcctMetric diff = metric1.cumulativeSince(metric0); + Assert.assertEquals(total, diff.usrTime()); + Assert.assertEquals(total << 1, diff.sysTime()); + Assert.assertNotEquals(0, total); + final CpuAcct.CpuAcctMetric zeroDiff = metric1.cumulativeSince(metric1); + Assert.assertEquals(0, zeroDiff.usrTime()); + Assert.assertEquals(0, zeroDiff.sysTime()); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/metrics/cgroups/ProcCgroupDiscovererTest.java b/java-util/src/test/java/io/druid/java/util/metrics/cgroups/ProcCgroupDiscovererTest.java new file mode 100644 index 000000000000..195422e59523 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/metrics/cgroups/ProcCgroupDiscovererTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics.cgroups; + +import com.google.common.collect.ImmutableSet; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.nio.file.Paths; + +public class ProcCgroupDiscovererTest +{ + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + private File procDir; + private File cgroupDir; + private CgroupDiscoverer discoverer; + + @Before + public void setUp() throws Exception + { + cgroupDir = temporaryFolder.newFolder(); + procDir = temporaryFolder.newFolder(); + discoverer = new ProcCgroupDiscoverer(procDir.toPath()); + TestUtils.setUpCgroups(procDir, cgroupDir); + } + + @Test + public void testSimpleProc() throws Exception + { + Assert.assertEquals( + new File( + cgroupDir, + "cpu,cpuacct/system.slice/mesos-agent-druid.service/f12ba7e0-fa16-462e-bb9d-652ccc27f0ee" + ).toPath(), + discoverer.discover("cpu") + ); + } + + @Test + public void testParse() throws Exception + { + final ProcCgroupDiscoverer.ProcMountsEntry entry = ProcCgroupDiscoverer.ProcMountsEntry.parse( + "/dev/md126 /ebs xfs rw,seclabel,noatime,attr2,inode64,sunit=1024,swidth=16384,noquota 0 0" + ); + Assert.assertEquals("/dev/md126", entry.dev); + Assert.assertEquals(Paths.get("/ebs"), entry.path); + Assert.assertEquals("xfs", entry.type); + Assert.assertEquals(ImmutableSet.of( + "rw", + "seclabel", + "noatime", + "attr2", + "inode64", + "sunit=1024", + "swidth=16384", + "noquota" + ), entry.options); + } + + @Test + public void testNullCgroup() + { + expectedException.expect(NullPointerException.class); + Assert.assertNull(new ProcCgroupDiscoverer(procDir.toPath()).discover(null)); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/metrics/cgroups/TestUtils.java b/java-util/src/test/java/io/druid/java/util/metrics/cgroups/TestUtils.java new file mode 100644 index 000000000000..1ddc006ca567 --- /dev/null +++ b/java-util/src/test/java/io/druid/java/util/metrics/cgroups/TestUtils.java @@ -0,0 +1,63 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.java.util.metrics.cgroups; + +import io.druid.java.util.common.StringUtils; +import org.junit.Assert; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.regex.Pattern; + +public class TestUtils +{ + public static void setUpCgroups( + File procDir, + File cgroupDir + ) throws IOException + { + final File procMountsTemplate = new File(procDir, "mounts.template"); + final File procMounts = new File(procDir, "mounts"); + copyResource("/proc.mounts", procMountsTemplate); + + final String procMountsString = StringUtils.fromUtf8(Files.readAllBytes(procMountsTemplate.toPath())); + Files.write( + procMounts.toPath(), + StringUtils.toUtf8(procMountsString.replaceAll( + Pattern.quote("/sys/fs/cgroup"), + cgroupDir.getAbsolutePath() + )) + ); + + Assert.assertTrue(new File( + cgroupDir, + "cpu,cpuacct/system.slice/mesos-agent-druid.service/f12ba7e0-fa16-462e-bb9d-652ccc27f0ee" + ).mkdirs()); + copyResource("/proc.pid.cgroup", new File(procDir, "cgroup")); + } + + public static void copyResource(String resource, File out) throws IOException + { + Files.copy(TestUtils.class.getResourceAsStream(resource), out.toPath()); + Assert.assertTrue(out.exists()); + Assert.assertNotEquals(0, out.length()); + } +} diff --git a/java-util/src/test/resources/cpuacct.usage_all b/java-util/src/test/resources/cpuacct.usage_all new file mode 100644 index 000000000000..f3d1c534708a --- /dev/null +++ b/java-util/src/test/resources/cpuacct.usage_all @@ -0,0 +1,129 @@ +cpu user system +0 7344294132655 4583688852335 +1 28183572804378 385888457233 +2 29552215219002 370465239852 +3 29478124053329 363572894675 +4 29829248571038 334329517808 +5 30290864470719 327800567541 +6 30561719193413 289395249935 +7 30638606697446 302853990791 +8 39251561450889 255558344564 +9 39082643428276 274043522998 +10 38829852195583 256014012773 +11 39158341842449 253276920707 +12 39490263697181 257971375228 +13 39363774325162 244926573383 +14 39569806302164 240692265222 +15 39410558504372 241820386298 +16 44907796060505 74203786299 +17 42522297123640 79965589957 +18 41920625622542 73735621808 +19 40593391967420 74851270413 +20 40350585953295 84689908189 +21 40139554930678 69618323977 +22 40019783380923 76316055513 +23 40182686097717 87434816528 +24 39778858132385 80680588218 +25 40252938541440 70083776283 +26 40476150948365 59931280896 +27 40277874584618 67678277638 +28 39938509407084 85713335794 +29 39914644718371 74594785949 +30 40010393213659 69367790895 +31 39938252119551 68817875729 +32 44958993952996 83346318662 +33 42967015146867 80872459027 +34 41742610896758 69882000641 +35 40751067975683 59230207177 +36 40390633464986 80985355290 +37 40143331504478 88305162767 +38 40486014164571 79610055475 +39 40565630824649 77097429500 +40 39976774290845 72748340407 +41 39942348143441 77647202034 +42 40149234675554 61982641775 +43 39895306827546 63292828955 +44 40062736204343 71501429739 +45 39208930836306 101050648913 +46 40098687814379 67603152691 +47 39803234124100 85242844849 +48 44894501101599 53320735254 +49 43470418903266 59480233446 +50 41844924510711 53738738094 +51 41137017142223 50064771695 +52 40958534485692 49322497528 +53 40996749346830 60437383202 +54 40722256755299 58974386647 +55 40715123538100 57254872107 +56 40756697196452 59214245666 +57 40388351638364 60135823463 +58 40607150623932 53295222550 +59 40799783862688 54850380995 +60 41085552637672 55260978656 +61 40406189914954 58478426264 +62 40723714534227 54870256138 +63 40594766265305 57541909382 +64 47966186930606 563254758724 +65 40950398764685 310547129324 +66 39773685629470 297956013630 +67 39799299693868 314333221636 +68 39962809136735 301425507083 +69 39621597321912 307849203392 +70 39576312003193 302818464346 +71 39306677714061 300708730620 +72 37450385749152 201706674463 +73 37262591956707 237643132530 +74 37867848418162 203202644049 +75 37583170923549 200889116684 +76 37565074790371 211272183842 +77 37490674520644 211346947120 +78 37627356285158 204440036165 +79 37841963931932 214268740497 +80 36467248910690 79712665639 +81 37168392893625 75090730135 +82 37299551044970 78826984017 +83 37765703017416 80223883420 +84 37799573327332 74309961837 +85 38049895238765 73501632933 +86 37985869086888 84726122364 +87 37696241330128 89432904192 +88 38292683839783 86169574674 +89 38120890685615 61631022854 +90 38045845683675 63064271269 +91 38182343881607 71350572902 +92 37729375994055 79603442903 +93 38074905443126 80632248259 +94 37912923241296 73143984225 +95 37937307782462 68368595815 +96 36325058440018 93212419757 +97 37157290185847 79875769874 +98 37403692187351 71939175528 +99 38153199365119 61279926839 +100 37880374831086 80531076242 +101 37651504251556 97919090064 +102 37739944955714 62379356509 +103 37627835848111 73687439569 +104 37903369827551 65237344029 +105 37981555620129 88874846168 +106 37848203152449 73032242506 +107 37990323769817 66715221936 +108 38347560243684 75213471783 +109 37887959856632 85339287415 +110 37937702600487 70516346051 +111 38221455324656 80265423715 +112 37035158753494 55020848133 +113 37519359498531 68103451501 +114 38185495941617 56282008328 +115 38947633192125 57420205172 +116 38497334926906 51517654341 +117 38621231881393 53668755335 +118 38817038222494 58390679981 +119 38911615674430 59254129440 +120 38384669525324 60781018005 +121 38597980524270 60445939750 +122 38477107776771 62771597837 +123 38483564156449 57294449683 +124 38471547310020 57716404007 +125 38827188957783 55643587481 +126 38554167083817 53593212339 +127 38870461161179 52866253864 diff --git a/java-util/src/test/resources/keystore.jks b/java-util/src/test/resources/keystore.jks new file mode 100644 index 0000000000000000000000000000000000000000..a98918283982d72a7bd8cf97fa80219f04b3d757 GIT binary patch literal 2331 zcmd5-`8N~{7oHh|#x$rI`wU~rl5OT4hDcc=vNl7O$kvpx4YMdoGPX$GL>RJ1Bc&)J z8cSh@kX`o5ny9oO$*0bFzwiAMz8~%n&-0vn&ppq1&b_~ue=P$50MJeW{}d#U;_Dsg zN1@Som}4e;TQvXxeh`KU{)G`15K%_WyF0IZtH;5EtnS~tp&D*yc>CgT(^lb-p5H6gLaDyGzS00ddC7Fw1sE~lk_gs- z;Jm2RzKS z|9sR0c`=}LI}Ro}`0chRJDrs<_nO~fe?s%SH`~3%H}u}r>Wj*lV#l7(n=^h^GZ|3| zuM&NVskU{286{cz0){t&E`EDFNtyfpQSzYTGI6;B>?p6qzpM^-s8^A%m+J(ZC=@4k zIU0>#8);H+{J`UT*in;h8*0?1+V>3qk-a&z`_>s!&5_4n2{Xc`TxF)TaX?Np^qEnf zu;~{BmR_WoYH~7VP(Hg0bhpu-Uw~5I}Xi7+3xEsmjFq7?9({|Yw8LsS~1_Y-PbCFyB>Ikj~M&|~?v z${hC-A!BKW4#dNZZ5B3ZCoHyC6Uc`&X$Co9o9?yp!V-5AHS*CBfdRm)99GVSQ z%@N|G(_1HSOQ=+SA@;5I>Alh{*3*7Gr>O4oQG8=%-oe6rn88}tN-2c(Jj}KneGruh z6}ks!EJQilX+Bh#Wst=y*d4Duvc&He1`jMD}QbV4QeYRg}85nc*S zXPoq?-_p$QnW_7P@|sR1?(-~_NcIXz$w+v%@uI)OBq^IzpgcO)iz^b{ZSj#k)3Pi1 zmQ5ez#En-=3Z3Vj3tD(;y*I6w+GUklR2BH+znTKI?SEDUU zY`f9=m~HaxWOy=9)rMfO9s`6Y*cz_oNga=_U?A1orNwH?)h$$^sAC zftgR=WsAlV2SmS6|K^0? z@IM?4O*w<6?{qs;sDY=kXpHP{6$+QIr=BKJ{}C;`0|QCHXnXH~e`fN*|993gvJxSEn?pe_dYU-*B!HxsDvZ;N+65FZmL0>ChVLLep(2za(}^SkwM;kG1c zc%`?yg;kRk-?3!0Nf_a5lo%h?KZ6(r2u^9p&>AHg;t=Fj~SgboW<%=rS8fsY;o@_w|%|FduCvrZ6?Ld(((7%`V z%npbAq3C5_J?!6eiU@a_F3)8gF{CCJYMKEn{CJ1HonnfC_<#Ulu{=f&gW5?2N*F8- zMznoFAFwBzxwzxKZa#*u4%jV>2>#o|ot*BN2n2v9H@cNHn&kcGE&Zje<*8ree6o_b z`|p)d9I>$(YWY$!!Evr;o6Hzvj!yVd-zdY}FywiyQ9nbyypLD4%sOLVNWL+~(~mP2 zcteM~fp01@#GMgo*NmBUxhhFP%TnHnjrY09@m2O>f-z#&4WrbE*(h06ijov{wq9b7 zr#G=>;Hbcadh0N+ob2~-nRMcn+WUk#x-t^WabNBIK) literal 0 HcmV?d00001 diff --git a/java-util/src/test/resources/log4j2.xml b/java-util/src/test/resources/log4j2.xml new file mode 100644 index 000000000000..689cbbf6b1c2 --- /dev/null +++ b/java-util/src/test/resources/log4j2.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + diff --git a/java-util/src/test/resources/proc.cgroups b/java-util/src/test/resources/proc.cgroups new file mode 100644 index 000000000000..09eff7760532 --- /dev/null +++ b/java-util/src/test/resources/proc.cgroups @@ -0,0 +1,13 @@ +#subsys_name hierarchy num_cgroups enabled +cpuset 4 1 1 +cpu 9 108 1 +cpuacct 9 108 1 +blkio 8 60 1 +memory 3 160 1 +devices 7 60 1 +freezer 6 52 1 +net_cls 2 1 1 +perf_event 10 1 1 +net_prio 2 1 1 +hugetlb 5 1 1 +pids 11 63 1 diff --git a/java-util/src/test/resources/proc.mounts b/java-util/src/test/resources/proc.mounts new file mode 100644 index 000000000000..c1b3ee74fe3a --- /dev/null +++ b/java-util/src/test/resources/proc.mounts @@ -0,0 +1,36 @@ +sysfs /sys sysfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 +proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 +devtmpfs /dev devtmpfs rw,seclabel,nosuid,size=1007354668k,nr_inodes=251838667,mode=755 0 0 +securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 +tmpfs /dev/shm tmpfs rw,seclabel,nosuid,nodev 0 0 +devpts /dev/pts devpts rw,seclabel,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 +tmpfs /run tmpfs rw,seclabel,nosuid,nodev,mode=755 0 0 +tmpfs /sys/fs/cgroup tmpfs ro,seclabel,nosuid,nodev,noexec,mode=755 0 0 +cgroup /sys/fs/cgroup/systemd cgroup rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd 0 0 +pstore /sys/fs/pstore pstore rw,seclabel,nosuid,nodev,noexec,relatime 0 0 +cgroup /sys/fs/cgroup/net_cls,net_prio cgroup rw,nosuid,nodev,noexec,relatime,net_cls,net_prio 0 0 +cgroup /sys/fs/cgroup/memory cgroup rw,nosuid,nodev,noexec,relatime,memory 0 0 +cgroup /sys/fs/cgroup/cpuset cgroup rw,nosuid,nodev,noexec,relatime,cpuset 0 0 +cgroup /sys/fs/cgroup/hugetlb cgroup rw,nosuid,nodev,noexec,relatime,hugetlb 0 0 +cgroup /sys/fs/cgroup/freezer cgroup rw,nosuid,nodev,noexec,relatime,freezer 0 0 +cgroup /sys/fs/cgroup/devices cgroup rw,nosuid,nodev,noexec,relatime,devices 0 0 +cgroup /sys/fs/cgroup/blkio cgroup rw,nosuid,nodev,noexec,relatime,blkio 0 0 +cgroup /sys/fs/cgroup/cpu,cpuacct cgroup rw,nosuid,nodev,noexec,relatime,cpu,cpuacct 0 0 +cgroup /sys/fs/cgroup/perf_event cgroup rw,nosuid,nodev,noexec,relatime,perf_event 0 0 +cgroup /sys/fs/cgroup/pids cgroup rw,nosuid,nodev,noexec,relatime,pids 0 0 +/dev/xvda9 / ext4 rw,seclabel,relatime,data=ordered 0 0 +/dev/mapper/usr /usr ext4 ro,seclabel,relatime,block_validity,delalloc,barrier,user_xattr,acl 0 0 +selinuxfs /sys/fs/selinux selinuxfs rw,relatime 0 0 +tmpfs /media tmpfs rw,seclabel,nosuid,nodev,noexec,relatime 0 0 +systemd-1 /boot autofs rw,relatime,fd=34,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=18428 0 0 +xenfs /proc/xen xenfs rw,relatime 0 0 +tmpfs /tmp tmpfs rw,seclabel,nosuid,nodev 0 0 +hugetlbfs /dev/hugepages hugetlbfs rw,seclabel,relatime 0 0 +systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=38,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=140296 0 0 +debugfs /sys/kernel/debug debugfs rw,seclabel,relatime 0 0 +mqueue /dev/mqueue mqueue rw,seclabel,relatime 0 0 +/dev/xvda6 /usr/share/oem ext4 rw,seclabel,nodev,relatime,commit=600,data=ordered 0 0 +/dev/xvda1 /boot vfat rw,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=ascii,shortname=mixed,errors=remount-ro 0 0 +/dev/md127 /mnt xfs rw,seclabel,noatime,attr2,inode64,sunit=1024,swidth=2048,noquota 0 0 +/dev/md126 /ebs xfs rw,seclabel,noatime,attr2,inode64,sunit=1024,swidth=16384,noquota 0 0 +/dev/xvda9 /var/lib/docker/overlay ext4 rw,seclabel,relatime,data=ordered 0 0 diff --git a/java-util/src/test/resources/proc.pid.cgroup b/java-util/src/test/resources/proc.pid.cgroup new file mode 100644 index 000000000000..139f81b29c8f --- /dev/null +++ b/java-util/src/test/resources/proc.pid.cgroup @@ -0,0 +1,11 @@ +11:pids:/system.slice/mesos-agent-druid.service +10:perf_event:/ +9:cpu,cpuacct:/system.slice/mesos-agent-druid.service/f12ba7e0-fa16-462e-bb9d-652ccc27f0ee +8:blkio:/system.slice/mesos-agent-druid.service +7:devices:/system.slice/mesos-agent-druid.service +6:freezer:/system.slice/mesos-agent-druid.service/f12ba7e0-fa16-462e-bb9d-652ccc27f0ee +5:hugetlb:/ +4:cpuset:/ +3:memory:/system.slice/mesos-agent-druid.service +2:net_cls,net_prio:/ +1:name=systemd:/mesos_executors.slice diff --git a/pom.xml b/pom.xml index 5d187eb938ef..273441a4c183 100644 --- a/pom.xml +++ b/pom.xml @@ -62,6 +62,7 @@ 2.12.0 1.10.0 1.15.0 + 8.1.0 16.0.1 4.1.0 9.3.19.v20170502 @@ -69,9 +70,11 @@ 2.4.6 2.5 + + 3.10.6.Final - 4.0.52.Final + 4.0.52.Final 1.7.12 2.7.3 @@ -146,11 +149,6 @@ - - com.metamx - java-util - 1.3.2 - commons-codec commons-codec @@ -481,10 +479,15 @@ jetty-security ${jetty.version} + + io.netty + netty + ${netty3.version} + io.netty netty-all - ${netty.version} + ${netty4.version} joda-time @@ -707,6 +710,16 @@ asm-commons 5.2 + + org.asynchttpclient + async-http-client + 2.0.37 + + + org.gridkit.lab + jvm-attach-api + 1.2 + diff --git a/processing/pom.xml b/processing/pom.xml index d964346cce94..92ddda9e04ec 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -35,11 +35,11 @@ druid-common ${project.parent.version} - - io.druid - java-util - ${project.parent.version} - + + io.druid + java-util + ${project.parent.version} + io.druid druid-hll @@ -70,10 +70,6 @@ it.unimi.dsi fastutil - - com.metamx - java-util - com.ning compress-lzf diff --git a/processing/src/main/java/io/druid/query/CPUTimeMetricQueryRunner.java b/processing/src/main/java/io/druid/query/CPUTimeMetricQueryRunner.java index bce67514d97b..15bc40de9cf4 100644 --- a/processing/src/main/java/io/druid/query/CPUTimeMetricQueryRunner.java +++ b/processing/src/main/java/io/druid/query/CPUTimeMetricQueryRunner.java @@ -20,7 +20,7 @@ package io.druid.query; import com.google.common.base.Supplier; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.common.utils.VMUtils; import io.druid.java.util.common.ISE; import io.druid.java.util.common.guava.Sequence; diff --git a/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java b/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java index 7d830fedac00..8e13ae5bead1 100644 --- a/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java +++ b/processing/src/main/java/io/druid/query/DefaultQueryMetrics.java @@ -23,8 +23,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.collections.bitmap.BitmapFactory; import io.druid.query.filter.Filter; import org.joda.time.Interval; diff --git a/processing/src/main/java/io/druid/query/ExecutorServiceMonitor.java b/processing/src/main/java/io/druid/query/ExecutorServiceMonitor.java index bf15e9e99418..2c227884e731 100644 --- a/processing/src/main/java/io/druid/query/ExecutorServiceMonitor.java +++ b/processing/src/main/java/io/druid/query/ExecutorServiceMonitor.java @@ -21,9 +21,9 @@ import com.google.common.collect.Lists; import com.google.inject.Inject; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; -import com.metamx.metrics.AbstractMonitor; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; +import io.druid.java.util.metrics.AbstractMonitor; import java.util.List; @@ -58,5 +58,4 @@ public interface MetricEmitter { void emitMetrics(ServiceEmitter emitter, ServiceMetricEvent.Builder metricBuilder); } - } diff --git a/processing/src/main/java/io/druid/query/FluentQueryRunnerBuilder.java b/processing/src/main/java/io/druid/query/FluentQueryRunnerBuilder.java index 5d3fefaad802..8a79d7e00f9c 100644 --- a/processing/src/main/java/io/druid/query/FluentQueryRunnerBuilder.java +++ b/processing/src/main/java/io/druid/query/FluentQueryRunnerBuilder.java @@ -19,7 +19,7 @@ package io.druid.query; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.java.util.common.guava.Sequence; import java.util.Map; diff --git a/processing/src/main/java/io/druid/query/IntervalChunkingQueryRunner.java b/processing/src/main/java/io/druid/query/IntervalChunkingQueryRunner.java index 264225105400..2e114f72124f 100644 --- a/processing/src/main/java/io/druid/query/IntervalChunkingQueryRunner.java +++ b/processing/src/main/java/io/druid/query/IntervalChunkingQueryRunner.java @@ -21,7 +21,7 @@ import com.google.common.base.Function; import com.google.common.collect.Lists; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.granularity.PeriodGranularity; import io.druid.java.util.common.guava.FunctionalIterable; diff --git a/processing/src/main/java/io/druid/query/IntervalChunkingQueryRunnerDecorator.java b/processing/src/main/java/io/druid/query/IntervalChunkingQueryRunnerDecorator.java index 5f7e10c5bccf..4f47462b1b5f 100644 --- a/processing/src/main/java/io/druid/query/IntervalChunkingQueryRunnerDecorator.java +++ b/processing/src/main/java/io/druid/query/IntervalChunkingQueryRunnerDecorator.java @@ -20,7 +20,7 @@ package io.druid.query; import com.google.inject.Inject; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.guice.annotations.Processing; import io.druid.guice.annotations.PublicApi; diff --git a/processing/src/main/java/io/druid/query/MetricsEmittingExecutorService.java b/processing/src/main/java/io/druid/query/MetricsEmittingExecutorService.java index 9fbcec1e47a3..da39026fa62e 100644 --- a/processing/src/main/java/io/druid/query/MetricsEmittingExecutorService.java +++ b/processing/src/main/java/io/druid/query/MetricsEmittingExecutorService.java @@ -22,8 +22,8 @@ import com.google.common.util.concurrent.ForwardingListeningExecutorService; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import java.util.concurrent.Callable; diff --git a/processing/src/main/java/io/druid/query/MetricsEmittingQueryRunner.java b/processing/src/main/java/io/druid/query/MetricsEmittingQueryRunner.java index 69d1db8094c2..0734c2b6a948 100644 --- a/processing/src/main/java/io/druid/query/MetricsEmittingQueryRunner.java +++ b/processing/src/main/java/io/druid/query/MetricsEmittingQueryRunner.java @@ -19,7 +19,7 @@ package io.druid.query; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.java.util.common.guava.LazySequence; import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.SequenceWrapper; diff --git a/processing/src/main/java/io/druid/query/QueryMetrics.java b/processing/src/main/java/io/druid/query/QueryMetrics.java index 9abd4ce0c619..c560dc908220 100644 --- a/processing/src/main/java/io/druid/query/QueryMetrics.java +++ b/processing/src/main/java/io/druid/query/QueryMetrics.java @@ -19,7 +19,7 @@ package io.druid.query; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.collections.bitmap.BitmapFactory; import io.druid.guice.annotations.ExtensionPoint; import io.druid.guice.annotations.PublicApi; @@ -30,7 +30,7 @@ import java.util.List; /** - * Abstraction wrapping {@link com.metamx.emitter.service.ServiceMetricEvent.Builder} and allowing to control what + * Abstraction wrapping {@link io.druid.java.util.emitter.service.ServiceMetricEvent.Builder} and allowing to control what * metrics are actually emitted, what dimensions do they have, etc. * * diff --git a/processing/src/main/java/io/druid/query/RetryQueryRunner.java b/processing/src/main/java/io/druid/query/RetryQueryRunner.java index f4a7f178ce5f..5dbff2a3a7b6 100644 --- a/processing/src/main/java/io/druid/query/RetryQueryRunner.java +++ b/processing/src/main/java/io/druid/query/RetryQueryRunner.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.guava.MergeSequence; import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; diff --git a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java index be2dd5c84e4e..4c254a75b86f 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/DoubleFirstAggregatorFactory.java @@ -24,7 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; import com.google.common.primitives.Longs; -import com.metamx.common.StringUtils; +import io.druid.java.util.common.StringUtils; import io.druid.collections.SerializablePair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; diff --git a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java index 0b62e47aebb1..84f185e93939 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/FloatFirstAggregatorFactory.java @@ -24,7 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; import com.google.common.primitives.Longs; -import com.metamx.common.StringUtils; +import io.druid.java.util.common.StringUtils; import io.druid.collections.SerializablePair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; diff --git a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java index b000d9d5c363..a2780e94e6a4 100644 --- a/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/first/LongFirstAggregatorFactory.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; -import com.metamx.common.StringUtils; +import io.druid.java.util.common.StringUtils; import io.druid.collections.SerializablePair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; diff --git a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java index 3241d49aee62..c028247b5813 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/DoubleLastAggregatorFactory.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; -import com.metamx.common.StringUtils; +import io.druid.java.util.common.StringUtils; import io.druid.collections.SerializablePair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; diff --git a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java index 73d671b69c96..2ab1f2856825 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/FloatLastAggregatorFactory.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; -import com.metamx.common.StringUtils; +import io.druid.java.util.common.StringUtils; import io.druid.collections.SerializablePair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; diff --git a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java index 60f9669f5913..4bee9e9a6659 100644 --- a/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/last/LongLastAggregatorFactory.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.metamx.common.StringUtils; +import io.druid.java.util.common.StringUtils; import io.druid.collections.SerializablePair; import io.druid.java.util.common.UOE; import io.druid.query.aggregation.AggregateCombiner; diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/ByteBufferIntList.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/ByteBufferIntList.java index 958426fcce0d..5ee9d73d6d30 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/ByteBufferIntList.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/ByteBufferIntList.java @@ -20,7 +20,7 @@ package io.druid.query.groupby.epinephelinae; import com.google.common.primitives.Ints; -import com.metamx.common.IAE; +import io.druid.java.util.common.IAE; import io.druid.java.util.common.StringUtils; import java.nio.ByteBuffer; diff --git a/processing/src/main/java/io/druid/query/search/AutoStrategy.java b/processing/src/main/java/io/druid/query/search/AutoStrategy.java index b3b692c298dd..665669e5476c 100644 --- a/processing/src/main/java/io/druid/query/search/AutoStrategy.java +++ b/processing/src/main/java/io/druid/query/search/AutoStrategy.java @@ -19,7 +19,7 @@ package io.druid.query.search; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.BitmapIndexSelector; import io.druid.segment.ColumnSelectorBitmapIndexSelector; diff --git a/processing/src/main/java/io/druid/query/search/DefaultSearchQueryMetrics.java b/processing/src/main/java/io/druid/query/search/DefaultSearchQueryMetrics.java index e554ffa317cc..9fdf1a8a6af7 100644 --- a/processing/src/main/java/io/druid/query/search/DefaultSearchQueryMetrics.java +++ b/processing/src/main/java/io/druid/query/search/DefaultSearchQueryMetrics.java @@ -19,7 +19,7 @@ package io.druid.query.search; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.collections.bitmap.BitmapFactory; import io.druid.java.util.common.ISE; import io.druid.query.BitmapResultFactory; diff --git a/processing/src/main/java/io/druid/query/search/SearchStrategySelector.java b/processing/src/main/java/io/druid/query/search/SearchStrategySelector.java index 91e431737914..ef5e40440cec 100644 --- a/processing/src/main/java/io/druid/query/search/SearchStrategySelector.java +++ b/processing/src/main/java/io/druid/query/search/SearchStrategySelector.java @@ -21,7 +21,7 @@ import com.google.common.base.Supplier; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.ISE; public class SearchStrategySelector diff --git a/processing/src/main/java/io/druid/query/select/DefaultSelectQueryMetrics.java b/processing/src/main/java/io/druid/query/select/DefaultSelectQueryMetrics.java index ab6238a8bee5..7a73d527b9f4 100644 --- a/processing/src/main/java/io/druid/query/select/DefaultSelectQueryMetrics.java +++ b/processing/src/main/java/io/druid/query/select/DefaultSelectQueryMetrics.java @@ -19,7 +19,7 @@ package io.druid.query.select; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.collections.bitmap.BitmapFactory; import io.druid.java.util.common.ISE; import io.druid.query.BitmapResultFactory; diff --git a/processing/src/main/java/io/druid/segment/IndexIO.java b/processing/src/main/java/io/druid/segment/IndexIO.java index bab5c8432a91..5f28a7fd58df 100644 --- a/processing/src/main/java/io/druid/segment/IndexIO.java +++ b/processing/src/main/java/io/druid/segment/IndexIO.java @@ -31,7 +31,7 @@ import com.google.common.io.Files; import com.google.common.primitives.Ints; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.collections.bitmap.ConciseBitmapFactory; import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.collections.spatial.ImmutableRTree; diff --git a/processing/src/main/java/io/druid/segment/ReferenceCountingSegment.java b/processing/src/main/java/io/druid/segment/ReferenceCountingSegment.java index f8ec72b2be81..bcb5ea0f2979 100644 --- a/processing/src/main/java/io/druid/segment/ReferenceCountingSegment.java +++ b/processing/src/main/java/io/druid/segment/ReferenceCountingSegment.java @@ -20,7 +20,7 @@ package io.druid.segment; import com.google.common.base.Preconditions; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import org.joda.time.Interval; import java.io.Closeable; diff --git a/processing/src/test/java/io/druid/query/CachingEmitter.java b/processing/src/test/java/io/druid/query/CachingEmitter.java index aa2663904744..be0b6cfd7ece 100644 --- a/processing/src/test/java/io/druid/query/CachingEmitter.java +++ b/processing/src/test/java/io/druid/query/CachingEmitter.java @@ -19,8 +19,8 @@ package io.druid.query; -import com.metamx.emitter.core.Emitter; -import com.metamx.emitter.core.Event; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.Event; import java.io.IOException; diff --git a/processing/src/test/java/io/druid/query/DefaultQueryMetricsTest.java b/processing/src/test/java/io/druid/query/DefaultQueryMetricsTest.java index add3360d107c..e78618e7875a 100644 --- a/processing/src/test/java/io/druid/query/DefaultQueryMetricsTest.java +++ b/processing/src/test/java/io/druid/query/DefaultQueryMetricsTest.java @@ -20,7 +20,7 @@ package io.druid.query; import com.google.common.collect.ImmutableSet; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.java.util.common.granularity.Granularities; import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.dimension.DefaultDimensionSpec; diff --git a/processing/src/test/java/io/druid/query/IntervalChunkingQueryRunnerTest.java b/processing/src/test/java/io/druid/query/IntervalChunkingQueryRunnerTest.java index ef389424f009..cd62d4f47717 100644 --- a/processing/src/test/java/io/druid/query/IntervalChunkingQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/IntervalChunkingQueryRunnerTest.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.java.util.common.guava.Sequences; import io.druid.query.Druids.TimeseriesQueryBuilder; import io.druid.query.aggregation.AggregatorFactory; diff --git a/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java index 0c2a781bf4f2..07733c946bfd 100644 --- a/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java +++ b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java @@ -25,8 +25,8 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.emitter.core.NoopEmitter; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.core.NoopEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.granularity.Granularities; diff --git a/processing/src/test/java/io/druid/query/groupby/DefaultGroupByQueryMetricsTest.java b/processing/src/test/java/io/druid/query/groupby/DefaultGroupByQueryMetricsTest.java index 3d16b0108945..dfef95950efc 100644 --- a/processing/src/test/java/io/druid/query/groupby/DefaultGroupByQueryMetricsTest.java +++ b/processing/src/test/java/io/druid/query/groupby/DefaultGroupByQueryMetricsTest.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.granularity.PeriodGranularity; import io.druid.query.CachingEmitter; diff --git a/processing/src/test/java/io/druid/query/search/DefaultSearchQueryMetricsTest.java b/processing/src/test/java/io/druid/query/search/DefaultSearchQueryMetricsTest.java index dbb2db70320e..3db6b64b9b84 100644 --- a/processing/src/test/java/io/druid/query/search/DefaultSearchQueryMetricsTest.java +++ b/processing/src/test/java/io/druid/query/search/DefaultSearchQueryMetricsTest.java @@ -20,7 +20,7 @@ package io.druid.query.search; import com.google.common.collect.ImmutableSet; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.query.CachingEmitter; import io.druid.query.DefaultQueryMetricsTest; import io.druid.query.DruidMetrics; diff --git a/processing/src/test/java/io/druid/query/select/DefaultSelectQueryMetricsTest.java b/processing/src/test/java/io/druid/query/select/DefaultSelectQueryMetricsTest.java index 6aeefe2b2957..a422a3ad421a 100644 --- a/processing/src/test/java/io/druid/query/select/DefaultSelectQueryMetricsTest.java +++ b/processing/src/test/java/io/druid/query/select/DefaultSelectQueryMetricsTest.java @@ -19,7 +19,7 @@ package io.druid.query.select; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.query.CachingEmitter; import io.druid.query.DefaultQueryMetricsTest; import io.druid.query.DruidMetrics; diff --git a/processing/src/test/java/io/druid/query/timeseries/DefaultTimeseriesQueryMetricsTest.java b/processing/src/test/java/io/druid/query/timeseries/DefaultTimeseriesQueryMetricsTest.java index 912eaf6a55c9..d5027bf5e9cc 100644 --- a/processing/src/test/java/io/druid/query/timeseries/DefaultTimeseriesQueryMetricsTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/DefaultTimeseriesQueryMetricsTest.java @@ -19,7 +19,7 @@ package io.druid.query.timeseries; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.query.CachingEmitter; import io.druid.query.DefaultQueryMetricsTest; import io.druid.query.DruidMetrics; diff --git a/processing/src/test/java/io/druid/query/topn/DefaultTopNQueryMetricsTest.java b/processing/src/test/java/io/druid/query/topn/DefaultTopNQueryMetricsTest.java index a61b393a094c..b7ad4730219c 100644 --- a/processing/src/test/java/io/druid/query/topn/DefaultTopNQueryMetricsTest.java +++ b/processing/src/test/java/io/druid/query/topn/DefaultTopNQueryMetricsTest.java @@ -20,7 +20,7 @@ package io.druid.query.topn; import com.google.common.collect.ImmutableSet; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.java.util.common.granularity.Granularities; import io.druid.query.CachingEmitter; import io.druid.query.DefaultQueryMetricsTest; diff --git a/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java b/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java index 602868d48499..3ddc0d7633b2 100644 --- a/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java +++ b/processing/src/test/java/io/druid/segment/serde/HyperUniquesSerdeForTest.java @@ -21,7 +21,7 @@ import com.google.common.collect.Ordering; import com.google.common.hash.HashFunction; -import com.metamx.common.StringUtils; +import io.druid.java.util.common.StringUtils; import io.druid.data.input.InputRow; import io.druid.hll.HyperLogLogCollector; import io.druid.segment.writeout.SegmentWriteOutMedium; diff --git a/server/pom.xml b/server/pom.xml index 7c7aa8fe6df4..3a48aab63228 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -46,8 +46,9 @@ 0.0.2 - com.metamx + io.druid java-util + ${project.parent.version} commons-cli diff --git a/server/src/main/java/io/druid/client/AbstractCuratorServerInventoryView.java b/server/src/main/java/io/druid/client/AbstractCuratorServerInventoryView.java index fec18e5a654a..d6c731cdc58f 100644 --- a/server/src/main/java/io/druid/client/AbstractCuratorServerInventoryView.java +++ b/server/src/main/java/io/druid/client/AbstractCuratorServerInventoryView.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.base.Throwables; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.concurrent.Execs; import io.druid.curator.inventory.CuratorInventoryManager; import io.druid.curator.inventory.CuratorInventoryManagerStrategy; diff --git a/server/src/main/java/io/druid/client/BatchServerInventoryView.java b/server/src/main/java/io/druid/client/BatchServerInventoryView.java index abdeeaf2313e..44b23bd3d701 100644 --- a/server/src/main/java/io/druid/client/BatchServerInventoryView.java +++ b/server/src/main/java/io/druid/client/BatchServerInventoryView.java @@ -28,7 +28,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.guice.ManageLifecycle; import io.druid.java.util.common.ISE; import io.druid.java.util.common.Pair; diff --git a/server/src/main/java/io/druid/client/BrokerServerView.java b/server/src/main/java/io/druid/client/BrokerServerView.java index a1adf95a9fef..47ec9d414ac4 100644 --- a/server/src/main/java/io/druid/client/BrokerServerView.java +++ b/server/src/main/java/io/druid/client/BrokerServerView.java @@ -25,8 +25,8 @@ import com.google.common.collect.Maps; import com.google.common.collect.Ordering; import com.google.inject.Inject; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.http.client.HttpClient; import io.druid.client.selector.QueryableDruidServer; import io.druid.client.selector.ServerSelector; import io.druid.client.selector.TierSelectorStrategy; diff --git a/server/src/main/java/io/druid/client/CachingClusteredClient.java b/server/src/main/java/io/druid/client/CachingClusteredClient.java index ab7cae42dd02..0fd9762dc7e9 100644 --- a/server/src/main/java/io/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/io/druid/client/CachingClusteredClient.java @@ -40,7 +40,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.client.cache.Cache; import io.druid.client.cache.CacheConfig; import io.druid.client.selector.QueryableDruidServer; diff --git a/server/src/main/java/io/druid/client/DirectDruidClient.java b/server/src/main/java/io/druid/client/DirectDruidClient.java index 5399a01de593..d34efee089de 100644 --- a/server/src/main/java/io/druid/client/DirectDruidClient.java +++ b/server/src/main/java/io/druid/client/DirectDruidClient.java @@ -33,13 +33,13 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.ClientResponse; -import com.metamx.http.client.response.HttpResponseHandler; -import com.metamx.http.client.response.StatusResponseHandler; -import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.ClientResponse; +import io.druid.java.util.http.client.response.HttpResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHolder; import io.druid.java.util.common.IAE; import io.druid.java.util.common.Pair; import io.druid.java.util.common.RE; diff --git a/server/src/main/java/io/druid/client/FilteredHttpServerInventoryViewProvider.java b/server/src/main/java/io/druid/client/FilteredHttpServerInventoryViewProvider.java index 5c3195bc15db..98d634f30e79 100644 --- a/server/src/main/java/io/druid/client/FilteredHttpServerInventoryViewProvider.java +++ b/server/src/main/java/io/druid/client/FilteredHttpServerInventoryViewProvider.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Predicates; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClient; import io.druid.discovery.DruidNodeDiscoveryProvider; import io.druid.guice.annotations.EscalatedClient; import io.druid.guice.annotations.Smile; diff --git a/server/src/main/java/io/druid/client/HttpServerInventoryView.java b/server/src/main/java/io/druid/client/HttpServerInventoryView.java index 3bb8fd25a81a..cd73c2daf8db 100644 --- a/server/src/main/java/io/druid/client/HttpServerInventoryView.java +++ b/server/src/main/java/io/druid/client/HttpServerInventoryView.java @@ -30,12 +30,8 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.io.AppendableByteArrayInputStream; -import com.metamx.http.client.response.ClientResponse; -import com.metamx.http.client.response.InputStreamResponseHandler; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.http.client.HttpClient; import io.druid.concurrent.LifecycleLock; import io.druid.discovery.DataNodeService; import io.druid.discovery.DiscoveryDruidNode; @@ -50,6 +46,10 @@ import io.druid.java.util.common.concurrent.ScheduledExecutors; import io.druid.java.util.common.lifecycle.LifecycleStart; import io.druid.java.util.common.lifecycle.LifecycleStop; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.io.AppendableByteArrayInputStream; +import io.druid.java.util.http.client.response.ClientResponse; +import io.druid.java.util.http.client.response.InputStreamResponseHandler; import io.druid.server.coordination.DataSegmentChangeRequest; import io.druid.server.coordination.DruidServerMetadata; import io.druid.server.coordination.SegmentChangeRequestDrop; diff --git a/server/src/main/java/io/druid/client/HttpServerInventoryViewProvider.java b/server/src/main/java/io/druid/client/HttpServerInventoryViewProvider.java index bfcb3b262384..2a0ff5615911 100644 --- a/server/src/main/java/io/druid/client/HttpServerInventoryViewProvider.java +++ b/server/src/main/java/io/druid/client/HttpServerInventoryViewProvider.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Predicates; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClient; import io.druid.discovery.DruidNodeDiscoveryProvider; import io.druid.guice.annotations.EscalatedClient; import io.druid.guice.annotations.Smile; diff --git a/server/src/main/java/io/druid/client/ImmutableDruidServer.java b/server/src/main/java/io/druid/client/ImmutableDruidServer.java index d215a88aaba4..b43f9d00bbd9 100644 --- a/server/src/main/java/io/druid/client/ImmutableDruidServer.java +++ b/server/src/main/java/io/druid/client/ImmutableDruidServer.java @@ -21,7 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; -import com.metamx.common.StringUtils; +import io.druid.java.util.common.StringUtils; import io.druid.server.coordination.DruidServerMetadata; import io.druid.server.coordination.ServerType; import io.druid.timeline.DataSegment; @@ -113,9 +113,9 @@ public Map getSegments() public String getURL() { if (metadata.getHostAndTlsPort() != null) { - return StringUtils.safeFormat("https://%s", metadata.getHostAndTlsPort()); + return StringUtils.nonStrictFormat("https://%s", metadata.getHostAndTlsPort()); } else { - return StringUtils.safeFormat("http://%s", metadata.getHostAndPort()); + return StringUtils.nonStrictFormat("http://%s", metadata.getHostAndPort()); } } diff --git a/server/src/main/java/io/druid/client/SingleServerInventoryView.java b/server/src/main/java/io/druid/client/SingleServerInventoryView.java index ef69679be23a..af5aaa423e92 100644 --- a/server/src/main/java/io/druid/client/SingleServerInventoryView.java +++ b/server/src/main/java/io/druid/client/SingleServerInventoryView.java @@ -25,7 +25,7 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.guice.ManageLifecycle; import io.druid.java.util.common.Pair; import io.druid.server.coordination.DruidServerMetadata; diff --git a/server/src/main/java/io/druid/client/cache/Cache.java b/server/src/main/java/io/druid/client/cache/Cache.java index 7db32b1be10d..70d42e4baf3d 100644 --- a/server/src/main/java/io/druid/client/cache/Cache.java +++ b/server/src/main/java/io/druid/client/cache/Cache.java @@ -21,7 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.primitives.Ints; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.java.util.common.StringUtils; import javax.annotation.Nullable; diff --git a/server/src/main/java/io/druid/client/cache/CacheMonitor.java b/server/src/main/java/io/druid/client/cache/CacheMonitor.java index 752b90614c01..9a7bb141d242 100644 --- a/server/src/main/java/io/druid/client/cache/CacheMonitor.java +++ b/server/src/main/java/io/druid/client/cache/CacheMonitor.java @@ -20,9 +20,9 @@ package io.druid.client.cache; import com.google.inject.Inject; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; -import com.metamx.metrics.AbstractMonitor; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; +import io.druid.java.util.metrics.AbstractMonitor; import io.druid.java.util.common.StringUtils; public class CacheMonitor extends AbstractMonitor diff --git a/server/src/main/java/io/druid/client/cache/CaffeineCache.java b/server/src/main/java/io/druid/client/cache/CaffeineCache.java index ba4688a3ef44..139d879bac5e 100644 --- a/server/src/main/java/io/druid/client/cache/CaffeineCache.java +++ b/server/src/main/java/io/druid/client/cache/CaffeineCache.java @@ -27,8 +27,8 @@ import com.google.common.collect.Maps; import com.google.common.primitives.Chars; import com.google.common.primitives.Ints; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.java.util.common.logger.Logger; import net.jpountz.lz4.LZ4Compressor; diff --git a/server/src/main/java/io/druid/client/cache/HybridCache.java b/server/src/main/java/io/druid/client/cache/HybridCache.java index 330549a6bae0..8787a985a395 100644 --- a/server/src/main/java/io/druid/client/cache/HybridCache.java +++ b/server/src/main/java/io/druid/client/cache/HybridCache.java @@ -21,7 +21,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.java.util.common.logger.Logger; import javax.annotation.Nullable; diff --git a/server/src/main/java/io/druid/client/cache/MapCache.java b/server/src/main/java/io/druid/client/cache/MapCache.java index fe058523e5af..f9257fc52352 100644 --- a/server/src/main/java/io/druid/client/cache/MapCache.java +++ b/server/src/main/java/io/druid/client/cache/MapCache.java @@ -22,7 +22,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import java.nio.ByteBuffer; import java.util.Collections; diff --git a/server/src/main/java/io/druid/client/cache/MemcachedCache.java b/server/src/main/java/io/druid/client/cache/MemcachedCache.java index 7cd6e10223d6..59c9c9a4b9f2 100644 --- a/server/src/main/java/io/druid/client/cache/MemcachedCache.java +++ b/server/src/main/java/io/druid/client/cache/MemcachedCache.java @@ -32,9 +32,9 @@ import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; import com.google.common.primitives.Ints; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; -import com.metamx.metrics.AbstractMonitor; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; +import io.druid.java.util.metrics.AbstractMonitor; import io.druid.collections.ResourceHolder; import io.druid.collections.StupidResourceHolder; import io.druid.java.util.common.logger.Logger; diff --git a/server/src/main/java/io/druid/client/coordinator/CoordinatorClient.java b/server/src/main/java/io/druid/client/coordinator/CoordinatorClient.java index 20b65f065bef..131708b99406 100644 --- a/server/src/main/java/io/druid/client/coordinator/CoordinatorClient.java +++ b/server/src/main/java/io/druid/client/coordinator/CoordinatorClient.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; import com.google.inject.Inject; -import com.metamx.http.client.response.FullResponseHolder; +import io.druid.java.util.http.client.response.FullResponseHolder; import io.druid.client.ImmutableSegmentLoadInfo; import io.druid.discovery.DruidLeaderClient; import io.druid.java.util.common.ISE; diff --git a/server/src/main/java/io/druid/client/indexing/IndexingServiceClient.java b/server/src/main/java/io/druid/client/indexing/IndexingServiceClient.java index 39aa1e037df3..3eca7d355bbd 100644 --- a/server/src/main/java/io/druid/client/indexing/IndexingServiceClient.java +++ b/server/src/main/java/io/druid/client/indexing/IndexingServiceClient.java @@ -24,7 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.inject.Inject; -import com.metamx.http.client.response.FullResponseHolder; +import io.druid.java.util.http.client.response.FullResponseHolder; import io.druid.discovery.DruidLeaderClient; import io.druid.indexer.TaskStatusPlus; import io.druid.java.util.common.DateTimes; diff --git a/server/src/main/java/io/druid/curator/discovery/CuratorDruidLeaderSelector.java b/server/src/main/java/io/druid/curator/discovery/CuratorDruidLeaderSelector.java index 36687d550a69..9b51a76580ef 100644 --- a/server/src/main/java/io/druid/curator/discovery/CuratorDruidLeaderSelector.java +++ b/server/src/main/java/io/druid/curator/discovery/CuratorDruidLeaderSelector.java @@ -21,7 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Throwables; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.concurrent.LifecycleLock; import io.druid.discovery.DruidLeaderSelector; import io.druid.guice.annotations.Self; diff --git a/server/src/main/java/io/druid/curator/discovery/CuratorServiceAnnouncer.java b/server/src/main/java/io/druid/curator/discovery/CuratorServiceAnnouncer.java index 8b737125bf9c..7a76946f046a 100644 --- a/server/src/main/java/io/druid/curator/discovery/CuratorServiceAnnouncer.java +++ b/server/src/main/java/io/druid/curator/discovery/CuratorServiceAnnouncer.java @@ -22,7 +22,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.Maps; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.server.DruidNode; import org.apache.curator.x.discovery.ServiceDiscovery; import org.apache.curator.x.discovery.ServiceInstance; diff --git a/server/src/main/java/io/druid/discovery/DruidLeaderClient.java b/server/src/main/java/io/druid/discovery/DruidLeaderClient.java index 3955f9329b8b..a77b8312937c 100644 --- a/server/src/main/java/io/druid/discovery/DruidLeaderClient.java +++ b/server/src/main/java/io/druid/discovery/DruidLeaderClient.java @@ -22,11 +22,11 @@ import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.FullResponseHandler; -import com.metamx.http.client.response.FullResponseHolder; -import com.metamx.http.client.response.HttpResponseHandler; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.FullResponseHandler; +import io.druid.java.util.http.client.response.FullResponseHolder; +import io.druid.java.util.http.client.response.HttpResponseHandler; import io.druid.client.selector.Server; import io.druid.concurrent.LifecycleLock; import io.druid.curator.discovery.ServerDiscoverySelector; diff --git a/server/src/main/java/io/druid/guice/CoordinatorDiscoveryModule.java b/server/src/main/java/io/druid/guice/CoordinatorDiscoveryModule.java index 6af0f2d02250..a054287303fd 100644 --- a/server/src/main/java/io/druid/guice/CoordinatorDiscoveryModule.java +++ b/server/src/main/java/io/druid/guice/CoordinatorDiscoveryModule.java @@ -22,7 +22,7 @@ import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Provides; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClient; import io.druid.client.coordinator.Coordinator; import io.druid.client.coordinator.CoordinatorSelectorConfig; import io.druid.curator.discovery.ServerDiscoveryFactory; diff --git a/server/src/main/java/io/druid/guice/DruidBinders.java b/server/src/main/java/io/druid/guice/DruidBinders.java index e9fecc090a56..81d6d71adbe7 100644 --- a/server/src/main/java/io/druid/guice/DruidBinders.java +++ b/server/src/main/java/io/druid/guice/DruidBinders.java @@ -23,7 +23,7 @@ import com.google.inject.TypeLiteral; import com.google.inject.multibindings.MapBinder; import com.google.inject.multibindings.Multibinder; -import com.metamx.metrics.Monitor; +import io.druid.java.util.metrics.Monitor; import io.druid.query.Query; import io.druid.query.QueryRunnerFactory; import io.druid.query.QueryToolChest; diff --git a/server/src/main/java/io/druid/guice/IndexingServiceDiscoveryModule.java b/server/src/main/java/io/druid/guice/IndexingServiceDiscoveryModule.java index 05da8f301bb4..d54f35d4506e 100644 --- a/server/src/main/java/io/druid/guice/IndexingServiceDiscoveryModule.java +++ b/server/src/main/java/io/druid/guice/IndexingServiceDiscoveryModule.java @@ -22,7 +22,7 @@ import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Provides; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClient; import io.druid.client.indexing.IndexingService; import io.druid.client.indexing.IndexingServiceSelectorConfig; import io.druid.curator.discovery.ServerDiscoveryFactory; diff --git a/server/src/main/java/io/druid/guice/http/HttpClientModule.java b/server/src/main/java/io/druid/guice/http/HttpClientModule.java index 73b06163c0ce..748045c48983 100644 --- a/server/src/main/java/io/druid/guice/http/HttpClientModule.java +++ b/server/src/main/java/io/druid/guice/http/HttpClientModule.java @@ -23,9 +23,9 @@ import com.google.inject.Binder; import com.google.inject.Inject; import com.google.inject.Module; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.HttpClientConfig; -import com.metamx.http.client.HttpClientInit; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClientConfig; +import io.druid.java.util.http.client.HttpClientInit; import io.druid.guice.JsonConfigProvider; import io.druid.guice.LazySingleton; import io.druid.guice.annotations.EscalatedClient; diff --git a/server/src/main/java/io/druid/guice/http/LifecycleUtils.java b/server/src/main/java/io/druid/guice/http/LifecycleUtils.java index dd52d1d75df8..02393e623a61 100644 --- a/server/src/main/java/io/druid/guice/http/LifecycleUtils.java +++ b/server/src/main/java/io/druid/guice/http/LifecycleUtils.java @@ -24,9 +24,9 @@ public class LifecycleUtils { - public static com.metamx.common.lifecycle.Lifecycle asMmxLifecycle(Lifecycle lifecycle) + public static Lifecycle asMmxLifecycle(Lifecycle lifecycle) { - final com.metamx.common.lifecycle.Lifecycle metamxLifecycle = new com.metamx.common.lifecycle.Lifecycle(); + final Lifecycle metamxLifecycle = new Lifecycle(); try { lifecycle.addMaybeStartHandler(new Lifecycle.Handler() { diff --git a/server/src/main/java/io/druid/metadata/SQLMetadataRuleManager.java b/server/src/main/java/io/druid/metadata/SQLMetadataRuleManager.java index e527d632ded0..8bec701757da 100644 --- a/server/src/main/java/io/druid/metadata/SQLMetadataRuleManager.java +++ b/server/src/main/java/io/druid/metadata/SQLMetadataRuleManager.java @@ -31,7 +31,7 @@ import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.audit.AuditEntry; import io.druid.audit.AuditInfo; import io.druid.audit.AuditManager; diff --git a/server/src/main/java/io/druid/metadata/SQLMetadataSegmentManager.java b/server/src/main/java/io/druid/metadata/SQLMetadataSegmentManager.java index eca6620f17a4..59e923855be3 100644 --- a/server/src/main/java/io/druid/metadata/SQLMetadataSegmentManager.java +++ b/server/src/main/java/io/druid/metadata/SQLMetadataSegmentManager.java @@ -33,7 +33,7 @@ import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.client.DruidDataSource; import io.druid.client.ImmutableDruidDataSource; import io.druid.concurrent.LifecycleLock; diff --git a/server/src/main/java/io/druid/metadata/SQLMetadataStorageActionHandler.java b/server/src/main/java/io/druid/metadata/SQLMetadataStorageActionHandler.java index 90b508f05052..66208be95692 100644 --- a/server/src/main/java/io/druid/metadata/SQLMetadataStorageActionHandler.java +++ b/server/src/main/java/io/druid/metadata/SQLMetadataStorageActionHandler.java @@ -27,7 +27,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; diff --git a/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java b/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java index c9ab30c35bae..f2c0036707b0 100644 --- a/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java +++ b/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java @@ -28,8 +28,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; -import com.metamx.http.client.response.FullResponseHolder; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.http.client.response.FullResponseHolder; import io.druid.client.coordinator.Coordinator; import io.druid.concurrent.LifecycleLock; import io.druid.discovery.DruidLeaderClient; diff --git a/server/src/main/java/io/druid/segment/loading/SegmentLoaderLocalCacheManager.java b/server/src/main/java/io/druid/segment/loading/SegmentLoaderLocalCacheManager.java index fbdb31cbcc8e..83c95f595c7e 100644 --- a/server/src/main/java/io/druid/segment/loading/SegmentLoaderLocalCacheManager.java +++ b/server/src/main/java/io/druid/segment/loading/SegmentLoaderLocalCacheManager.java @@ -23,7 +23,7 @@ import com.google.common.collect.Lists; import com.google.common.primitives.Longs; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.guice.annotations.Json; import io.druid.segment.IndexIO; import io.druid.segment.Segment; diff --git a/server/src/main/java/io/druid/segment/realtime/RealtimeManager.java b/server/src/main/java/io/druid/segment/realtime/RealtimeManager.java index 73bf34fb0b28..f054ad94aaa9 100644 --- a/server/src/main/java/io/druid/segment/realtime/RealtimeManager.java +++ b/server/src/main/java/io/druid/segment/realtime/RealtimeManager.java @@ -29,7 +29,7 @@ import com.google.common.collect.Maps; import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.concurrent.Execs; import io.druid.data.input.Committer; import io.druid.data.input.Firehose; diff --git a/server/src/main/java/io/druid/segment/realtime/RealtimeMetricsMonitor.java b/server/src/main/java/io/druid/segment/realtime/RealtimeMetricsMonitor.java index 376181e11a1f..f65ee0627c78 100644 --- a/server/src/main/java/io/druid/segment/realtime/RealtimeMetricsMonitor.java +++ b/server/src/main/java/io/druid/segment/realtime/RealtimeMetricsMonitor.java @@ -22,11 +22,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; -import com.metamx.metrics.AbstractMonitor; -import com.metamx.metrics.MonitorUtils; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; +import io.druid.java.util.metrics.AbstractMonitor; +import io.druid.java.util.metrics.MonitorUtils; import io.druid.query.DruidMetrics; import java.util.List; diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java index c8fd9c52d41d..deb7bd7fb332 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java @@ -37,8 +37,8 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.cache.Cache; import io.druid.client.cache.CacheConfig; import io.druid.common.guava.ThreadRenamingCallable; @@ -1033,7 +1033,7 @@ public Object apply(@Nullable Object input) { if (sinks.get(identifier) != sink) { // Only abandon sink if it is the same one originally requested to be abandoned. - log.warn("Sink for segment[%s] no longer valid, not abandoning."); + log.warn("Sink for segment[%s] no longer valid, not abandoning.", identifier); return null; } diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorPlumber.java b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorPlumber.java index 2d34065d5023..2edfa4ed6cbc 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorPlumber.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorPlumber.java @@ -29,7 +29,7 @@ import com.google.common.collect.Lists; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.common.guava.ThreadRenamingCallable; import io.druid.java.util.common.concurrent.Execs; import io.druid.data.input.Committer; diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderators.java b/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderators.java index 1014f1380a74..e3fabbc0b410 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderators.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderators.java @@ -20,7 +20,7 @@ package io.druid.segment.realtime.appenderator; import com.fasterxml.jackson.databind.ObjectMapper; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.cache.Cache; import io.druid.client.cache.CacheConfig; import io.druid.java.util.common.StringUtils; diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/DefaultRealtimeAppenderatorFactory.java b/server/src/main/java/io/druid/segment/realtime/appenderator/DefaultRealtimeAppenderatorFactory.java index 6be9b1b355fe..46b5f1a7afa5 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/DefaultRealtimeAppenderatorFactory.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/DefaultRealtimeAppenderatorFactory.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.databind.ObjectMapper; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.cache.Cache; import io.druid.client.cache.CacheConfig; diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/SinkQuerySegmentWalker.java b/server/src/main/java/io/druid/segment/realtime/appenderator/SinkQuerySegmentWalker.java index 3905adbbcbab..9d758553e71d 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/SinkQuerySegmentWalker.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/SinkQuerySegmentWalker.java @@ -24,8 +24,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.CachingQueryRunner; import io.druid.client.cache.Cache; import io.druid.client.cache.CacheConfig; diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/CombiningFirehoseFactory.java b/server/src/main/java/io/druid/segment/realtime/firehose/CombiningFirehoseFactory.java index 4267f6409a29..ce292eddb64c 100644 --- a/server/src/main/java/io/druid/segment/realtime/firehose/CombiningFirehoseFactory.java +++ b/server/src/main/java/io/druid/segment/realtime/firehose/CombiningFirehoseFactory.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.data.input.Firehose; import io.druid.data.input.FirehoseFactory; import io.druid.data.input.InputRow; diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/EventReceiverFirehoseFactory.java b/server/src/main/java/io/druid/segment/realtime/firehose/EventReceiverFirehoseFactory.java index 4343e873a564..9624f8bbcb0d 100644 --- a/server/src/main/java/io/druid/segment/realtime/firehose/EventReceiverFirehoseFactory.java +++ b/server/src/main/java/io/druid/segment/realtime/firehose/EventReceiverFirehoseFactory.java @@ -32,7 +32,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.io.CountingInputStream; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.concurrent.Execs; import io.druid.data.input.Firehose; import io.druid.data.input.FirehoseFactory; diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/LocalFirehoseFactory.java b/server/src/main/java/io/druid/segment/realtime/firehose/LocalFirehoseFactory.java index 6b80f2e96fbc..6db1e8c30941 100644 --- a/server/src/main/java/io/druid/segment/realtime/firehose/LocalFirehoseFactory.java +++ b/server/src/main/java/io/druid/segment/realtime/firehose/LocalFirehoseFactory.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.data.input.impl.AbstractTextFilesFirehoseFactory; import io.druid.data.input.impl.StringInputRowParser; import io.druid.java.util.common.CompressionUtils; diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/TimedShutoffFirehoseFactory.java b/server/src/main/java/io/druid/segment/realtime/firehose/TimedShutoffFirehoseFactory.java index 68a19c4482e9..8c87ececff60 100644 --- a/server/src/main/java/io/druid/segment/realtime/firehose/TimedShutoffFirehoseFactory.java +++ b/server/src/main/java/io/druid/segment/realtime/firehose/TimedShutoffFirehoseFactory.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.concurrent.Execs; import io.druid.data.input.Firehose; import io.druid.data.input.FirehoseFactory; diff --git a/server/src/main/java/io/druid/segment/realtime/plumber/FlushingPlumber.java b/server/src/main/java/io/druid/segment/realtime/plumber/FlushingPlumber.java index 546181b1fb02..0c01d99d66ab 100644 --- a/server/src/main/java/io/druid/segment/realtime/plumber/FlushingPlumber.java +++ b/server/src/main/java/io/druid/segment/realtime/plumber/FlushingPlumber.java @@ -21,8 +21,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.cache.Cache; import io.druid.client.cache.CacheConfig; import io.druid.common.guava.ThreadRenamingCallable; diff --git a/server/src/main/java/io/druid/segment/realtime/plumber/FlushingPlumberSchool.java b/server/src/main/java/io/druid/segment/realtime/plumber/FlushingPlumberSchool.java index 11416650ff9a..c2ef8344cada 100644 --- a/server/src/main/java/io/druid/segment/realtime/plumber/FlushingPlumberSchool.java +++ b/server/src/main/java/io/druid/segment/realtime/plumber/FlushingPlumberSchool.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.cache.Cache; import io.druid.client.cache.CacheConfig; import io.druid.guice.annotations.Processing; diff --git a/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java b/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java index 345e687cf2e5..2f6a4a1dd995 100644 --- a/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java +++ b/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java @@ -30,8 +30,8 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.primitives.Ints; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.cache.Cache; import io.druid.client.cache.CacheConfig; import io.druid.common.guava.ThreadRenamingCallable; diff --git a/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumberSchool.java b/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumberSchool.java index 8aaf0f2bf7a2..989cf29ea798 100644 --- a/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumberSchool.java +++ b/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumberSchool.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.cache.Cache; import io.druid.client.cache.CacheConfig; import io.druid.guice.annotations.Processing; diff --git a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java index ebe1b13c67f3..daa26455a48a 100644 --- a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java +++ b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java @@ -26,8 +26,8 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.google.inject.Provider; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.selector.Server; import io.druid.guice.annotations.Json; import io.druid.guice.annotations.Smile; diff --git a/server/src/main/java/io/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/io/druid/server/ClientQuerySegmentWalker.java index 993747b7a3e2..768e284ca1c5 100644 --- a/server/src/main/java/io/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/io/druid/server/ClientQuerySegmentWalker.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.CachingClusteredClient; import io.druid.query.FluentQueryRunnerBuilder; import io.druid.query.PostProcessingOperator; diff --git a/server/src/main/java/io/druid/server/QueryLifecycle.java b/server/src/main/java/io/druid/server/QueryLifecycle.java index 1f25f370e279..70486b9b7bb9 100644 --- a/server/src/main/java/io/druid/server/QueryLifecycle.java +++ b/server/src/main/java/io/druid/server/QueryLifecycle.java @@ -21,7 +21,7 @@ import com.google.common.base.Strings; import com.google.common.collect.Iterables; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.DirectDruidClient; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.ISE; diff --git a/server/src/main/java/io/druid/server/QueryLifecycleFactory.java b/server/src/main/java/io/druid/server/QueryLifecycleFactory.java index 9b8b1c5f147b..e22d1ba7ccb3 100644 --- a/server/src/main/java/io/druid/server/QueryLifecycleFactory.java +++ b/server/src/main/java/io/druid/server/QueryLifecycleFactory.java @@ -20,7 +20,7 @@ package io.druid.server; import com.google.inject.Inject; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.guice.LazySingleton; import io.druid.query.GenericQueryMetricsFactory; import io.druid.query.QuerySegmentWalker; diff --git a/server/src/main/java/io/druid/server/QueryResource.java b/server/src/main/java/io/druid/server/QueryResource.java index 8c1208709116..cd8cc14a6000 100644 --- a/server/src/main/java/io/druid/server/QueryResource.java +++ b/server/src/main/java/io/druid/server/QueryResource.java @@ -30,7 +30,7 @@ import com.google.common.collect.Sets; import com.google.common.io.CountingOutputStream; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.client.DirectDruidClient; import io.druid.guice.LazySingleton; import io.druid.guice.annotations.Json; diff --git a/server/src/main/java/io/druid/server/SegmentManager.java b/server/src/main/java/io/druid/server/SegmentManager.java index 3a7ba3cdbfd1..c93528327315 100644 --- a/server/src/main/java/io/druid/server/SegmentManager.java +++ b/server/src/main/java/io/druid/server/SegmentManager.java @@ -22,7 +22,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Ordering; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.common.guava.SettableSupplier; import io.druid.segment.ReferenceCountingSegment; import io.druid.segment.Segment; diff --git a/server/src/main/java/io/druid/server/audit/SQLAuditManager.java b/server/src/main/java/io/druid/server/audit/SQLAuditManager.java index da221868960a..de556c349c43 100644 --- a/server/src/main/java/io/druid/server/audit/SQLAuditManager.java +++ b/server/src/main/java/io/druid/server/audit/SQLAuditManager.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Supplier; import com.google.inject.Inject; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.audit.AuditEntry; import io.druid.audit.AuditManager; diff --git a/server/src/main/java/io/druid/server/audit/SQLAuditManagerProvider.java b/server/src/main/java/io/druid/server/audit/SQLAuditManagerProvider.java index 4ecbc602cf5c..864c81de1c43 100644 --- a/server/src/main/java/io/druid/server/audit/SQLAuditManagerProvider.java +++ b/server/src/main/java/io/druid/server/audit/SQLAuditManagerProvider.java @@ -23,7 +23,7 @@ import com.google.common.base.Supplier; import com.google.common.base.Throwables; import com.google.inject.Inject; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.audit.AuditManager; import io.druid.guice.annotations.Json; import io.druid.java.util.common.lifecycle.Lifecycle; diff --git a/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java b/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java index 1e84da5e20a9..04de12d50970 100644 --- a/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java +++ b/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java @@ -35,7 +35,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.guice.ManageLifecycle; import io.druid.java.util.common.ISE; import io.druid.java.util.common.concurrent.Execs; diff --git a/server/src/main/java/io/druid/server/coordination/ServerManager.java b/server/src/main/java/io/druid/server/coordination/ServerManager.java index 39da89d03fd3..7cee07003cbb 100644 --- a/server/src/main/java/io/druid/server/coordination/ServerManager.java +++ b/server/src/main/java/io/druid/server/coordination/ServerManager.java @@ -23,8 +23,8 @@ import com.google.common.base.Function; import com.google.common.collect.Iterables; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.CachingQueryRunner; import io.druid.client.cache.Cache; import io.druid.client.cache.CacheConfig; diff --git a/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java index 98d226e99448..ea02415b9d23 100644 --- a/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java +++ b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.concurrent.Execs; import io.druid.java.util.common.lifecycle.LifecycleStart; import io.druid.java.util.common.lifecycle.LifecycleStop; diff --git a/server/src/main/java/io/druid/server/coordinator/CachingCostBalancerStrategyFactory.java b/server/src/main/java/io/druid/server/coordinator/CachingCostBalancerStrategyFactory.java index 6a206b0f87a1..a0bd835d1839 100644 --- a/server/src/main/java/io/druid/server/coordinator/CachingCostBalancerStrategyFactory.java +++ b/server/src/main/java/io/druid/server/coordinator/CachingCostBalancerStrategyFactory.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.google.common.util.concurrent.ListeningExecutorService; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.client.ServerInventoryView; import io.druid.client.ServerView; import io.druid.java.util.common.concurrent.Execs; diff --git a/server/src/main/java/io/druid/server/coordinator/CostBalancerStrategy.java b/server/src/main/java/io/druid/server/coordinator/CostBalancerStrategy.java index b9a82fe5468e..c4cd0f7cf882 100644 --- a/server/src/main/java/io/druid/server/coordinator/CostBalancerStrategy.java +++ b/server/src/main/java/io/druid/server/coordinator/CostBalancerStrategy.java @@ -25,7 +25,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.Pair; import io.druid.timeline.DataSegment; diff --git a/server/src/main/java/io/druid/server/coordinator/CuratorLoadQueuePeon.java b/server/src/main/java/io/druid/server/coordinator/CuratorLoadQueuePeon.java index 99761cd43a28..597495493b64 100644 --- a/server/src/main/java/io/druid/server/coordinator/CuratorLoadQueuePeon.java +++ b/server/src/main/java/io/druid/server/coordinator/CuratorLoadQueuePeon.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.ISE; import io.druid.java.util.common.concurrent.ScheduledExecutors; import io.druid.server.coordination.DataSegmentChangeRequest; diff --git a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java index 7239aca5b223..1be52f43cac9 100644 --- a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java +++ b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java @@ -30,8 +30,8 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.DruidDataSource; import io.druid.client.DruidServer; import io.druid.client.ImmutableDruidDataSource; diff --git a/server/src/main/java/io/druid/server/coordinator/DruidCoordinatorRuntimeParams.java b/server/src/main/java/io/druid/server/coordinator/DruidCoordinatorRuntimeParams.java index e2b51771c175..936dd93a8c55 100644 --- a/server/src/main/java/io/druid/server/coordinator/DruidCoordinatorRuntimeParams.java +++ b/server/src/main/java/io/druid/server/coordinator/DruidCoordinatorRuntimeParams.java @@ -21,7 +21,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.ImmutableDruidDataSource; import io.druid.java.util.common.DateTimes; import io.druid.metadata.MetadataRuleManager; diff --git a/server/src/main/java/io/druid/server/coordinator/HttpLoadQueuePeon.java b/server/src/main/java/io/druid/server/coordinator/HttpLoadQueuePeon.java index 17e65ce804c9..dbeeb7386018 100644 --- a/server/src/main/java/io/druid/server/coordinator/HttpLoadQueuePeon.java +++ b/server/src/main/java/io/druid/server/coordinator/HttpLoadQueuePeon.java @@ -28,14 +28,14 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.metamx.common.RE; -import com.metamx.common.StringUtils; -import com.metamx.emitter.EmittingLogger; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.io.AppendableByteArrayInputStream; -import com.metamx.http.client.response.ClientResponse; -import com.metamx.http.client.response.InputStreamResponseHandler; +import io.druid.java.util.common.RE; +import io.druid.java.util.common.StringUtils; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.io.AppendableByteArrayInputStream; +import io.druid.java.util.http.client.response.ClientResponse; +import io.druid.java.util.http.client.response.InputStreamResponseHandler; import io.druid.java.util.common.ISE; import io.druid.java.util.common.concurrent.ScheduledExecutors; import io.druid.server.coordination.DataSegmentChangeCallback; @@ -135,7 +135,7 @@ public HttpLoadQueuePeon( try { this.changeRequestURL = new URL( new URL(baseUrl), - StringUtils.safeFormat( + StringUtils.nonStrictFormat( "druid-internal/v1/segments/changeRequests?timeout=%d", config.getHttpLoadQueuePeonHostTimeout().getMillis() ) @@ -272,7 +272,7 @@ private void logRequestFailure(Throwable t) { log.error( t, - "Request[%s] Failed with code[%s] and status[%s]. Reason[%s].", + "Request[%s] Failed with status[%s]. Reason[%s].", changeRequestURL, responseHandler.status, responseHandler.description diff --git a/server/src/main/java/io/druid/server/coordinator/LoadQueueTaskMaster.java b/server/src/main/java/io/druid/server/coordinator/LoadQueueTaskMaster.java index c24a98a2ff2f..f395e09be17a 100644 --- a/server/src/main/java/io/druid/server/coordinator/LoadQueueTaskMaster.java +++ b/server/src/main/java/io/druid/server/coordinator/LoadQueueTaskMaster.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClient; import io.druid.client.ImmutableDruidServer; import io.druid.guice.annotations.Global; import io.druid.guice.annotations.Json; diff --git a/server/src/main/java/io/druid/server/coordinator/ReplicationThrottler.java b/server/src/main/java/io/druid/server/coordinator/ReplicationThrottler.java index dcf69b5381c1..68f48ece8dc9 100644 --- a/server/src/main/java/io/druid/server/coordinator/ReplicationThrottler.java +++ b/server/src/main/java/io/druid/server/coordinator/ReplicationThrottler.java @@ -21,7 +21,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.StringUtils; import java.util.List; diff --git a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorBalancer.java b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorBalancer.java index 88cd9b50a128..cc266d705df6 100644 --- a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorBalancer.java +++ b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorBalancer.java @@ -20,7 +20,7 @@ package io.druid.server.coordinator.helper; import com.google.common.collect.Lists; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.client.ImmutableDruidServer; import io.druid.java.util.common.StringUtils; import io.druid.server.coordinator.BalancerSegmentHolder; diff --git a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorLogger.java b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorLogger.java index ced5efb1dc78..190a9fb0d507 100644 --- a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorLogger.java +++ b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorLogger.java @@ -19,8 +19,8 @@ package io.druid.server.coordinator.helper; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.client.ImmutableDruidServer; import io.druid.java.util.common.logger.Logger; import io.druid.query.DruidMetrics; diff --git a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorRuleRunner.java b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorRuleRunner.java index 7924e45adbb5..5fa0ff17b220 100644 --- a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorRuleRunner.java +++ b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorRuleRunner.java @@ -20,8 +20,8 @@ package io.druid.server.coordinator.helper; import com.google.common.collect.Lists; -import com.metamx.common.guava.Comparators; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.common.guava.Comparators; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.DateTimes; import io.druid.metadata.MetadataRuleManager; import io.druid.server.coordinator.CoordinatorStats; diff --git a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorSegmentMerger.java b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorSegmentMerger.java index 708e73287a68..9a15c6a3d154 100644 --- a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorSegmentMerger.java +++ b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorSegmentMerger.java @@ -29,7 +29,7 @@ import com.google.common.collect.Multiset; import com.google.common.collect.Ordering; import com.google.inject.Inject; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.client.indexing.IndexingServiceClient; import io.druid.common.config.JacksonConfigManager; import io.druid.java.util.common.DateTimes; diff --git a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorVersionConverter.java b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorVersionConverter.java index d37d0e660f51..f14bf0dbbce1 100644 --- a/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorVersionConverter.java +++ b/server/src/main/java/io/druid/server/coordinator/helper/DruidCoordinatorVersionConverter.java @@ -20,7 +20,7 @@ package io.druid.server.coordinator.helper; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.client.indexing.IndexingServiceClient; import io.druid.common.config.JacksonConfigManager; import io.druid.segment.IndexIO; diff --git a/server/src/main/java/io/druid/server/coordinator/rules/BroadcastDistributionRule.java b/server/src/main/java/io/druid/server/coordinator/rules/BroadcastDistributionRule.java index e32268044b4c..6ce76f7f421f 100644 --- a/server/src/main/java/io/druid/server/coordinator/rules/BroadcastDistributionRule.java +++ b/server/src/main/java/io/druid/server/coordinator/rules/BroadcastDistributionRule.java @@ -19,7 +19,7 @@ package io.druid.server.coordinator.rules; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.server.coordinator.CoordinatorStats; import io.druid.server.coordinator.DruidCoordinator; import io.druid.server.coordinator.DruidCoordinatorRuntimeParams; diff --git a/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java b/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java index 2d6c163ced81..5af4d822c4b3 100644 --- a/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java +++ b/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java @@ -19,7 +19,7 @@ package io.druid.server.coordinator.rules; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.IAE; import io.druid.server.coordinator.CoordinatorStats; import io.druid.server.coordinator.DruidCluster; diff --git a/server/src/main/java/io/druid/server/emitter/ComposingEmitterModule.java b/server/src/main/java/io/druid/server/emitter/ComposingEmitterModule.java index ee014f1d69b9..ce8b34e8c36d 100644 --- a/server/src/main/java/io/druid/server/emitter/ComposingEmitterModule.java +++ b/server/src/main/java/io/druid/server/emitter/ComposingEmitterModule.java @@ -28,8 +28,8 @@ import com.google.inject.Provides; import com.google.inject.name.Named; import com.google.inject.name.Names; -import com.metamx.emitter.core.ComposingEmitter; -import com.metamx.emitter.core.Emitter; +import io.druid.java.util.emitter.core.ComposingEmitter; +import io.druid.java.util.emitter.core.Emitter; import io.druid.guice.JsonConfigProvider; import io.druid.guice.ManageLifecycle; import io.druid.initialization.DruidModule; diff --git a/server/src/main/java/io/druid/server/emitter/EmitterModule.java b/server/src/main/java/io/druid/server/emitter/EmitterModule.java index a0e08225ee1a..7df0137b59c3 100644 --- a/server/src/main/java/io/druid/server/emitter/EmitterModule.java +++ b/server/src/main/java/io/druid/server/emitter/EmitterModule.java @@ -34,9 +34,9 @@ import com.google.inject.multibindings.MapBinder; import com.google.inject.name.Named; import com.google.inject.name.Names; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.core.Emitter; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.guice.LazySingleton; import io.druid.guice.ManageLifecycle; import io.druid.guice.annotations.Self; diff --git a/server/src/main/java/io/druid/server/emitter/HttpEmitterModule.java b/server/src/main/java/io/druid/server/emitter/HttpEmitterModule.java index 36e5f46ab16d..b09ba21d63f1 100644 --- a/server/src/main/java/io/druid/server/emitter/HttpEmitterModule.java +++ b/server/src/main/java/io/druid/server/emitter/HttpEmitterModule.java @@ -26,9 +26,9 @@ import com.google.inject.Provides; import com.google.inject.name.Named; import com.google.inject.util.Providers; -import com.metamx.emitter.core.Emitter; -import com.metamx.emitter.core.HttpEmitterConfig; -import com.metamx.emitter.core.HttpPostEmitter; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.HttpEmitterConfig; +import io.druid.java.util.emitter.core.HttpPostEmitter; import io.druid.guice.JsonConfigProvider; import io.druid.guice.LazySingleton; import io.druid.guice.ManageLifecycle; diff --git a/server/src/main/java/io/druid/server/emitter/HttpEmittingMonitor.java b/server/src/main/java/io/druid/server/emitter/HttpEmittingMonitor.java index 0bb85ef55ce8..f55a7837ab4a 100644 --- a/server/src/main/java/io/druid/server/emitter/HttpEmittingMonitor.java +++ b/server/src/main/java/io/druid/server/emitter/HttpEmittingMonitor.java @@ -21,14 +21,14 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; -import com.metamx.emitter.core.Emitter; -import com.metamx.emitter.core.HttpPostEmitter; -import com.metamx.emitter.core.ParametrizedUriEmitter; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.metrics.AbstractMonitor; -import com.metamx.metrics.FeedDefiningMonitor; -import com.metamx.metrics.HttpPostEmitterMonitor; -import com.metamx.metrics.ParametrizedUriEmitterMonitor; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.HttpPostEmitter; +import io.druid.java.util.emitter.core.ParametrizedUriEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.metrics.AbstractMonitor; +import io.druid.java.util.metrics.FeedDefiningMonitor; +import io.druid.java.util.metrics.HttpPostEmitterMonitor; +import io.druid.java.util.metrics.ParametrizedUriEmitterMonitor; /** * Able to monitor {@link HttpPostEmitter} or {@link ParametrizedUriEmitter}, which is based on the former. diff --git a/server/src/main/java/io/druid/server/emitter/LogEmitterModule.java b/server/src/main/java/io/druid/server/emitter/LogEmitterModule.java index 6372321d70fb..a0b5877e3e5d 100644 --- a/server/src/main/java/io/druid/server/emitter/LogEmitterModule.java +++ b/server/src/main/java/io/druid/server/emitter/LogEmitterModule.java @@ -25,9 +25,9 @@ import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.name.Named; -import com.metamx.emitter.core.Emitter; -import com.metamx.emitter.core.LoggingEmitter; -import com.metamx.emitter.core.LoggingEmitterConfig; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.LoggingEmitter; +import io.druid.java.util.emitter.core.LoggingEmitterConfig; import io.druid.guice.JsonConfigProvider; import io.druid.guice.ManageLifecycle; diff --git a/server/src/main/java/io/druid/server/emitter/NoopEmitterModule.java b/server/src/main/java/io/druid/server/emitter/NoopEmitterModule.java index ea5a838fe65d..3ed5d96c8247 100644 --- a/server/src/main/java/io/druid/server/emitter/NoopEmitterModule.java +++ b/server/src/main/java/io/druid/server/emitter/NoopEmitterModule.java @@ -23,8 +23,8 @@ import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.name.Named; -import com.metamx.emitter.core.Emitter; -import com.metamx.emitter.core.NoopEmitter; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.NoopEmitter; import io.druid.guice.ManageLifecycle; /** diff --git a/server/src/main/java/io/druid/server/emitter/ParametrizedUriEmitterModule.java b/server/src/main/java/io/druid/server/emitter/ParametrizedUriEmitterModule.java index 6cba3bea7696..3ea1f672eb9b 100644 --- a/server/src/main/java/io/druid/server/emitter/ParametrizedUriEmitterModule.java +++ b/server/src/main/java/io/druid/server/emitter/ParametrizedUriEmitterModule.java @@ -25,9 +25,9 @@ import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.name.Named; -import com.metamx.emitter.core.Emitter; -import com.metamx.emitter.core.ParametrizedUriEmitter; -import com.metamx.emitter.core.ParametrizedUriEmitterConfig; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.ParametrizedUriEmitter; +import io.druid.java.util.emitter.core.ParametrizedUriEmitterConfig; import io.druid.guice.JsonConfigProvider; import io.druid.guice.ManageLifecycle; import io.druid.java.util.common.lifecycle.Lifecycle; diff --git a/server/src/main/java/io/druid/server/http/SegmentListerResource.java b/server/src/main/java/io/druid/server/http/SegmentListerResource.java index 600806cc7e13..20f6512e2cd7 100644 --- a/server/src/main/java/io/druid/server/http/SegmentListerResource.java +++ b/server/src/main/java/io/druid/server/http/SegmentListerResource.java @@ -25,7 +25,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import com.sun.jersey.spi.container.ResourceFilters; import io.druid.guice.annotations.Json; import io.druid.guice.annotations.Smile; diff --git a/server/src/main/java/io/druid/server/initialization/jetty/JettyServerModule.java b/server/src/main/java/io/druid/server/initialization/jetty/JettyServerModule.java index f482e91f547b..4eb92b253edd 100644 --- a/server/src/main/java/io/druid/server/initialization/jetty/JettyServerModule.java +++ b/server/src/main/java/io/druid/server/initialization/jetty/JettyServerModule.java @@ -33,10 +33,10 @@ import com.google.inject.Scopes; import com.google.inject.Singleton; import com.google.inject.multibindings.Multibinder; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; -import com.metamx.metrics.AbstractMonitor; -import com.metamx.metrics.MonitorUtils; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; +import io.druid.java.util.metrics.AbstractMonitor; +import io.druid.java.util.metrics.MonitorUtils; import com.sun.jersey.api.core.DefaultResourceConfig; import com.sun.jersey.api.core.ResourceConfig; import com.sun.jersey.guice.JerseyServletModule; diff --git a/server/src/main/java/io/druid/server/log/EmittingRequestLogger.java b/server/src/main/java/io/druid/server/log/EmittingRequestLogger.java index 2e57f747f467..108636a3ddd1 100644 --- a/server/src/main/java/io/druid/server/log/EmittingRequestLogger.java +++ b/server/src/main/java/io/druid/server/log/EmittingRequestLogger.java @@ -22,9 +22,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonValue; import com.google.common.collect.ImmutableMap; -import com.metamx.emitter.core.Event; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceEventBuilder; +import io.druid.java.util.emitter.core.Event; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEventBuilder; import io.druid.query.Query; import io.druid.server.QueryStats; import io.druid.server.RequestLogLine; diff --git a/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java index 534a744e42d0..0b130d8e3002 100644 --- a/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java +++ b/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.inject.Inject; import com.google.inject.Injector; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.java.util.common.logger.Logger; import javax.validation.constraints.NotNull; diff --git a/server/src/main/java/io/druid/server/lookup/cache/LookupCoordinatorManager.java b/server/src/main/java/io/druid/server/lookup/cache/LookupCoordinatorManager.java index 613c9b0d1663..eb8995b840ed 100644 --- a/server/src/main/java/io/druid/server/lookup/cache/LookupCoordinatorManager.java +++ b/server/src/main/java/io/druid/server/lookup/cache/LookupCoordinatorManager.java @@ -36,12 +36,12 @@ import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.ClientResponse; -import com.metamx.http.client.response.HttpResponseHandler; -import com.metamx.http.client.response.SequenceInputStreamResponseHandler; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.ClientResponse; +import io.druid.java.util.http.client.response.HttpResponseHandler; +import io.druid.java.util.http.client.response.SequenceInputStreamResponseHandler; import io.druid.audit.AuditInfo; import io.druid.common.config.JacksonConfigManager; import io.druid.java.util.common.concurrent.Execs; diff --git a/server/src/main/java/io/druid/server/metrics/DruidMonitorSchedulerConfig.java b/server/src/main/java/io/druid/server/metrics/DruidMonitorSchedulerConfig.java index 65838149d48c..770d6bf5d1b9 100644 --- a/server/src/main/java/io/druid/server/metrics/DruidMonitorSchedulerConfig.java +++ b/server/src/main/java/io/druid/server/metrics/DruidMonitorSchedulerConfig.java @@ -20,7 +20,7 @@ package io.druid.server.metrics; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.metrics.MonitorSchedulerConfig; +import io.druid.java.util.metrics.MonitorSchedulerConfig; import org.joda.time.Duration; import org.joda.time.Period; diff --git a/server/src/main/java/io/druid/server/metrics/DruidSysMonitor.java b/server/src/main/java/io/druid/server/metrics/DruidSysMonitor.java index 83f23040519f..9fe9778b9bd2 100644 --- a/server/src/main/java/io/druid/server/metrics/DruidSysMonitor.java +++ b/server/src/main/java/io/druid/server/metrics/DruidSysMonitor.java @@ -21,7 +21,7 @@ import com.google.common.collect.Lists; import com.google.inject.Inject; -import com.metamx.metrics.SysMonitor; +import io.druid.java.util.metrics.SysMonitor; import io.druid.segment.loading.SegmentLoaderConfig; import io.druid.segment.loading.StorageLocationConfig; diff --git a/server/src/main/java/io/druid/server/metrics/EventReceiverFirehoseMonitor.java b/server/src/main/java/io/druid/server/metrics/EventReceiverFirehoseMonitor.java index 66fd4c1a6fda..ad5b36b5bf90 100644 --- a/server/src/main/java/io/druid/server/metrics/EventReceiverFirehoseMonitor.java +++ b/server/src/main/java/io/druid/server/metrics/EventReceiverFirehoseMonitor.java @@ -22,11 +22,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.inject.Inject; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; -import com.metamx.metrics.AbstractMonitor; -import com.metamx.metrics.KeyedDiff; -import com.metamx.metrics.MonitorUtils; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; +import io.druid.java.util.metrics.AbstractMonitor; +import io.druid.java.util.metrics.KeyedDiff; +import io.druid.java.util.metrics.MonitorUtils; import io.druid.query.DruidMetrics; import java.util.Map; diff --git a/server/src/main/java/io/druid/server/metrics/HistoricalMetricsMonitor.java b/server/src/main/java/io/druid/server/metrics/HistoricalMetricsMonitor.java index 3d775984a135..b5954d579fd2 100644 --- a/server/src/main/java/io/druid/server/metrics/HistoricalMetricsMonitor.java +++ b/server/src/main/java/io/druid/server/metrics/HistoricalMetricsMonitor.java @@ -20,9 +20,9 @@ package io.druid.server.metrics; import com.google.inject.Inject; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; -import com.metamx.metrics.AbstractMonitor; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; +import io.druid.java.util.metrics.AbstractMonitor; import io.druid.client.DruidServerConfig; import io.druid.query.DruidMetrics; import io.druid.server.SegmentManager; diff --git a/server/src/main/java/io/druid/server/metrics/MetricsModule.java b/server/src/main/java/io/druid/server/metrics/MetricsModule.java index 1f24942c9351..6ad8ef43b59f 100644 --- a/server/src/main/java/io/druid/server/metrics/MetricsModule.java +++ b/server/src/main/java/io/druid/server/metrics/MetricsModule.java @@ -28,12 +28,12 @@ import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.name.Names; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.metrics.JvmCpuMonitor; -import com.metamx.metrics.JvmMonitor; -import com.metamx.metrics.Monitor; -import com.metamx.metrics.MonitorScheduler; -import com.metamx.metrics.SysMonitor; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.metrics.JvmCpuMonitor; +import io.druid.java.util.metrics.JvmMonitor; +import io.druid.java.util.metrics.Monitor; +import io.druid.java.util.metrics.MonitorScheduler; +import io.druid.java.util.metrics.SysMonitor; import io.druid.guice.DruidBinders; import io.druid.guice.JsonConfigProvider; import io.druid.guice.LazySingleton; diff --git a/server/src/main/java/io/druid/server/metrics/MonitorsConfig.java b/server/src/main/java/io/druid/server/metrics/MonitorsConfig.java index b7ba7912462f..08d48a1d466f 100644 --- a/server/src/main/java/io/druid/server/metrics/MonitorsConfig.java +++ b/server/src/main/java/io/druid/server/metrics/MonitorsConfig.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.metamx.metrics.Monitor; +import io.druid.java.util.metrics.Monitor; import io.druid.query.DruidMetrics; import javax.validation.constraints.NotNull; diff --git a/server/src/main/java/io/druid/server/metrics/QueryCountStatsMonitor.java b/server/src/main/java/io/druid/server/metrics/QueryCountStatsMonitor.java index dcf09c928c83..3a30aab81280 100644 --- a/server/src/main/java/io/druid/server/metrics/QueryCountStatsMonitor.java +++ b/server/src/main/java/io/druid/server/metrics/QueryCountStatsMonitor.java @@ -20,10 +20,10 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; -import com.metamx.metrics.AbstractMonitor; -import com.metamx.metrics.KeyedDiff; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; +import io.druid.java.util.metrics.AbstractMonitor; +import io.druid.java.util.metrics.KeyedDiff; import java.util.Map; diff --git a/server/src/main/java/io/druid/server/router/CoordinatorRuleManager.java b/server/src/main/java/io/druid/server/router/CoordinatorRuleManager.java index 755424475923..85d42a8e4c36 100644 --- a/server/src/main/java/io/druid/server/router/CoordinatorRuleManager.java +++ b/server/src/main/java/io/druid/server/router/CoordinatorRuleManager.java @@ -24,7 +24,7 @@ import com.google.common.base.Supplier; import com.google.common.collect.Lists; import com.google.inject.Inject; -import com.metamx.http.client.response.FullResponseHolder; +import io.druid.java.util.http.client.response.FullResponseHolder; import io.druid.java.util.common.concurrent.Execs; import io.druid.discovery.DruidLeaderClient; import io.druid.guice.ManageLifecycle; diff --git a/server/src/main/java/io/druid/server/router/QueryHostFinder.java b/server/src/main/java/io/druid/server/router/QueryHostFinder.java index 55fcfd6ffc8b..527fd6340d68 100644 --- a/server/src/main/java/io/druid/server/router/QueryHostFinder.java +++ b/server/src/main/java/io/druid/server/router/QueryHostFinder.java @@ -20,7 +20,7 @@ package io.druid.server.router; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.client.selector.Server; import io.druid.java.util.common.ISE; import io.druid.java.util.common.Pair; diff --git a/server/src/main/java/io/druid/server/router/TieredBrokerHostSelector.java b/server/src/main/java/io/druid/server/router/TieredBrokerHostSelector.java index f52151fa23db..373b52f3e82b 100644 --- a/server/src/main/java/io/druid/server/router/TieredBrokerHostSelector.java +++ b/server/src/main/java/io/druid/server/router/TieredBrokerHostSelector.java @@ -25,7 +25,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.client.selector.Server; import io.druid.discovery.DiscoveryDruidNode; import io.druid.discovery.DruidNodeDiscovery; diff --git a/server/src/main/java/io/druid/server/security/Escalator.java b/server/src/main/java/io/druid/server/security/Escalator.java index 67b64363cf38..b8fe920f98fe 100644 --- a/server/src/main/java/io/druid/server/security/Escalator.java +++ b/server/src/main/java/io/druid/server/security/Escalator.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClient; /** * This interface provides methods needed for escalating internal system requests with priveleged authentication diff --git a/server/src/main/java/io/druid/server/security/NoopEscalator.java b/server/src/main/java/io/druid/server/security/NoopEscalator.java index 970c8eb07d70..45f22deecc05 100644 --- a/server/src/main/java/io/druid/server/security/NoopEscalator.java +++ b/server/src/main/java/io/druid/server/security/NoopEscalator.java @@ -19,7 +19,7 @@ package io.druid.server.security; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClient; public class NoopEscalator implements Escalator { diff --git a/server/src/main/java/io/druid/server/security/PreResponseAuthorizationCheckFilter.java b/server/src/main/java/io/druid/server/security/PreResponseAuthorizationCheckFilter.java index ad0144a0b8c7..1c8e5e63914f 100644 --- a/server/src/main/java/io/druid/server/security/PreResponseAuthorizationCheckFilter.java +++ b/server/src/main/java/io/druid/server/security/PreResponseAuthorizationCheckFilter.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.ISE; import io.druid.query.QueryInterruptedException; import io.druid.server.DruidNode; diff --git a/server/src/test/java/io/druid/client/BrokerServerViewTest.java b/server/src/test/java/io/druid/client/BrokerServerViewTest.java index 27a731fb2b78..1024c3878a19 100644 --- a/server/src/test/java/io/druid/client/BrokerServerViewTest.java +++ b/server/src/test/java/io/druid/client/BrokerServerViewTest.java @@ -28,7 +28,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClient; import io.druid.client.selector.HighestPriorityTierSelectorStrategy; import io.druid.client.selector.RandomServerSelectorStrategy; import io.druid.client.selector.ServerSelector; diff --git a/server/src/test/java/io/druid/client/CachingQueryRunnerTest.java b/server/src/test/java/io/druid/client/CachingQueryRunnerTest.java index 0ba202eb0091..3fdc02ddf30d 100644 --- a/server/src/test/java/io/druid/client/CachingQueryRunnerTest.java +++ b/server/src/test/java/io/druid/client/CachingQueryRunnerTest.java @@ -25,7 +25,7 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.cache.Cache; import io.druid.client.cache.CacheConfig; import io.druid.client.cache.CacheStats; diff --git a/server/src/test/java/io/druid/client/DirectDruidClientTest.java b/server/src/test/java/io/druid/client/DirectDruidClientTest.java index ad59b6485580..3347d7d29bc5 100644 --- a/server/src/test/java/io/druid/client/DirectDruidClientTest.java +++ b/server/src/test/java/io/druid/client/DirectDruidClientTest.java @@ -25,10 +25,10 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.HttpResponseHandler; -import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.HttpResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHolder; import io.druid.client.selector.ConnectionCountServerSelectorStrategy; import io.druid.client.selector.HighestPriorityTierSelectorStrategy; import io.druid.client.selector.QueryableDruidServer; diff --git a/server/src/test/java/io/druid/client/HttpServerInventoryViewTest.java b/server/src/test/java/io/druid/client/HttpServerInventoryViewTest.java index 73b6c38aef61..7e10ad8734e5 100644 --- a/server/src/test/java/io/druid/client/HttpServerInventoryViewTest.java +++ b/server/src/test/java/io/druid/client/HttpServerInventoryViewTest.java @@ -25,9 +25,9 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.HttpResponseHandler; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.HttpResponseHandler; import io.druid.discovery.DataNodeService; import io.druid.discovery.DiscoveryDruidNode; import io.druid.discovery.DruidNodeDiscovery; diff --git a/server/src/test/java/io/druid/client/cache/MemcachedCacheTest.java b/server/src/test/java/io/druid/client/cache/MemcachedCacheTest.java index 5bfbfa10038c..805cccecbb34 100644 --- a/server/src/test/java/io/druid/client/cache/MemcachedCacheTest.java +++ b/server/src/test/java/io/druid/client/cache/MemcachedCacheTest.java @@ -30,10 +30,10 @@ import com.google.inject.Injector; import com.google.inject.Module; import com.google.inject.name.Names; -import com.metamx.emitter.core.Emitter; -import com.metamx.emitter.core.Event; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.metrics.AbstractMonitor; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.Event; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.metrics.AbstractMonitor; import io.druid.collections.ResourceHolder; import io.druid.collections.StupidResourceHolder; import io.druid.guice.GuiceInjectors; diff --git a/server/src/test/java/io/druid/curator/discovery/CuratorDruidLeaderSelectorTest.java b/server/src/test/java/io/druid/curator/discovery/CuratorDruidLeaderSelectorTest.java index 66b458ef2209..8182737da63f 100644 --- a/server/src/test/java/io/druid/curator/discovery/CuratorDruidLeaderSelectorTest.java +++ b/server/src/test/java/io/druid/curator/discovery/CuratorDruidLeaderSelectorTest.java @@ -19,8 +19,8 @@ package io.druid.curator.discovery; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.curator.CuratorTestBase; import io.druid.discovery.DruidLeaderSelector; import io.druid.java.util.common.logger.Logger; diff --git a/server/src/test/java/io/druid/discovery/DruidLeaderClientTest.java b/server/src/test/java/io/druid/discovery/DruidLeaderClientTest.java index cc88f6660ff1..f1ed409e9125 100644 --- a/server/src/test/java/io/druid/discovery/DruidLeaderClientTest.java +++ b/server/src/test/java/io/druid/discovery/DruidLeaderClientTest.java @@ -29,8 +29,8 @@ import com.google.inject.name.Named; import com.google.inject.name.Names; import com.google.inject.servlet.GuiceFilter; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; import io.druid.curator.discovery.ServerDiscoverySelector; import io.druid.guice.GuiceInjectors; import io.druid.guice.Jerseys; diff --git a/server/src/test/java/io/druid/initialization/ComposingEmitterModuleTest.java b/server/src/test/java/io/druid/initialization/ComposingEmitterModuleTest.java index 646f1fa9597f..5cc911bc8c1c 100644 --- a/server/src/test/java/io/druid/initialization/ComposingEmitterModuleTest.java +++ b/server/src/test/java/io/druid/initialization/ComposingEmitterModuleTest.java @@ -26,7 +26,7 @@ import com.google.inject.Key; import com.google.inject.Module; import com.google.inject.name.Names; -import com.metamx.emitter.core.Emitter; +import io.druid.java.util.emitter.core.Emitter; import io.druid.guice.DruidGuiceExtensions; import io.druid.guice.LifecycleModule; import io.druid.server.emitter.ComposingEmitterConfig; diff --git a/server/src/test/java/io/druid/metadata/MetadataSegmentManagerTest.java b/server/src/test/java/io/druid/metadata/MetadataSegmentManagerTest.java index 82252935b4b7..648033ee1c03 100644 --- a/server/src/test/java/io/druid/metadata/MetadataSegmentManagerTest.java +++ b/server/src/test/java/io/druid/metadata/MetadataSegmentManagerTest.java @@ -25,7 +25,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.StringUtils; import io.druid.segment.TestHelper; diff --git a/server/src/test/java/io/druid/query/lookup/LookupReferencesManagerTest.java b/server/src/test/java/io/druid/query/lookup/LookupReferencesManagerTest.java index 8cc51b91f0bf..5f9539384dc5 100644 --- a/server/src/test/java/io/druid/query/lookup/LookupReferencesManagerTest.java +++ b/server/src/test/java/io/druid/query/lookup/LookupReferencesManagerTest.java @@ -21,9 +21,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import com.metamx.emitter.EmittingLogger; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.FullResponseHolder; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.FullResponseHolder; import io.druid.discovery.DruidLeaderClient; import io.druid.jackson.DefaultObjectMapper; import io.druid.server.metrics.NoopServiceEmitter; diff --git a/server/src/test/java/io/druid/segment/loading/SegmentLoaderLocalCacheManagerTest.java b/server/src/test/java/io/druid/segment/loading/SegmentLoaderLocalCacheManagerTest.java index ffacdb5f6a96..9b5e20d5c9d0 100644 --- a/server/src/test/java/io/druid/segment/loading/SegmentLoaderLocalCacheManagerTest.java +++ b/server/src/test/java/io/druid/segment/loading/SegmentLoaderLocalCacheManagerTest.java @@ -25,7 +25,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Intervals; import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTester.java b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTester.java index 186434c01167..91ea83cc449e 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTester.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTester.java @@ -21,9 +21,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.core.NoopEmitter; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.core.NoopEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.cache.CacheConfig; import io.druid.client.cache.MapCache; import io.druid.data.input.impl.DimensionsSpec; diff --git a/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java b/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java index cfddca4aa90e..20375d45754a 100644 --- a/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java +++ b/server/src/test/java/io/druid/segment/realtime/plumber/RealtimePlumberSchoolTest.java @@ -26,7 +26,7 @@ import com.google.common.collect.Maps; import com.google.common.io.Files; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.cache.MapCache; import io.druid.data.input.Committer; import io.druid.data.input.InputRow; diff --git a/server/src/test/java/io/druid/server/QueryResourceTest.java b/server/src/test/java/io/druid/server/QueryResourceTest.java index c07002b05e98..b099f002d267 100644 --- a/server/src/test/java/io/druid/server/QueryResourceTest.java +++ b/server/src/test/java/io/druid/server/QueryResourceTest.java @@ -25,8 +25,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.java.util.common.concurrent.Execs; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.guava.Sequence; diff --git a/server/src/test/java/io/druid/server/coordination/ServerManagerTest.java b/server/src/test/java/io/druid/server/coordination/ServerManagerTest.java index e53e756f094c..6670a3278ec2 100644 --- a/server/src/test/java/io/druid/server/coordination/ServerManagerTest.java +++ b/server/src/test/java/io/druid/server/coordination/ServerManagerTest.java @@ -27,7 +27,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.client.cache.CacheConfig; import io.druid.client.cache.LocalCacheProvider; import io.druid.jackson.DefaultObjectMapper; diff --git a/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java b/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java index 1ef643f81a96..d9e19034ad39 100644 --- a/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java +++ b/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.curator.CuratorTestBase; import io.druid.java.util.common.Intervals; import io.druid.segment.IndexIO; diff --git a/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorBalancerProfiler.java b/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorBalancerProfiler.java index b49445905f47..9f7e1ec85181 100644 --- a/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorBalancerProfiler.java +++ b/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorBalancerProfiler.java @@ -24,8 +24,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.DruidServer; import io.druid.client.ImmutableDruidServer; import io.druid.java.util.common.DateTimes; diff --git a/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorRuleRunnerTest.java b/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorRuleRunnerTest.java index 7f2515415099..9c17bab60c38 100644 --- a/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorRuleRunnerTest.java +++ b/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorRuleRunnerTest.java @@ -24,9 +24,9 @@ import com.google.common.collect.Maps; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceEventBuilder; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEventBuilder; import io.druid.client.DruidServer; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; diff --git a/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorSegmentMergerTest.java b/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorSegmentMergerTest.java index 686d7399d57e..71cfb62bb4e9 100644 --- a/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorSegmentMergerTest.java +++ b/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorSegmentMergerTest.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.indexing.IndexingServiceClient; import io.druid.common.config.JacksonConfigManager; import io.druid.java.util.common.Intervals; diff --git a/server/src/test/java/io/druid/server/coordinator/HttpLoadQueuePeonTest.java b/server/src/test/java/io/druid/server/coordinator/HttpLoadQueuePeonTest.java index 885058f141a7..72fb9a36a5d5 100644 --- a/server/src/test/java/io/druid/server/coordinator/HttpLoadQueuePeonTest.java +++ b/server/src/test/java/io/druid/server/coordinator/HttpLoadQueuePeonTest.java @@ -24,9 +24,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.HttpResponseHandler; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.HttpResponseHandler; import io.druid.discovery.DiscoveryDruidNode; import io.druid.discovery.DruidNodeDiscovery; import io.druid.java.util.common.Intervals; diff --git a/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java b/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java index 76174172d9fe..0388318b7262 100644 --- a/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java +++ b/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java @@ -27,11 +27,11 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; //CHECKSTYLE.OFF: Regexp -import com.metamx.common.logger.Logger; +import io.druid.java.util.common.logger.Logger; //CHECKSTYLE.ON: Regexp -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.core.LoggingEmitter; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.core.LoggingEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.DruidServer; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.DateTimes; diff --git a/server/src/test/java/io/druid/server/emitter/EmitterModuleTest.java b/server/src/test/java/io/druid/server/emitter/EmitterModuleTest.java index 0cb94f6d447f..50a2956bffed 100644 --- a/server/src/test/java/io/druid/server/emitter/EmitterModuleTest.java +++ b/server/src/test/java/io/druid/server/emitter/EmitterModuleTest.java @@ -24,8 +24,8 @@ import com.google.inject.Guice; import com.google.inject.Injector; import com.google.inject.Module; -import com.metamx.emitter.core.Emitter; -import com.metamx.emitter.core.ParametrizedUriEmitter; +import io.druid.java.util.emitter.core.Emitter; +import io.druid.java.util.emitter.core.ParametrizedUriEmitter; import io.druid.guice.DruidGuiceExtensions; import io.druid.guice.JsonConfigurator; import io.druid.guice.LazySingleton; diff --git a/server/src/test/java/io/druid/server/http/security/PreResponseAuthorizationCheckFilterTest.java b/server/src/test/java/io/druid/server/http/security/PreResponseAuthorizationCheckFilterTest.java index 7503b4041397..c027bde092c5 100644 --- a/server/src/test/java/io/druid/server/http/security/PreResponseAuthorizationCheckFilterTest.java +++ b/server/src/test/java/io/druid/server/http/security/PreResponseAuthorizationCheckFilterTest.java @@ -20,8 +20,8 @@ package io.druid.server.http.security; import com.google.common.collect.Lists; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.ISE; import io.druid.server.security.AllowAllAuthenticator; diff --git a/server/src/test/java/io/druid/server/initialization/BaseJettyTest.java b/server/src/test/java/io/druid/server/initialization/BaseJettyTest.java index 643e8494ad75..484204ab57da 100644 --- a/server/src/test/java/io/druid/server/initialization/BaseJettyTest.java +++ b/server/src/test/java/io/druid/server/initialization/BaseJettyTest.java @@ -23,9 +23,9 @@ import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.servlet.GuiceFilter; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.HttpClientConfig; -import com.metamx.http.client.HttpClientInit; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClientConfig; +import io.druid.java.util.http.client.HttpClientInit; import io.druid.guice.annotations.Self; import io.druid.guice.http.LifecycleUtils; import io.druid.java.util.common.lifecycle.Lifecycle; diff --git a/server/src/test/java/io/druid/server/initialization/JettyQosTest.java b/server/src/test/java/io/druid/server/initialization/JettyQosTest.java index 44c9cb721f40..de852e651f20 100644 --- a/server/src/test/java/io/druid/server/initialization/JettyQosTest.java +++ b/server/src/test/java/io/druid/server/initialization/JettyQosTest.java @@ -28,10 +28,10 @@ import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Module; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.StatusResponseHandler; -import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.StatusResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHolder; import io.druid.java.util.common.concurrent.Execs; import io.druid.guice.GuiceInjectors; import io.druid.guice.Jerseys; diff --git a/server/src/test/java/io/druid/server/initialization/JettyTest.java b/server/src/test/java/io/druid/server/initialization/JettyTest.java index 207c7d853614..ef0610fb58dc 100644 --- a/server/src/test/java/io/druid/server/initialization/JettyTest.java +++ b/server/src/test/java/io/druid/server/initialization/JettyTest.java @@ -27,10 +27,10 @@ import com.google.inject.Key; import com.google.inject.Module; import com.google.inject.multibindings.Multibinder; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.InputStreamResponseHandler; -import com.metamx.http.client.response.StatusResponseHandler; -import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.InputStreamResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHolder; import io.druid.guice.GuiceInjectors; import io.druid.guice.Jerseys; import io.druid.guice.JsonConfigProvider; diff --git a/server/src/test/java/io/druid/server/lookup/cache/LookupCoordinatorManagerTest.java b/server/src/test/java/io/druid/server/lookup/cache/LookupCoordinatorManagerTest.java index b128b0bc9b5d..b0b9aaf71efd 100644 --- a/server/src/test/java/io/druid/server/lookup/cache/LookupCoordinatorManagerTest.java +++ b/server/src/test/java/io/druid/server/lookup/cache/LookupCoordinatorManagerTest.java @@ -26,14 +26,14 @@ import com.google.common.collect.ImmutableSet; import com.google.common.net.HostAndPort; import com.google.common.util.concurrent.SettableFuture; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.core.Event; -import com.metamx.emitter.core.LoggingEmitter; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.Request; -import com.metamx.http.client.response.HttpResponseHandler; -import com.metamx.http.client.response.SequenceInputStreamResponseHandler; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.core.Event; +import io.druid.java.util.emitter.core.LoggingEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.HttpResponseHandler; +import io.druid.java.util.http.client.response.SequenceInputStreamResponseHandler; import io.druid.audit.AuditInfo; import io.druid.common.config.JacksonConfigManager; import io.druid.discovery.DruidNodeDiscoveryProvider; diff --git a/server/src/test/java/io/druid/server/metrics/HistoricalMetricsMonitorTest.java b/server/src/test/java/io/druid/server/metrics/HistoricalMetricsMonitorTest.java index 652600a7d9ba..b87139a27826 100644 --- a/server/src/test/java/io/druid/server/metrics/HistoricalMetricsMonitorTest.java +++ b/server/src/test/java/io/druid/server/metrics/HistoricalMetricsMonitorTest.java @@ -23,9 +23,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceEventBuilder; -import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceEventBuilder; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.client.DruidServerConfig; import io.druid.java.util.common.Intervals; import io.druid.server.SegmentManager; diff --git a/server/src/test/java/io/druid/server/metrics/NoopServiceEmitter.java b/server/src/test/java/io/druid/server/metrics/NoopServiceEmitter.java index 99399bf21875..e16e7723b7ed 100644 --- a/server/src/test/java/io/druid/server/metrics/NoopServiceEmitter.java +++ b/server/src/test/java/io/druid/server/metrics/NoopServiceEmitter.java @@ -19,8 +19,8 @@ package io.druid.server.metrics; -import com.metamx.emitter.core.Event; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.core.Event; +import io.druid.java.util.emitter.service.ServiceEmitter; public class NoopServiceEmitter extends ServiceEmitter { diff --git a/services/src/main/java/io/druid/cli/CliCoordinator.java b/services/src/main/java/io/druid/cli/CliCoordinator.java index af706fdeb4a3..4156b65e460e 100644 --- a/services/src/main/java/io/druid/cli/CliCoordinator.java +++ b/services/src/main/java/io/druid/cli/CliCoordinator.java @@ -28,7 +28,7 @@ import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.name.Names; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClient; import io.airlift.airline.Command; import io.druid.audit.AuditManager; import io.druid.client.CoordinatorServerView; diff --git a/services/src/main/java/io/druid/cli/CliRouter.java b/services/src/main/java/io/druid/cli/CliRouter.java index 6f35939bbc89..b480c13125f0 100644 --- a/services/src/main/java/io/druid/cli/CliRouter.java +++ b/services/src/main/java/io/druid/cli/CliRouter.java @@ -26,7 +26,7 @@ import com.google.inject.Provides; import com.google.inject.TypeLiteral; import com.google.inject.name.Names; -import com.metamx.http.client.HttpClient; +import io.druid.java.util.http.client.HttpClient; import io.airlift.airline.Command; import io.druid.curator.discovery.DiscoveryModule; import io.druid.curator.discovery.ServerDiscoveryFactory; diff --git a/sql/src/main/java/io/druid/sql/avatica/AvaticaMonitor.java b/sql/src/main/java/io/druid/sql/avatica/AvaticaMonitor.java index 88097653c9f2..c14e1d812254 100644 --- a/sql/src/main/java/io/druid/sql/avatica/AvaticaMonitor.java +++ b/sql/src/main/java/io/druid/sql/avatica/AvaticaMonitor.java @@ -19,9 +19,9 @@ package io.druid.sql.avatica; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; -import com.metamx.metrics.AbstractMonitor; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.service.ServiceMetricEvent; +import io.druid.java.util.metrics.AbstractMonitor; import io.druid.java.util.common.logger.Logger; import org.apache.calcite.avatica.metrics.Counter; import org.apache.calcite.avatica.metrics.Gauge; diff --git a/sql/src/main/java/io/druid/sql/calcite/schema/DruidSchema.java b/sql/src/main/java/io/druid/sql/calcite/schema/DruidSchema.java index af52973cfcfe..a24b13424ffb 100644 --- a/sql/src/main/java/io/druid/sql/calcite/schema/DruidSchema.java +++ b/sql/src/main/java/io/druid/sql/calcite/schema/DruidSchema.java @@ -28,7 +28,7 @@ import com.google.common.collect.Sets; import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; -import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.client.ServerView; import io.druid.client.TimelineServerView; import io.druid.guice.ManageLifecycle; diff --git a/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java index 5db831df0327..bfff546e11d0 100644 --- a/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java +++ b/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java @@ -32,8 +32,8 @@ import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Module; -import com.metamx.emitter.core.NoopEmitter; -import com.metamx.emitter.service.ServiceEmitter; +import io.druid.java.util.emitter.core.NoopEmitter; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.collections.StupidPool; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; From 53eb4c702bb12049cd735dd5c1fddcaef771dfd0 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Thu, 15 Feb 2018 13:52:05 -0800 Subject: [PATCH 26/82] fix segment info in Kafka indexing service docs (#5390) (#5393) * fix segment info in Kafka indexing service docs * review updates --- .../extensions-core/kafka-ingestion.md | 51 +++++-------------- 1 file changed, 14 insertions(+), 37 deletions(-) diff --git a/docs/content/development/extensions-core/kafka-ingestion.md b/docs/content/development/extensions-core/kafka-ingestion.md index b32c479fe7ba..c254aad114e2 100644 --- a/docs/content/development/extensions-core/kafka-ingestion.md +++ b/docs/content/development/extensions-core/kafka-ingestion.md @@ -311,40 +311,17 @@ In this way, configuration changes can be applied without requiring any pause in ### On the Subject of Segments -The Kafka indexing service may generate a significantly large number of segments which over time will cause query -performance issues if not properly managed. One important characteristic to understand is that the Kafka indexing task -will generate a Druid partition in each segment granularity interval for each partition in the Kafka topic. As an -example, if you are ingesting realtime data and your segment granularity is 15 minutes with 10 partitions in the Kafka -topic, you would generate a minimum of 40 segments an hour. This is a limitation imposed by the Kafka architecture which -guarantees delivery order within a partition but not across partitions. Therefore as a consumer of Kafka, in order to -generate segments deterministically (and be able to provide exactly-once ingestion semantics) partitions need to be -handled separately. - -Compounding this, if your taskDuration was also set to 15 minutes, you would actually generate 80 segments an hour since -any given 15 minute interval would be handled by two tasks. For an example of this behavior, let's say we started the -supervisor at 9:05 with a 15 minute segment granularity. The first task would create a segment for 9:00-9:15 and a -segment for 9:15-9:30 before stopping at 9:20. A second task would be created at 9:20 which would create another segment -for 9:15-9:30 and a segment for 9:30-9:45 before stopping at 9:35. Hence, if taskDuration and segmentGranularity are the -same duration, you will get two tasks generating a segment for each segment granularity interval. - -Understanding this behavior is the first step to managing the number of segments produced. Some recommendations for -keeping the number of segments low are: - - * Keep the number of Kafka partitions to the minimum required to sustain the required throughput for your event streams. - * Increase segment granularity and task duration so that more events are written into the same segment. One - consideration here is that segments are only handed off to historical nodes after the task duration has elapsed. - Since workers tend to be configured with less query-serving resources than historical nodes, query performance may - suffer if tasks run excessively long without handing off segments. - -In many production installations which have been ingesting events for a long period of time, these suggestions alone -will not be sufficient to keep the number of segments at an optimal level. It is recommended that scheduled re-indexing -tasks be run to merge segments together into new segments of an ideal size (in the range of ~500-700 MB per segment). -Currently, the recommended way of doing this is by running periodic Hadoop batch ingestion jobs and using a `dataSource` -inputSpec to read from the segments generated by the Kafka indexing tasks. Details on how to do this can be found under -['Updating Existing Data'](../../ingestion/update-existing-data.html). Note that the Merge Task and Append Task described -[here](../../ingestion/tasks.html) will not work as they require unsharded segments while Kafka indexing tasks always -generated sharded segments. - -There is ongoing work to support automatic segment compaction of sharded segments as well as compaction not requiring -Hadoop (see [here](https://github.com/druid-io/druid/pull/1998) and [here](https://github.com/druid-io/druid/pull/3611) -for related PRs). +Each Kafka Indexing Task puts events consumed from Kafka partitions assigned to it in a single segment for each segment +granular interval until maxRowsPerSegment limit is reached, at this point a new partition for this segment granularity is +created for further events. Kafka Indexing Task also does incremental hand-offs which means that all the segments created by a +task will not be held up till the task duration is over. As soon as maxRowsPerSegment limit is hit, all the segments held +by the task at that point in time will be handed-off and new set of segments will be created for further events. +This means that the task can run for longer durations of time without accumulating old segments locally on Middle Manager +nodes and it is encouraged to do so. + +Kafka Indexing Service may still produce some small segments. Lets say the task duration is 4 hours, segment granularity +is set to an HOUR and Supervisor was started at 9:10 then after 4 hours at 13:10, new set of tasks will be started and +events for the interval 13:00 - 14:00 may be split across previous and new set of tasks. If you see it becoming a problem then +one can schedule re-indexing tasks be run to merge segments together into new segments of an ideal size (in the range of ~500-700 MB per segment). +There is also ongoing work to support automatic segment compaction of sharded segments as well as compaction not requiring +Hadoop (see [here](https://github.com/druid-io/druid/pull/5102)). From 93b79eda8a9c34a756279b707367df9dc978d3c9 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Thu, 15 Feb 2018 15:23:35 -0800 Subject: [PATCH 27/82] Debug logging in HttpPostEmitter and Batch (#5365) (#5394) --- .../druid/java/util/emitter/core/Batch.java | 27 +++++++++++++++++-- .../util/emitter/core/HttpPostEmitter.java | 4 +++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/Batch.java b/java-util/src/main/java/io/druid/java/util/emitter/core/Batch.java index 16dbab4532f1..aaf9f11fee82 100644 --- a/java-util/src/main/java/io/druid/java/util/emitter/core/Batch.java +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/Batch.java @@ -20,6 +20,7 @@ package io.druid.java.util.emitter.core; import com.google.common.base.Preconditions; +import io.druid.java.util.common.logger.Logger; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.AbstractQueuedLongSynchronizer; @@ -44,6 +45,8 @@ */ class Batch extends AbstractQueuedLongSynchronizer { + private static final Logger log = new Logger(Batch.class); + private static final long PARTY = 1L << 32; private static final long SEAL_BIT = 1L << 63; @@ -278,6 +281,9 @@ protected boolean tryReleaseShared(long tag) if (compareAndSetState(state, newState)) { // Ensures only one thread calls emitter.onSealExclusive() for each batch. if (!isSealed(state)) { + log.debug("Unlocked and sealed batch [%d]", batchNumber); + debugLogState("old state", state); + debugLogState("new state", newState); emitter.onSealExclusive( this, firstEventTimestamp > 0 ? System.currentTimeMillis() - firstEventTimestamp : -1 @@ -296,6 +302,9 @@ protected boolean tryReleaseShared(long tag) } long newState = state | SEAL_BIT; if (compareAndSetState(state, newState)) { + log.debug("Sealed batch [%d]", batchNumber); + debugLogState("old state", state); + debugLogState("new state", newState); emitter.onSealExclusive( this, firstEventTimestamp > 0 ? System.currentTimeMillis() - firstEventTimestamp : -1 @@ -323,10 +332,24 @@ protected long tryAcquireShared(long ignored) public String toString() { long state = getState(); - return "Batch{" - + "bufferWatermark=" + bufferWatermark(state) + + return "Batch{" + + "batchNumber=" + batchNumber + + ", bufferWatermark=" + bufferWatermark(state) + ", parties=" + parties(state) + ", isSealed=" + isSealed(state) + "}"; } + + private static void debugLogState(String name, long state) + { + if (log.isDebugEnabled()) { + log.debug( + "%s[bufferWatermark=%d, parties=%d, isSealed=%s]", + name, + bufferWatermark(state), + parties(state), + isSealed(state) + ); + } + } } diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java b/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java index a9303d9ccaed..82eb6eda1676 100644 --- a/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java @@ -37,6 +37,7 @@ import org.asynchttpclient.RequestBuilder; import org.asynchttpclient.Response; +import javax.annotation.Nullable; import java.io.Closeable; import java.io.Flushable; import java.io.IOException; @@ -224,6 +225,7 @@ public void emit(Event event) } @VisibleForTesting + @Nullable Batch emitAndReturnBatch(Event event) { awaitStarted(); @@ -252,6 +254,8 @@ Batch emitAndReturnBatch(Event event) } if (batch.tryAddEvent(eventBytes)) { return batch; + } else { + log.debug("Failed to emit an event in batch [%s]", batch); } // Spin loop, until the thread calling onSealExclusive() updates the concurrentBatch. This update becomes visible // eventually, because concurrentBatch.get() is a volatile read. From 359d93ef4cd77c75586cce407bab88da74eca5b0 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Thu, 15 Feb 2018 16:41:32 -0800 Subject: [PATCH 28/82] Make HttpPostEmitter more robust in the face of serious errors (like OutOfMemoryError) during onSealExclusive() (#5386) (#5395) --- .../druid/java/util/emitter/core/Batch.java | 6 +- .../util/emitter/core/HttpPostEmitter.java | 83 +++++++++++++++---- 2 files changed, 72 insertions(+), 17 deletions(-) diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/Batch.java b/java-util/src/main/java/io/druid/java/util/emitter/core/Batch.java index aaf9f11fee82..818f54f91a94 100644 --- a/java-util/src/main/java/io/druid/java/util/emitter/core/Batch.java +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/Batch.java @@ -90,8 +90,12 @@ private static boolean isEmittingAllowed(long state) /** * Ordering number of this batch, as they filled & emitted in {@link HttpPostEmitter} serially, starting from 0. + * It's a boxed Integer rather than int, because we want to minimize the number of allocations done in + * {@link HttpPostEmitter#onSealExclusive} and so the probability of {@link OutOfMemoryError}. + * @see HttpPostEmitter#onSealExclusive + * @see HttpPostEmitter#concurrentBatch */ - final int batchNumber; + final Integer batchNumber; /** * The number of events in this batch, needed for event count-based batch emitting. diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java b/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java index 82eb6eda1676..f954d458bf75 100644 --- a/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java @@ -111,9 +111,12 @@ public class HttpPostEmitter implements Flushable, Closeable, Emitter private final AtomicInteger approximateBuffersToReuseCount = new AtomicInteger(); /** - * concurrentBatch.get() == null means the service is closed. + * concurrentBatch.get() == null means the service is closed. concurrentBatch.get() is the instance of Integer, + * it means that some thread has failed with a serious error during {@link #onSealExclusive} (with the batch number + * corresponding to the Integer object) and {@link #tryRecoverCurrentBatch} needs to be called. Otherwise (i. e. + * normally), an instance of {@link Batch} is stored in this atomic reference. */ - private final AtomicReference concurrentBatch = new AtomicReference<>(); + private final AtomicReference concurrentBatch = new AtomicReference<>(); private final ConcurrentLinkedDeque buffersToEmit = new ConcurrentLinkedDeque<>(); /** @@ -248,10 +251,15 @@ Batch emitAndReturnBatch(Event event) } while (true) { - Batch batch = concurrentBatch.get(); - if (batch == null) { + Object batchObj = concurrentBatch.get(); + if (batchObj instanceof Integer) { + tryRecoverCurrentBatch((Integer) batchObj); + continue; + } + if (batchObj == null) { throw new RejectedExecutionException("Service is closed."); } + Batch batch = (Batch) batchObj; if (batch.tryAddEvent(eventBytes)) { return batch; } else { @@ -294,6 +302,25 @@ private void writeLargeEvent(byte[] eventBytes) * Called from {@link Batch} only once for each Batch in existence. */ void onSealExclusive(Batch batch, long elapsedTimeMillis) + { + try { + doOnSealExclusive(batch, elapsedTimeMillis); + } + catch (Throwable t) { + try { + if (!concurrentBatch.compareAndSet(batch, batch.batchNumber)) { + log.error("Unexpected failure to set currentBatch to the failed Batch.batchNumber"); + } + log.error(t, "Serious error during onSealExclusive(), set currentBatch to the failed Batch.batchNumber"); + } + catch (Throwable t2) { + t.addSuppressed(t2); + } + throw t; + } + } + + private void doOnSealExclusive(Batch batch, long elapsedTimeMillis) { batchFillingTimeCounter.add((int) Math.max(elapsedTimeMillis, 0)); if (elapsedTimeMillis > 0) { @@ -305,13 +332,29 @@ void onSealExclusive(Batch batch, long elapsedTimeMillis) wakeUpEmittingThread(); if (!isTerminated()) { int nextBatchNumber = EmittedBatchCounter.nextBatchNumber(batch.batchNumber); - if (!concurrentBatch.compareAndSet(batch, new Batch(this, acquireBuffer(), nextBatchNumber))) { - // If compareAndSet failed, the service is closed concurrently. + byte[] newBuffer = acquireBuffer(); + if (!concurrentBatch.compareAndSet(batch, new Batch(this, newBuffer, nextBatchNumber))) { + buffersToReuse.add(newBuffer); + // If compareAndSet failed, the service should be closed concurrently, i. e. we expect isTerminated() = true. + // If we don't see this, there should be some bug in HttpPostEmitter. Preconditions.checkState(isTerminated()); } } } + private void tryRecoverCurrentBatch(Integer failedBatchNumber) + { + log.info("Trying to recover currentBatch"); + int nextBatchNumber = EmittedBatchCounter.nextBatchNumber(failedBatchNumber); + byte[] newBuffer = acquireBuffer(); + if (concurrentBatch.compareAndSet(failedBatchNumber, new Batch(this, newBuffer, nextBatchNumber))) { + log.info("Successfully recovered currentBatch"); + } else { + // It's normal, a concurrent thread could succeed to recover first. + buffersToReuse.add(newBuffer); + } + } + private void addBatchToEmitQueue(Batch batch) { limitBuffersToEmitSize(); @@ -363,7 +406,10 @@ private void wakeUpEmittingThread() public void flush() throws IOException { awaitStarted(); - flush(concurrentBatch.get()); + Object batchObj = concurrentBatch.get(); + if (batchObj instanceof Batch) { + flush((Batch) batchObj); + } } private void flush(Batch batch) throws IOException @@ -396,8 +442,10 @@ public void close() throws IOException synchronized (startLock) { if (running) { running = false; - Batch lastBatch = concurrentBatch.getAndSet(null); - flush(lastBatch); + Object lastBatch = concurrentBatch.getAndSet(null); + if (lastBatch instanceof Batch) { + flush((Batch) lastBatch); + } emittingThread.shuttingDown = true; // EmittingThread is interrupted after the last batch is flushed. emittingThread.interrupt(); @@ -478,16 +526,19 @@ private boolean needsToShutdown() { boolean needsToShutdown = Thread.interrupted() || shuttingDown; if (needsToShutdown) { - Batch lastBatch = concurrentBatch.getAndSet(null); - if (lastBatch != null) { - lastBatch.seal(); + Object lastBatch = concurrentBatch.getAndSet(null); + if (lastBatch instanceof Batch) { + ((Batch) lastBatch).seal(); } } else { - Batch batch = concurrentBatch.get(); - if (batch != null) { - batch.sealIfFlushNeeded(); + Object batch = concurrentBatch.get(); + if (batch instanceof Batch) { + ((Batch) batch).sealIfFlushNeeded(); } else { - needsToShutdown = true; + // batch == null means that HttpPostEmitter is terminated. Batch object could also be Integer, if some + // thread just failed with a serious error in onSealExclusive(), in this case we don't want to shutdown + // the emitter thread. + needsToShutdown = batch == null; } } return needsToShutdown; From 13b6ffd891d563ea8278f8316d084fa41eddfba9 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Fri, 16 Feb 2018 12:58:36 -0800 Subject: [PATCH 29/82] Fix early publishing to early pushing in batch indexing & refactor appenderatorDriver (#5397) --- .../druid/indexing/kafka/KafkaIndexTask.java | 38 +- .../druid/indexing/common/task/IndexTask.java | 260 +++--- .../common/task/CompactionTaskTest.java | 3 +- .../indexing/common/task/IndexTaskTest.java | 13 +- .../indexing/common/task/TaskSerdeTest.java | 4 +- .../indexing/overlord/TaskLifecycleTest.java | 6 +- .../realtime/appenderator/Appenderator.java | 31 +- .../appenderator/AppenderatorDriver.java | 776 ------------------ .../AppenderatorDriverAddResult.java | 2 +- .../AppenderatorDriverMetadata.java | 20 +- .../appenderator/AppenderatorImpl.java | 148 ++-- .../appenderator/BaseAppenderatorDriver.java | 541 ++++++++++++ .../appenderator/BatchAppenderatorDriver.java | 209 +++++ .../appenderator/SegmentWithState.java | 153 ++++ .../appenderator/SegmentsAndMetadata.java | 4 +- .../StreamAppenderatorDriver.java | 416 ++++++++++ .../TransactionalSegmentPublisher.java | 3 +- .../BatchAppenderatorDriverTest.java | 189 +++++ .../appenderator/SegmentWithStateTest.java | 54 ++ ... => StreamAppenderatorDriverFailTest.java} | 28 +- ...java => StreamAppenderatorDriverTest.java} | 62 +- .../appenderator/TestUsedSegmentChecker.java | 66 ++ 22 files changed, 1932 insertions(+), 1094 deletions(-) delete mode 100644 server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriver.java create mode 100644 server/src/main/java/io/druid/segment/realtime/appenderator/BaseAppenderatorDriver.java create mode 100644 server/src/main/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriver.java create mode 100644 server/src/main/java/io/druid/segment/realtime/appenderator/SegmentWithState.java create mode 100644 server/src/main/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriver.java create mode 100644 server/src/test/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriverTest.java create mode 100644 server/src/test/java/io/druid/segment/realtime/appenderator/SegmentWithStateTest.java rename server/src/test/java/io/druid/segment/realtime/appenderator/{AppenderatorDriverFailTest.java => StreamAppenderatorDriverFailTest.java} (94%) rename server/src/test/java/io/druid/segment/realtime/appenderator/{AppenderatorDriverTest.java => StreamAppenderatorDriverTest.java} (90%) create mode 100644 server/src/test/java/io/druid/segment/realtime/appenderator/TestUsedSegmentChecker.java diff --git a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java index 26dfb19b712b..b80ec2493288 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java +++ b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java @@ -82,9 +82,9 @@ import io.druid.segment.realtime.FireDepartmentMetrics; import io.druid.segment.realtime.RealtimeMetricsMonitor; import io.druid.segment.realtime.appenderator.Appenderator; -import io.druid.segment.realtime.appenderator.AppenderatorDriver; import io.druid.segment.realtime.appenderator.AppenderatorDriverAddResult; import io.druid.segment.realtime.appenderator.Appenderators; +import io.druid.segment.realtime.appenderator.StreamAppenderatorDriver; import io.druid.segment.realtime.appenderator.SegmentIdentifier; import io.druid.segment.realtime.appenderator.SegmentsAndMetadata; import io.druid.segment.realtime.appenderator.TransactionalSegmentPublisher; @@ -136,7 +136,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; @@ -187,7 +186,7 @@ public enum Status private TaskToolbox toolbox; private volatile Appenderator appenderator = null; - private volatile AppenderatorDriver driver = null; + private volatile StreamAppenderatorDriver driver = null; private volatile FireDepartmentMetrics fireDepartmentMetrics = null; private volatile DateTime startTime; private volatile Status status = Status.NOT_STARTED; // this is only ever set by the task runner thread (runThread) @@ -372,7 +371,7 @@ private void createAndStartPublishExecutor() Joiner.on(", ").join( result.getSegments().stream().map(DataSegment::getIdentifier).collect(Collectors.toList()) ), - result.getCommitMetadata() + Preconditions.checkNotNull(result.getCommitMetadata(), "commitMetadata") ); } @@ -423,9 +422,11 @@ public TaskStatus run(final TaskToolbox toolbox) throws Exception if (getContext() != null && getContext().get("checkpoints") != null) { log.info("Got checkpoints [%s]", (String) getContext().get("checkpoints")); final TreeMap> checkpoints = toolbox.getObjectMapper().readValue( - (String) getContext().get("checkpoints"), new TypeReference>>() + (String) getContext().get("checkpoints"), + new TypeReference>>() { - }); + } + ); Iterator>> sequenceOffsets = checkpoints.entrySet().iterator(); Map.Entry> previous = sequenceOffsets.next(); @@ -709,7 +710,7 @@ public void run() if (addResult.isOk()) { // If the number of rows in the segment exceeds the threshold after adding a row, - // move the segment out from the active segments of AppenderatorDriver to make a new segment. + // move the segment out from the active segments of BaseAppenderatorDriver to make a new segment. if (addResult.getNumRowsInSegment() > tuningConfig.getMaxRowsPerSegment()) { if (!sequenceToUse.isCheckpointed()) { sequenceToCheckpoint = sequenceToUse; @@ -747,7 +748,6 @@ public void onFailure(Throwable t) } } ); - } } catch (ParseException e) { @@ -850,7 +850,7 @@ public void onFailure(Throwable t) Joiner.on(", ").join( handedOff.getSegments().stream().map(DataSegment::getIdentifier).collect(Collectors.toList()) ), - handedOff.getCommitMetadata() + Preconditions.checkNotNull(handedOff.getCommitMetadata(), "commitMetadata") ); } } @@ -940,9 +940,9 @@ private TaskStatus runLegacy(final TaskToolbox toolbox) throws Exception ); try ( - final Appenderator appenderator0 = newAppenderator(fireDepartmentMetrics, toolbox); - final AppenderatorDriver driver = newDriver(appenderator0, toolbox, fireDepartmentMetrics); - final KafkaConsumer consumer = newConsumer() + final Appenderator appenderator0 = newAppenderator(fireDepartmentMetrics, toolbox); + final StreamAppenderatorDriver driver = newDriver(appenderator0, toolbox, fireDepartmentMetrics); + final KafkaConsumer consumer = newConsumer() ) { toolbox.getDataSegmentServerAnnouncer().announce(); toolbox.getDruidNodeAnnouncer().announce(discoveryDruidNode); @@ -1104,7 +1104,7 @@ public void run() if (addResult.isOk()) { // If the number of rows in the segment exceeds the threshold after adding a row, - // move the segment out from the active segments of AppenderatorDriver to make a new segment. + // move the segment out from the active segments of BaseAppenderatorDriver to make a new segment. if (addResult.getNumRowsInSegment() > tuningConfig.getMaxRowsPerSegment()) { segmentsToMoveOut.computeIfAbsent(sequenceName, k -> new HashSet<>()) .add(addResult.getSegmentIdentifier()); @@ -1170,7 +1170,7 @@ public void run() final TransactionalSegmentPublisher publisher = (segments, commitMetadata) -> { final KafkaPartitions finalPartitions = toolbox.getObjectMapper().convertValue( - ((Map) commitMetadata).get(METADATA_NEXT_PARTITIONS), + ((Map) Preconditions.checkNotNull(commitMetadata, "commitMetadata")).get(METADATA_NEXT_PARTITIONS), KafkaPartitions.class ); @@ -1230,7 +1230,7 @@ public String apply(DataSegment input) } ) ), - handedOff.getCommitMetadata() + Preconditions.checkNotNull(handedOff.getCommitMetadata(), "commitMetadata") ); } } @@ -1269,7 +1269,7 @@ private void checkAndMaybeThrowException() } private void maybePersistAndPublishSequences(Supplier committerSupplier) - throws ExecutionException, InterruptedException + throws InterruptedException { for (SequenceMetadata sequenceMetadata : sequences) { sequenceMetadata.updateAssignments(nextOffsets); @@ -1793,13 +1793,13 @@ private Appenderator newAppenderator(FireDepartmentMetrics metrics, TaskToolbox ); } - private AppenderatorDriver newDriver( + private StreamAppenderatorDriver newDriver( final Appenderator appenderator, final TaskToolbox toolbox, final FireDepartmentMetrics metrics ) { - return new AppenderatorDriver( + return new StreamAppenderatorDriver( appenderator, new ActionBasedSegmentAllocator(toolbox.getTaskActionClient(), dataSchema), toolbox.getSegmentHandoffNotifierFactory(), @@ -2222,7 +2222,7 @@ public TransactionalSegmentPublisher getPublisher(TaskToolbox toolbox, boolean u { return (segments, commitMetadata) -> { final KafkaPartitions finalPartitions = toolbox.getObjectMapper().convertValue( - ((Map) commitMetadata).get(METADATA_PUBLISH_PARTITIONS), + ((Map) Preconditions.checkNotNull(commitMetadata, "commitMetadata")).get(METADATA_PUBLISH_PARTITIONS), KafkaPartitions.class ); diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java index 1a9bade78c9f..55296f05a8de 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java @@ -24,11 +24,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Function; import com.google.common.base.Joiner; import com.google.common.base.Optional; import com.google.common.base.Preconditions; -import com.google.common.base.Supplier; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -36,7 +34,6 @@ import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; import com.google.common.util.concurrent.ListenableFuture; -import io.druid.data.input.Committer; import io.druid.data.input.Firehose; import io.druid.data.input.FirehoseFactory; import io.druid.data.input.InputRow; @@ -57,7 +54,6 @@ import io.druid.java.util.common.guava.Comparators; import io.druid.java.util.common.logger.Logger; import io.druid.java.util.common.parsers.ParseException; -import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.query.DruidMetrics; import io.druid.segment.IndexSpec; import io.druid.segment.indexing.DataSchema; @@ -71,15 +67,15 @@ import io.druid.segment.realtime.RealtimeMetricsMonitor; import io.druid.segment.realtime.appenderator.Appenderator; import io.druid.segment.realtime.appenderator.AppenderatorConfig; -import io.druid.segment.realtime.appenderator.AppenderatorDriver; +import io.druid.segment.realtime.appenderator.BaseAppenderatorDriver; import io.druid.segment.realtime.appenderator.AppenderatorDriverAddResult; import io.druid.segment.realtime.appenderator.Appenderators; +import io.druid.segment.realtime.appenderator.BatchAppenderatorDriver; import io.druid.segment.realtime.appenderator.SegmentAllocator; import io.druid.segment.realtime.appenderator.SegmentIdentifier; import io.druid.segment.realtime.appenderator.SegmentsAndMetadata; import io.druid.segment.realtime.appenderator.TransactionalSegmentPublisher; -import io.druid.segment.realtime.plumber.Committers; -import io.druid.segment.realtime.plumber.NoopSegmentHandoffNotifierFactory; +import io.druid.segment.writeout.SegmentWriteOutMediumFactory; import io.druid.timeline.DataSegment; import io.druid.timeline.partition.HashBasedNumberedShardSpec; import io.druid.timeline.partition.NoneShardSpec; @@ -546,7 +542,7 @@ private static BiFunction getShardSpecCreateFunctio } /** - * This method reads input data row by row and adds the read row to a proper segment using {@link AppenderatorDriver}. + * This method reads input data row by row and adds the read row to a proper segment using {@link BaseAppenderatorDriver}. * If there is no segment for the row, a new one is created. Segments can be published in the middle of reading inputs * if one of below conditions are satisfied. * @@ -555,7 +551,7 @@ private static BiFunction getShardSpecCreateFunctio * If the number of rows in a segment exceeds {@link IndexTuningConfig#targetPartitionSize} * *
  • - * If the number of rows added to {@link AppenderatorDriver} so far exceeds {@link IndexTuningConfig#maxTotalRows} + * If the number of rows added to {@link BaseAppenderatorDriver} so far exceeds {@link IndexTuningConfig#maxTotalRows} *
    • * * @@ -590,11 +586,7 @@ dataSchema, new RealtimeIOConfig(null, null, null), null final IndexIOConfig ioConfig = ingestionSchema.getIOConfig(); final IndexTuningConfig tuningConfig = ingestionSchema.tuningConfig; - final long publishTimeout = tuningConfig.getPublishTimeout(); - final long maxRowsInAppenderator = tuningConfig.getMaxTotalRows(); - final int maxRowsInSegment = tuningConfig.getTargetPartitionSize() == null - ? Integer.MAX_VALUE - : tuningConfig.getTargetPartitionSize(); + final long pushTimeout = tuningConfig.getPushTimeout(); final boolean isGuaranteedRollup = isGuaranteedRollup(ioConfig, tuningConfig); final SegmentAllocator segmentAllocator; @@ -644,7 +636,12 @@ dataSchema, new RealtimeIOConfig(null, null, null), null } final int partitionNum = counters.computeIfAbsent(interval, x -> new AtomicInteger()).getAndIncrement(); - return new SegmentIdentifier(getDataSource(), interval, findVersion(versions, interval), new NumberedShardSpec(partitionNum, 0)); + return new SegmentIdentifier( + getDataSource(), + interval, + findVersion(versions, interval), + new NumberedShardSpec(partitionNum, 0) + ); }; } @@ -654,97 +651,73 @@ dataSchema, new RealtimeIOConfig(null, null, null), null }; try ( - final Appenderator appenderator = newAppenderator(fireDepartmentMetrics, toolbox, dataSchema, tuningConfig); - final AppenderatorDriver driver = newDriver( - appenderator, - toolbox, - segmentAllocator, - fireDepartmentMetrics - ); - final Firehose firehose = firehoseFactory.connect(dataSchema.getParser(), firehoseTempDir) + final Appenderator appenderator = newAppenderator(fireDepartmentMetrics, toolbox, dataSchema, tuningConfig); + final BatchAppenderatorDriver driver = newDriver(appenderator, toolbox, segmentAllocator); + final Firehose firehose = firehoseFactory.connect(dataSchema.getParser(), firehoseTempDir) ) { - final Supplier committerSupplier = Committers.supplierFromFirehose(firehose); + driver.startJob(); - if (driver.startJob() != null) { - driver.clear(); - } - - try { - while (firehose.hasMore()) { - try { - final InputRow inputRow = firehose.nextRow(); + while (firehose.hasMore()) { + try { + final InputRow inputRow = firehose.nextRow(); - if (inputRow == null) { - fireDepartmentMetrics.incrementThrownAway(); - continue; - } + if (inputRow == null) { + fireDepartmentMetrics.incrementThrownAway(); + continue; + } - final Optional optInterval = granularitySpec.bucketInterval(inputRow.getTimestamp()); - if (!optInterval.isPresent()) { - fireDepartmentMetrics.incrementThrownAway(); - continue; - } + final Optional optInterval = granularitySpec.bucketInterval(inputRow.getTimestamp()); + if (!optInterval.isPresent()) { + fireDepartmentMetrics.incrementThrownAway(); + continue; + } - final String sequenceName; - - if (isGuaranteedRollup) { - // Sequence name is based solely on the shardSpec, and there will only be one segment per sequence. - final Interval interval = optInterval.get(); - final ShardSpec shardSpec = shardSpecs.getShardSpec(interval, inputRow); - sequenceName = Appenderators.getSequenceName(interval, findVersion(versions, interval), shardSpec); - } else { - // Segments are created as needed, using a single sequence name. They may be allocated from the overlord - // (in append mode) or may be created on our own authority (in overwrite mode). - sequenceName = getId(); - } - final AppenderatorDriverAddResult addResult = driver.add(inputRow, sequenceName, committerSupplier); - - if (addResult.isOk()) { - // incremental segment publishment is allowed only when rollup don't have to be perfect. - if (!isGuaranteedRollup && - (addResult.getNumRowsInSegment() >= maxRowsInSegment || - addResult.getTotalNumRowsInAppenderator() >= maxRowsInAppenderator)) { - // There can be some segments waiting for being published even though any rows won't be added to them. - // If those segments are not published here, the available space in appenderator will be kept to be small - // which makes the size of segments smaller. - final SegmentsAndMetadata published = awaitPublish( - driver.publishAll( - publisher, - committerSupplier.get() - ), - publishTimeout - ); - // Even though IndexTask uses NoopHandoffNotifier which does nothing for segment handoff, - // the below code is needed to update the total number of rows added to the appenderator so far. - // See AppenderatorDriver.registerHandoff() and Appenderator.drop(). - // A hard-coded timeout is used here because the below get() is expected to return immediately. - driver.registerHandoff(published).get(30, TimeUnit.SECONDS); - } - } else { - throw new ISE("Failed to add a row with timestamp[%s]", inputRow.getTimestamp()); - } + final String sequenceName; - fireDepartmentMetrics.incrementProcessed(); + if (isGuaranteedRollup) { + // Sequence name is based solely on the shardSpec, and there will only be one segment per sequence. + final Interval interval = optInterval.get(); + final ShardSpec shardSpec = shardSpecs.getShardSpec(interval, inputRow); + sequenceName = Appenderators.getSequenceName(interval, findVersion(versions, interval), shardSpec); + } else { + // Segments are created as needed, using a single sequence name. They may be allocated from the overlord + // (in append mode) or may be created on our own authority (in overwrite mode). + sequenceName = getId(); } - catch (ParseException e) { - if (tuningConfig.isReportParseExceptions()) { - throw e; - } else { - fireDepartmentMetrics.incrementUnparseable(); + final AppenderatorDriverAddResult addResult = driver.add(inputRow, sequenceName); + + if (addResult.isOk()) { + // incremental segment publishment is allowed only when rollup don't have to be perfect. + if (!isGuaranteedRollup && + (exceedMaxRowsInSegment(addResult.getNumRowsInSegment(), tuningConfig) || + exceedMaxRowsInAppenderator(addResult.getTotalNumRowsInAppenderator(), tuningConfig))) { + // There can be some segments waiting for being published even though any rows won't be added to them. + // If those segments are not published here, the available space in appenderator will be kept to be small + // which makes the size of segments smaller. + final SegmentsAndMetadata pushed = driver.pushAllAndClear(pushTimeout); + log.info("Pushed segments[%s]", pushed.getSegments()); } + } else { + throw new ISE("Failed to add a row with timestamp[%s]", inputRow.getTimestamp()); + } + + fireDepartmentMetrics.incrementProcessed(); + } + catch (ParseException e) { + if (tuningConfig.isReportParseExceptions()) { + throw e; + } else { + fireDepartmentMetrics.incrementUnparseable(); } } } - finally { - driver.persist(committerSupplier.get()); - } + + final SegmentsAndMetadata pushed = driver.pushAllAndClear(pushTimeout); + log.info("Pushed segments[%s]", pushed.getSegments()); final SegmentsAndMetadata published = awaitPublish( - driver.publishAll( - publisher, - committerSupplier.get() - ), - publishTimeout + driver.publishAll(publisher), + pushTimeout ); if (published == null) { @@ -755,14 +728,7 @@ dataSchema, new RealtimeIOConfig(null, null, null), null "Published segments[%s]", Joiner.on(", ").join( Iterables.transform( published.getSegments(), - new Function() - { - @Override - public String apply(DataSegment input) - { - return input.getIdentifier(); - } - } + DataSegment::getIdentifier ) ) ); @@ -774,11 +740,24 @@ public String apply(DataSegment input) } } + private static boolean exceedMaxRowsInSegment(int numRowsInSegment, IndexTuningConfig indexTuningConfig) + { + // maxRowsInSegment should be null if numShards is set in indexTuningConfig + final Integer maxRowsInSegment = indexTuningConfig.getTargetPartitionSize(); + return maxRowsInSegment != null && maxRowsInSegment <= numRowsInSegment; + } + + private static boolean exceedMaxRowsInAppenderator(long numRowsInAppenderator, IndexTuningConfig indexTuningConfig) + { + // maxRowsInAppenderator should be null if numShards is set in indexTuningConfig + final Long maxRowsInAppenderator = indexTuningConfig.getMaxTotalRows(); + return maxRowsInAppenderator != null && maxRowsInAppenderator <= numRowsInAppenderator; + } + private static SegmentsAndMetadata awaitPublish( ListenableFuture publishFuture, long publishTimeout - ) - throws ExecutionException, InterruptedException, TimeoutException + ) throws ExecutionException, InterruptedException, TimeoutException { if (publishTimeout == 0) { return publishFuture.get(); @@ -805,20 +784,16 @@ private static Appenderator newAppenderator( ); } - private static AppenderatorDriver newDriver( + private static BatchAppenderatorDriver newDriver( final Appenderator appenderator, final TaskToolbox toolbox, - final SegmentAllocator segmentAllocator, - final FireDepartmentMetrics metrics + final SegmentAllocator segmentAllocator ) { - return new AppenderatorDriver( + return new BatchAppenderatorDriver( appenderator, segmentAllocator, - new NoopSegmentHandoffNotifierFactory(), // don't wait for handoff since we don't serve queries - new ActionBasedUsedSegmentChecker(toolbox.getTaskActionClient()), - toolbox.getObjectMapper(), - metrics + new ActionBasedUsedSegmentChecker(toolbox.getTaskActionClient()) ); } @@ -950,13 +925,13 @@ public static class IndexTuningConfig implements TuningConfig, AppenderatorConfi private static final boolean DEFAULT_FORCE_EXTENDABLE_SHARD_SPECS = false; private static final boolean DEFAULT_GUARANTEE_ROLLUP = false; private static final boolean DEFAULT_REPORT_PARSE_EXCEPTIONS = false; - private static final long DEFAULT_PUBLISH_TIMEOUT = 0; + private static final long DEFAULT_PUSH_TIMEOUT = 0; static final int DEFAULT_TARGET_PARTITION_SIZE = 5000000; private final Integer targetPartitionSize; private final int maxRowsInMemory; - private final int maxTotalRows; + private final Long maxTotalRows; private final Integer numShards; private final IndexSpec indexSpec; private final File basePersistDirectory; @@ -964,7 +939,7 @@ public static class IndexTuningConfig implements TuningConfig, AppenderatorConfi private final boolean forceExtendableShardSpecs; private final boolean forceGuaranteedRollup; private final boolean reportParseExceptions; - private final long publishTimeout; + private final long pushTimeout; @Nullable private final SegmentWriteOutMediumFactory segmentWriteOutMediumFactory; @@ -972,7 +947,7 @@ public static class IndexTuningConfig implements TuningConfig, AppenderatorConfi public IndexTuningConfig( @JsonProperty("targetPartitionSize") @Nullable Integer targetPartitionSize, @JsonProperty("maxRowsInMemory") @Nullable Integer maxRowsInMemory, - @JsonProperty("maxTotalRows") @Nullable Integer maxTotalRows, + @JsonProperty("maxTotalRows") @Nullable Long maxTotalRows, @JsonProperty("rowFlushBoundary") @Nullable Integer rowFlushBoundary_forBackCompatibility, // DEPRECATED @JsonProperty("numShards") @Nullable Integer numShards, @JsonProperty("indexSpec") @Nullable IndexSpec indexSpec, @@ -982,7 +957,8 @@ public IndexTuningConfig( @JsonProperty("forceExtendableShardSpecs") @Nullable Boolean forceExtendableShardSpecs, @JsonProperty("forceGuaranteedRollup") @Nullable Boolean forceGuaranteedRollup, @JsonProperty("reportParseExceptions") @Nullable Boolean reportParseExceptions, - @JsonProperty("publishTimeout") @Nullable Long publishTimeout, + @JsonProperty("publishTimeout") @Nullable Long publishTimeout, // deprecated + @JsonProperty("pushTimeout") @Nullable Long pushTimeout, @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) { @@ -996,7 +972,7 @@ public IndexTuningConfig( forceExtendableShardSpecs, forceGuaranteedRollup, reportParseExceptions, - publishTimeout, + pushTimeout != null ? pushTimeout : publishTimeout, null, segmentWriteOutMediumFactory ); @@ -1010,14 +986,14 @@ private IndexTuningConfig() private IndexTuningConfig( @Nullable Integer targetPartitionSize, @Nullable Integer maxRowsInMemory, - @Nullable Integer maxTotalRows, + @Nullable Long maxTotalRows, @Nullable Integer numShards, @Nullable IndexSpec indexSpec, @Nullable Integer maxPendingPersists, @Nullable Boolean forceExtendableShardSpecs, @Nullable Boolean forceGuaranteedRollup, @Nullable Boolean reportParseExceptions, - @Nullable Long publishTimeout, + @Nullable Long pushTimeout, @Nullable File basePersistDirectory, @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory ) @@ -1027,15 +1003,9 @@ private IndexTuningConfig( "targetPartitionSize and numShards cannot both be set" ); - this.targetPartitionSize = numShards != null && !numShards.equals(-1) - ? null - : (targetPartitionSize == null || targetPartitionSize.equals(-1) - ? DEFAULT_TARGET_PARTITION_SIZE - : targetPartitionSize); + this.targetPartitionSize = initializeTargetPartitionSize(numShards, targetPartitionSize); this.maxRowsInMemory = maxRowsInMemory == null ? DEFAULT_MAX_ROWS_IN_MEMORY : maxRowsInMemory; - this.maxTotalRows = maxTotalRows == null - ? DEFAULT_MAX_TOTAL_ROWS - : maxTotalRows; + this.maxTotalRows = initializeMaxTotalRows(numShards, maxTotalRows); this.numShards = numShards == null || numShards.equals(-1) ? null : numShards; this.indexSpec = indexSpec == null ? DEFAULT_INDEX_SPEC : indexSpec; this.maxPendingPersists = maxPendingPersists == null ? DEFAULT_MAX_PENDING_PERSISTS : maxPendingPersists; @@ -1046,7 +1016,7 @@ private IndexTuningConfig( this.reportParseExceptions = reportParseExceptions == null ? DEFAULT_REPORT_PARSE_EXCEPTIONS : reportParseExceptions; - this.publishTimeout = publishTimeout == null ? DEFAULT_PUBLISH_TIMEOUT : publishTimeout; + this.pushTimeout = pushTimeout == null ? DEFAULT_PUSH_TIMEOUT : pushTimeout; this.basePersistDirectory = basePersistDirectory; Preconditions.checkArgument( @@ -1057,6 +1027,26 @@ private IndexTuningConfig( this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory; } + private static Integer initializeTargetPartitionSize(Integer numShards, Integer targetPartitionSize) + { + if (numShards == null || numShards == -1) { + return targetPartitionSize == null || targetPartitionSize.equals(-1) + ? DEFAULT_TARGET_PARTITION_SIZE + : targetPartitionSize; + } else { + return null; + } + } + + private static Long initializeMaxTotalRows(Integer numShards, Long maxTotalRows) + { + if (numShards == null || numShards == -1) { + return maxTotalRows == null ? DEFAULT_MAX_TOTAL_ROWS : maxTotalRows; + } else { + return null; + } + } + public IndexTuningConfig withBasePersistDirectory(File dir) { return new IndexTuningConfig( @@ -1069,7 +1059,7 @@ public IndexTuningConfig withBasePersistDirectory(File dir) forceExtendableShardSpecs, forceGuaranteedRollup, reportParseExceptions, - publishTimeout, + pushTimeout, dir, segmentWriteOutMediumFactory ); @@ -1089,7 +1079,7 @@ public int getMaxRowsInMemory() } @JsonProperty - public int getMaxTotalRows() + public Long getMaxTotalRows() { return maxTotalRows; } @@ -1150,9 +1140,9 @@ public boolean isReportParseExceptions() } @JsonProperty - public long getPublishTimeout() + public long getPushTimeout() { - return publishTimeout; + return pushTimeout; } @Override @@ -1180,12 +1170,12 @@ public boolean equals(Object o) } IndexTuningConfig that = (IndexTuningConfig) o; return maxRowsInMemory == that.maxRowsInMemory && - maxTotalRows == that.maxTotalRows && + Objects.equals(maxTotalRows, that.maxTotalRows) && maxPendingPersists == that.maxPendingPersists && forceExtendableShardSpecs == that.forceExtendableShardSpecs && forceGuaranteedRollup == that.forceGuaranteedRollup && reportParseExceptions == that.reportParseExceptions && - publishTimeout == that.publishTimeout && + pushTimeout == that.pushTimeout && Objects.equals(targetPartitionSize, that.targetPartitionSize) && Objects.equals(numShards, that.numShards) && Objects.equals(indexSpec, that.indexSpec) && @@ -1207,7 +1197,7 @@ public int hashCode() forceExtendableShardSpecs, forceGuaranteedRollup, reportParseExceptions, - publishTimeout, + pushTimeout, segmentWriteOutMediumFactory ); } diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java index 7bccdbab3d83..017158200806 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java @@ -227,7 +227,7 @@ private static IndexTuningConfig createTuningConfig() return new IndexTuningConfig( 5000000, 500000, - 1000000, + 1000000L, null, null, new IndexSpec( @@ -241,6 +241,7 @@ private static IndexTuningConfig createTuningConfig() false, true, false, + null, 100L, null ); diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java index ece5c55c7eb1..d0cf4e39d2cf 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java @@ -57,14 +57,14 @@ import io.druid.segment.IndexMergerV9; import io.druid.segment.IndexSpec; import io.druid.segment.indexing.DataSchema; -import io.druid.segment.transform.ExpressionTransform; -import io.druid.segment.transform.TransformSpec; import io.druid.segment.indexing.granularity.ArbitraryGranularitySpec; import io.druid.segment.indexing.granularity.GranularitySpec; import io.druid.segment.indexing.granularity.UniformGranularitySpec; import io.druid.segment.loading.DataSegmentPusher; import io.druid.segment.realtime.appenderator.SegmentIdentifier; import io.druid.segment.realtime.firehose.LocalFirehoseFactory; +import io.druid.segment.transform.ExpressionTransform; +import io.druid.segment.transform.TransformSpec; import io.druid.timeline.DataSegment; import io.druid.timeline.partition.HashBasedNumberedShardSpec; import io.druid.timeline.partition.NoneShardSpec; @@ -581,7 +581,7 @@ public void testWithSmallMaxTotalRows() throws Exception Granularities.MINUTE, null ), - createTuningConfig(2, 2, 2, null, false, false, true), + createTuningConfig(2, 2, 2L, null, false, false, true), false ), null @@ -623,7 +623,7 @@ public void testPerfectRollup() throws Exception true, null ), - createTuningConfig(3, 2, 2, null, false, true, true), + createTuningConfig(3, 2, 2L, null, false, true, true), false ), null @@ -664,7 +664,7 @@ public void testBestEffortRollup() throws Exception true, null ), - createTuningConfig(3, 2, 2, null, false, false, true), + createTuningConfig(3, 2, 2L, null, false, false, true), false ), null @@ -1128,7 +1128,7 @@ private static IndexTuningConfig createTuningConfig( private static IndexTuningConfig createTuningConfig( Integer targetPartitionSize, Integer maxRowsInMemory, - Integer maxTotalRows, + Long maxTotalRows, Integer numShards, boolean forceExtendableShardSpecs, boolean forceGuaranteedRollup, @@ -1148,6 +1148,7 @@ private static IndexTuningConfig createTuningConfig( forceGuaranteedRollup, reportParseException, null, + null, null ); } diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java index b0484c882239..dd94f9508de1 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java @@ -190,7 +190,7 @@ public void testIndexTaskSerde() throws Exception jsonMapper ), new IndexIOConfig(new LocalFirehoseFactory(new File("lol"), "rofl", null), true), - new IndexTuningConfig(10000, 10, null, 9999, null, indexSpec, 3, true, true, false, null, null, null) + new IndexTuningConfig(10000, 10, null, 9999, null, indexSpec, 3, true, true, false, null, null, null, null) ), null ); @@ -253,7 +253,7 @@ public void testIndexTaskwithResourceSerde() throws Exception jsonMapper ), new IndexIOConfig(new LocalFirehoseFactory(new File("lol"), "rofl", null), true), - new IndexTuningConfig(10000, 10, null, null, null, indexSpec, 3, true, true, false, null, null, null) + new IndexTuningConfig(10000, 10, null, null, null, indexSpec, 3, true, true, false, null, null, null, null) ), null ); diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java index 08d72eab3553..6ce005867133 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java @@ -671,7 +671,7 @@ public void testIndexTask() throws Exception mapper ), new IndexIOConfig(new MockFirehoseFactory(false), false), - new IndexTuningConfig(10000, 10, null, null, null, indexSpec, 3, true, true, false, null, null, null) + new IndexTuningConfig(10000, 10, null, null, null, indexSpec, 3, true, true, false, null, null, null, null) ), null ); @@ -729,7 +729,7 @@ public void testIndexTaskFailure() throws Exception mapper ), new IndexIOConfig(new MockExceptionalFirehoseFactory(), false), - new IndexTuningConfig(10000, 10, null, null, null, indexSpec, 3, true, true, false, null, null, null) + new IndexTuningConfig(10000, 10, null, null, null, indexSpec, 3, true, true, false, null, null, null, null) ), null ); @@ -1094,7 +1094,7 @@ public void testResumeTasks() throws Exception mapper ), new IndexIOConfig(new MockFirehoseFactory(false), false), - new IndexTuningConfig(10000, 10, null, null, null, indexSpec, null, false, null, null, null, null, null) + new IndexTuningConfig(10000, 10, null, null, null, indexSpec, null, false, null, null, null, null, null, null) ), null ); diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderator.java b/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderator.java index 564ed4b9181d..dbd1ed831378 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderator.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderator.java @@ -26,6 +26,7 @@ import io.druid.query.QuerySegmentWalker; import io.druid.segment.incremental.IndexSizeExceededException; +import javax.annotation.Nullable; import java.io.Closeable; import java.util.Collection; import java.util.List; @@ -35,9 +36,9 @@ * both of those. It can also push data to deep storage. But, it does not decide which segments data should go into. * It also doesn't publish segments to the metadata store or monitor handoff; you have to do that yourself! *

    - * Any time you call one of the methods that adds, persists, or pushes data, you must provide a Committer, or a - * Supplier of one, that represents all data you have given to the Appenderator so far. The Committer will be used when - * that data has been persisted to disk. + * You can provide a {@link Committer} or a Supplier of one when you call one of the methods that adds, persists, or + * pushes data. The Committer should represent all data you have given to the Appenderator so far. This Committer will + * be used when that data has been persisted to disk. */ public interface Appenderator extends QuerySegmentWalker, Closeable { @@ -72,8 +73,8 @@ default AppenderatorAddResult add(SegmentIdentifier identifier, InputRow row, Su * Committer is guaranteed to be *created* synchronously with the call to add, but will actually be used * asynchronously. *

    - * The add, clear, persist, persistAll, and push methods should all be called from the same thread to keep the - * metadata committed by Committer in sync. + * If committer is not provided, no metadata is persisted. If it's provided, the add, clear, persist, persistAll, + * and push methods should all be called from the same thread to keep the metadata committed by Committer in sync. * * @param identifier the segment into which this row should be added * @param row the row to add @@ -94,7 +95,7 @@ default AppenderatorAddResult add(SegmentIdentifier identifier, InputRow row, Su AppenderatorAddResult add( SegmentIdentifier identifier, InputRow row, - Supplier committerSupplier, + @Nullable Supplier committerSupplier, boolean allowIncrementalPersists ) throws IndexSizeExceededException, SegmentNotWritableException; @@ -152,8 +153,8 @@ AppenderatorAddResult add( * persist, but will actually be used asynchronously. Any metadata returned by the committer will be associated with * the data persisted to disk. *

    - * The add, clear, persist, persistAll, and push methods should all be called from the same thread to keep the - * metadata committed by Committer in sync. + * If committer is not provided, no metadata is persisted. If it's provided, the add, clear, persist, persistAll, + * and push methods should all be called from the same thread to keep the metadata committed by Committer in sync. * * @param identifiers segment identifiers to be persisted * @param committer a committer associated with all data that has been added to segments of the given identifiers so @@ -162,7 +163,7 @@ AppenderatorAddResult add( * @return future that resolves when all pending data to segments of the identifiers has been persisted, contains * commit metadata for this persist */ - ListenableFuture persist(Collection identifiers, Committer committer); + ListenableFuture persist(Collection identifiers, @Nullable Committer committer); /** * Persist any in-memory indexed data to durable storage. This may be only somewhat durable, e.g. the @@ -170,14 +171,14 @@ AppenderatorAddResult add( * be used asynchronously. Any metadata returned by the committer will be associated with the data persisted to * disk. *

    - * The add, clear, persist, persistAll, and push methods should all be called from the same thread to keep the - * metadata committed by Committer in sync. + * If committer is not provided, no metadata is persisted. If it's provided, the add, clear, persist, persistAll, + * and push methods should all be called from the same thread to keep the metadata committed by Committer in sync. * * @param committer a committer associated with all data that has been added so far * * @return future that resolves when all pending data has been persisted, contains commit metadata for this persist */ - default ListenableFuture persistAll(Committer committer) + default ListenableFuture persistAll(@Nullable Committer committer) { return persist(getSegments(), committer); } @@ -188,8 +189,8 @@ default ListenableFuture persistAll(Committer committer) *

    * After this method is called, you cannot add new data to any segments that were previously under construction. *

    - * The add, clear, persist, persistAll, and push methods should all be called from the same thread to keep the - * metadata committed by Committer in sync. + * If committer is not provided, no metadata is persisted. If it's provided, the add, clear, persist, persistAll, + * and push methods should all be called from the same thread to keep the metadata committed by Committer in sync. * * @param identifiers list of segments to push * @param committer a committer associated with all data that has been added so far @@ -197,7 +198,7 @@ default ListenableFuture persistAll(Committer committer) * @return future that resolves when all segments have been pushed. The segment list will be the list of segments * that have been pushed and the commit metadata from the Committer. */ - ListenableFuture push(Collection identifiers, Committer committer); + ListenableFuture push(Collection identifiers, @Nullable Committer committer); /** * Stop any currently-running processing and clean up after ourselves. This allows currently running persists and pushes diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriver.java b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriver.java deleted file mode 100644 index 82dc9ef6759a..000000000000 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriver.java +++ /dev/null @@ -1,776 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.segment.realtime.appenderator; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Function; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.base.Supplier; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.SettableFuture; -import io.druid.data.input.Committer; -import io.druid.data.input.InputRow; -import io.druid.java.util.common.ISE; -import io.druid.java.util.common.concurrent.Execs; -import io.druid.java.util.common.logger.Logger; -import io.druid.query.SegmentDescriptor; -import io.druid.segment.realtime.FireDepartmentMetrics; -import io.druid.segment.realtime.plumber.SegmentHandoffNotifier; -import io.druid.segment.realtime.plumber.SegmentHandoffNotifierFactory; -import org.joda.time.DateTime; - -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.NavigableMap; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -/** - * A AppenderatorDriver drives an Appenderator to index a finite stream of data. This class does not help you - * index unbounded streams. All handoff is done at the end of indexing. - *

    - * This class helps with doing things that Appenderators don't, including deciding which segments to use (with a - * SegmentAllocator), publishing segments to the metadata store (with a SegmentPublisher), and monitoring handoff (with - * a SegmentHandoffNotifier). - *

    - * Note that the commit metadata stored by this class via the underlying Appenderator is not the same metadata as - * you pass in. It's wrapped in some extra metadata needed by the driver. - */ -public class AppenderatorDriver implements Closeable -{ - private static final Logger log = new Logger(AppenderatorDriver.class); - - private final Appenderator appenderator; - private final SegmentAllocator segmentAllocator; - private final SegmentHandoffNotifier handoffNotifier; - private final UsedSegmentChecker usedSegmentChecker; - private final ObjectMapper objectMapper; - private final FireDepartmentMetrics metrics; - - enum SegmentState - { - ACTIVE, - INACTIVE, - PUBLISHING - } - - static class SegmentWithState - { - private SegmentIdentifier segmentIdentifier; - private SegmentState state; - - @JsonCreator - SegmentWithState( - @JsonProperty("segmentIdentifier") SegmentIdentifier segmentIdentifier, - @JsonProperty("state") SegmentState state - ) - { - this.segmentIdentifier = segmentIdentifier; - this.state = state; - } - - @JsonProperty - public SegmentIdentifier getSegmentIdentifier() - { - return segmentIdentifier; - } - - @JsonProperty - public SegmentState getState() - { - return state; - } - - @Override - public String toString() - { - return "SegmentWithState{" + - "segmentIdentifier=" + segmentIdentifier + - ", state=" + state + - '}'; - } - } - - // sequenceName -> {Interval Start millis -> List of Segments for this interval} - // there might be multiple segments for a start interval, for example one segment - // can be in ACTIVE state and others might be in PUBLISHING state - private final Map>> segments = new TreeMap<>(); - - private final Set publishingSequences = new HashSet<>(); - - // sequenceName -> most recently allocated segment - private final Map lastSegmentIds = Maps.newHashMap(); - - private final ListeningExecutorService publishExecutor; - - /** - * Create a driver. - * - * @param appenderator appenderator - * @param segmentAllocator segment allocator - * @param handoffNotifierFactory handoff notifier factory - * @param usedSegmentChecker used segment checker - * @param objectMapper object mapper, used for serde of commit metadata - * @param metrics Firedepartment metrics - */ - public AppenderatorDriver( - Appenderator appenderator, - SegmentAllocator segmentAllocator, - SegmentHandoffNotifierFactory handoffNotifierFactory, - UsedSegmentChecker usedSegmentChecker, - ObjectMapper objectMapper, - FireDepartmentMetrics metrics - ) - { - this.appenderator = Preconditions.checkNotNull(appenderator, "appenderator"); - this.segmentAllocator = Preconditions.checkNotNull(segmentAllocator, "segmentAllocator"); - this.handoffNotifier = Preconditions.checkNotNull(handoffNotifierFactory, "handoffNotifierFactory") - .createSegmentHandoffNotifier(appenderator.getDataSource()); - this.usedSegmentChecker = Preconditions.checkNotNull(usedSegmentChecker, "usedSegmentChecker"); - this.objectMapper = Preconditions.checkNotNull(objectMapper, "objectMapper"); - this.metrics = Preconditions.checkNotNull(metrics, "metrics"); - this.publishExecutor = MoreExecutors.listeningDecorator(Execs.singleThreaded("publish-%d")); - } - - @VisibleForTesting - Map>> getSegments() - { - return segments; - } - - /** - * Perform any initial setup and return currently persisted commit metadata. - *

    - * Note that this method returns the same metadata you've passed in with your Committers, even though this class - * stores extra metadata on disk. - * - * @return currently persisted commit metadata - */ - public Object startJob() - { - handoffNotifier.start(); - - final AppenderatorDriverMetadata metadata = objectMapper.convertValue( - appenderator.startJob(), - AppenderatorDriverMetadata.class - ); - - log.info("Restored metadata[%s].", metadata); - - if (metadata != null) { - synchronized (segments) { - for (Map.Entry> entry : metadata.getSegments().entrySet()) { - final String sequenceName = entry.getKey(); - final TreeMap> segmentMap = Maps.newTreeMap(); - - segments.put(sequenceName, segmentMap); - - for (SegmentWithState segmentWithState : entry.getValue()) { - segmentMap.computeIfAbsent( - segmentWithState.getSegmentIdentifier().getInterval().getStartMillis(), - k -> new LinkedList<>() - ); - LinkedList segmentList = segmentMap.get(segmentWithState.getSegmentIdentifier() - .getInterval() - .getStartMillis()); - // always keep the ACTIVE segment for an interval start millis in the front - if (segmentWithState.getState() == SegmentState.ACTIVE) { - segmentList.addFirst(segmentWithState); - } else { - segmentList.addLast(segmentWithState); - } - } - } - lastSegmentIds.putAll(metadata.getLastSegmentIds()); - } - - return metadata.getCallerMetadata(); - } else { - return null; - } - } - - private void addSegment(String sequenceName, SegmentIdentifier identifier) - { - synchronized (segments) { - segments.computeIfAbsent(sequenceName, k -> new TreeMap<>()) - .computeIfAbsent(identifier.getInterval().getStartMillis(), k -> new LinkedList<>()) - .addFirst(new SegmentWithState(identifier, SegmentState.ACTIVE)); - lastSegmentIds.put(sequenceName, identifier.getIdentifierAsString()); - } - } - - /** - * Clears out all our state and also calls {@link Appenderator#clear()} on the underlying Appenderator. - */ - public void clear() throws InterruptedException - { - synchronized (segments) { - segments.clear(); - } - appenderator.clear(); - } - - public AppenderatorDriverAddResult add( - final InputRow row, - final String sequenceName, - final Supplier committerSupplier - ) throws IOException - { - return add(row, sequenceName, committerSupplier, false, true); - } - - /** - * Add a row. Must not be called concurrently from multiple threads. - * - * @param row the row to add - * @param sequenceName sequenceName for this row's segment - * @param committerSupplier supplier of a committer associated with all data that has been added, including this row - * if {@param allowIncrementalPersists} is set to false then this will not be used - * @param skipSegmentLineageCheck if true, perform lineage validation using previousSegmentId for this sequence. - * Should be set to false if replica tasks would index events in same order - * @param allowIncrementalPersists whether to allow persist to happen when maxRowsInMemory or intermediate persist period - * threshold is hit - * - * @return {@link AppenderatorDriverAddResult} - * - * @throws IOException if there is an I/O error while allocating or writing to a segment - */ - - public AppenderatorDriverAddResult add( - final InputRow row, - final String sequenceName, - final Supplier committerSupplier, - final boolean skipSegmentLineageCheck, - final boolean allowIncrementalPersists - ) throws IOException - { - Preconditions.checkNotNull(row, "row"); - Preconditions.checkNotNull(sequenceName, "sequenceName"); - Preconditions.checkNotNull(committerSupplier, "committerSupplier"); - - final SegmentIdentifier identifier = getSegment(row, sequenceName, skipSegmentLineageCheck); - - if (identifier != null) { - try { - final Appenderator.AppenderatorAddResult result = appenderator.add( - identifier, - row, - wrapCommitterSupplier(committerSupplier), - allowIncrementalPersists - ); - return AppenderatorDriverAddResult.ok( - identifier, - result.getNumRowsInSegment(), - appenderator.getTotalRowCount(), - result.isPersistRequired() - ); - } - catch (SegmentNotWritableException e) { - throw new ISE(e, "WTF?! Segment[%s] not writable when it should have been.", identifier); - } - } else { - return AppenderatorDriverAddResult.fail(); - } - } - - /** - * Persist all data indexed through this driver so far. Blocks until complete. - *

    - * Should be called after all data has been added through {@link #add(InputRow, String, Supplier, boolean, boolean)}. - * - * @param committer committer representing all data that has been added so far - * - * @return commitMetadata persisted - */ - public Object persist(final Committer committer) throws InterruptedException - { - try { - log.info("Persisting data."); - final long start = System.currentTimeMillis(); - final Object commitMetadata = appenderator.persistAll(wrapCommitter(committer)).get(); - log.info("Persisted pending data in %,dms.", System.currentTimeMillis() - start); - return commitMetadata; - } - catch (InterruptedException e) { - throw e; - } - catch (Exception e) { - throw Throwables.propagate(e); - } - } - - /** - * Persist all data indexed through this driver so far. Returns a future of persisted commitMetadata. - *

    - * Should be called after all data has been added through {@link #add(InputRow, String, Supplier, boolean, boolean)}. - * - * @param committer committer representing all data that has been added so far - * - * @return future containing commitMetadata persisted - */ - public ListenableFuture persistAsync(final Committer committer) - throws InterruptedException, ExecutionException - { - log.info("Persisting data asynchronously"); - return appenderator.persistAll(wrapCommitter(committer)); - } - - /** - * Register the segments in the given {@link SegmentsAndMetadata} to be handed off and execute a background task which - * waits until the hand off completes. - * - * @param segmentsAndMetadata the result segments and metadata of - * {@link #publish(TransactionalSegmentPublisher, Committer, Collection)} - * - * @return null if the input segmentsAndMetadata is null. Otherwise, a {@link ListenableFuture} for the submitted task - * which returns {@link SegmentsAndMetadata} containing the segments successfully handed off and the metadata - * of the caller of {@link AppenderatorDriverMetadata} - */ - public ListenableFuture registerHandoff(SegmentsAndMetadata segmentsAndMetadata) - { - if (segmentsAndMetadata == null) { - return Futures.immediateFuture(null); - - } else { - final List waitingSegmentIdList = segmentsAndMetadata.getSegments().stream() - .map(SegmentIdentifier::fromDataSegment) - .collect(Collectors.toList()); - - if (waitingSegmentIdList.isEmpty()) { - return Futures.immediateFuture( - new SegmentsAndMetadata( - segmentsAndMetadata.getSegments(), - ((AppenderatorDriverMetadata) segmentsAndMetadata.getCommitMetadata()) - .getCallerMetadata() - ) - ); - } - - log.info("Register handoff of segments: [%s]", waitingSegmentIdList); - - final SettableFuture resultFuture = SettableFuture.create(); - final AtomicInteger numRemainingHandoffSegments = new AtomicInteger(waitingSegmentIdList.size()); - - for (final SegmentIdentifier segmentIdentifier : waitingSegmentIdList) { - handoffNotifier.registerSegmentHandoffCallback( - new SegmentDescriptor( - segmentIdentifier.getInterval(), - segmentIdentifier.getVersion(), - segmentIdentifier.getShardSpec().getPartitionNum() - ), - MoreExecutors.sameThreadExecutor(), - () -> { - log.info("Segment[%s] successfully handed off, dropping.", segmentIdentifier); - metrics.incrementHandOffCount(); - - final ListenableFuture dropFuture = appenderator.drop(segmentIdentifier); - Futures.addCallback( - dropFuture, - new FutureCallback() - { - @Override - public void onSuccess(Object result) - { - if (numRemainingHandoffSegments.decrementAndGet() == 0) { - log.info("Successfully handed off [%d] segments.", segmentsAndMetadata.getSegments().size()); - resultFuture.set( - new SegmentsAndMetadata( - segmentsAndMetadata.getSegments(), - ((AppenderatorDriverMetadata) segmentsAndMetadata.getCommitMetadata()) - .getCallerMetadata() - ) - ); - } - } - - @Override - public void onFailure(Throwable e) - { - log.warn(e, "Failed to drop segment[%s]?!", segmentIdentifier); - numRemainingHandoffSegments.decrementAndGet(); - resultFuture.setException(e); - } - } - ); - } - ); - } - - return resultFuture; - } - } - - /** - * Closes this driver. Does not close the underlying Appenderator; you should do that yourself. - */ - @Override - public void close() - { - publishExecutor.shutdownNow(); - handoffNotifier.close(); - } - - private SegmentIdentifier getActiveSegment(final DateTime timestamp, final String sequenceName) - { - synchronized (segments) { - final NavigableMap> segmentsForSequence = segments.get(sequenceName); - - if (segmentsForSequence == null) { - return null; - } - - final Map.Entry> candidateEntry = segmentsForSequence.floorEntry(timestamp.getMillis()); - if (candidateEntry != null - && candidateEntry.getValue().getFirst().getSegmentIdentifier().getInterval().contains(timestamp) - && candidateEntry.getValue().getFirst().getState().equals(SegmentState.ACTIVE)) { - return candidateEntry.getValue().getFirst().getSegmentIdentifier(); - } else { - return null; - } - } - } - - /** - * Return a segment usable for "timestamp". May return null if no segment can be allocated. - * - * @param row input row - * @param sequenceName sequenceName for potential segment allocation - * @param skipSegmentLineageCheck if false, perform lineage validation using previousSegmentId for this sequence. - * Should be set to false if replica tasks would index events in same order - * - * @return identifier, or null - * - * @throws IOException if an exception occurs while allocating a segment - */ - private SegmentIdentifier getSegment( - final InputRow row, - final String sequenceName, - final boolean skipSegmentLineageCheck - ) throws IOException - { - synchronized (segments) { - final DateTime timestamp = row.getTimestamp(); - final SegmentIdentifier existing = getActiveSegment(timestamp, sequenceName); - if (existing != null) { - return existing; - } else { - // Allocate new segment. - final SegmentIdentifier newSegment = segmentAllocator.allocate( - row, - sequenceName, - lastSegmentIds.get(sequenceName), - // send lastSegmentId irrespective of skipSegmentLineageCheck so that - // unique constraint for sequence_name_prev_id_sha1 does not fail for - // allocatePendingSegment in IndexerSQLMetadataStorageCoordinator - skipSegmentLineageCheck - ); - - if (newSegment != null) { - for (SegmentIdentifier identifier : appenderator.getSegments()) { - if (identifier.equals(newSegment)) { - throw new ISE( - "WTF?! Allocated segment[%s] which conflicts with existing segment[%s].", - newSegment, - identifier - ); - } - } - - log.info("New segment[%s] for sequenceName[%s].", newSegment, sequenceName); - addSegment(sequenceName, newSegment); - } else { - // Well, we tried. - log.warn("Cannot allocate segment for timestamp[%s], sequenceName[%s]. ", timestamp, sequenceName); - } - - return newSegment; - } - } - } - - /** - * Move a set of identifiers out from "active", making way for newer segments. - */ - public void moveSegmentOut(final String sequenceName, final List identifiers) - { - synchronized (segments) { - final NavigableMap> activeSegmentsForSequence = segments.get(sequenceName); - if (activeSegmentsForSequence == null) { - throw new ISE("WTF?! Asked to remove segments for sequenceName[%s] which doesn't exist...", sequenceName); - } - - for (final SegmentIdentifier identifier : identifiers) { - log.info("Moving segment[%s] out of active list.", identifier); - final long key = identifier.getInterval().getStartMillis(); - if (activeSegmentsForSequence.get(key) == null || activeSegmentsForSequence.get(key).stream().noneMatch( - segmentWithState -> { - if (segmentWithState.getSegmentIdentifier().equals(identifier)) { - segmentWithState.state = SegmentState.INACTIVE; - return true; - } else { - return false; - } - } - )) { - throw new ISE("WTF?! Asked to remove segment[%s] that didn't exist...", identifier); - } - } - } - } - - /** - * Publish all pending segments. - * - * @param publisher segment publisher - * @param committer committer - * - * @return a {@link ListenableFuture} for the publish task which removes published {@code sequenceNames} from - * {@code activeSegments} and {@code publishPendingSegments} - */ - public ListenableFuture publishAll( - final TransactionalSegmentPublisher publisher, - final Committer committer - ) - { - final List theSequences; - synchronized (segments) { - theSequences = ImmutableList.copyOf(segments.keySet()); - } - return publish(publisher, wrapCommitter(committer), theSequences); - } - - /** - * Execute a task in background to publish all segments corresponding to the given sequence names. The task - * internally pushes the segments to the deep storage first, and then publishes the metadata to the metadata storage. - * - * @param publisher segment publisher - * @param committer committer - * @param sequenceNames a collection of sequence names to be published - * - * @return a {@link ListenableFuture} for the submitted task which removes published {@code sequenceNames} from - * {@code activeSegments} and {@code publishPendingSegments} - */ - public ListenableFuture publish( - final TransactionalSegmentPublisher publisher, - final Committer committer, - final Collection sequenceNames - ) - { - final List theSegments = new ArrayList<>(); - synchronized (segments) { - sequenceNames.stream() - .filter(sequenceName -> !publishingSequences.contains(sequenceName)) - .forEach(sequenceName -> { - if (segments.containsKey(sequenceName)) { - publishingSequences.add(sequenceName); - segments.get(sequenceName) - .values() - .stream() - .flatMap(Collection::stream) - .forEach(segmentWithState -> { - segmentWithState.state = SegmentState.PUBLISHING; - theSegments.add(segmentWithState.getSegmentIdentifier()); - }); - } - }); - } - - final ListenableFuture publishFuture = publish( - publisher, - wrapCommitter(committer), - theSegments - ); - - Futures.addCallback( - publishFuture, - new FutureCallback() - { - @Override - public void onSuccess(SegmentsAndMetadata result) - { - if (result != null) { - publishingSequences.removeAll(sequenceNames); - sequenceNames.forEach(segments::remove); - } - } - - @Override - public void onFailure(Throwable t) - { - // Do nothing, caller should handle the exception - log.error("Error publishing sequences [%s]", sequenceNames); - } - } - ); - - return publishFuture; - } - - /** - * Execute a task in background to publish the given segments. The task blocks until complete. - * Retries forever on transient failures, but may exit early on permanent failures. - *

    - * Should be called after all data has been added through {@link #add(InputRow, String, Supplier, boolean, boolean)}. - * - * @param publisher publisher to use for this set of segments - * @param wrappedCommitter committer representing all data that has been added so far - * - * @return segments and metadata published if successful, or null if segments could not be handed off due to - * transaction failure with commit metadata. - */ - private ListenableFuture publish( - final TransactionalSegmentPublisher publisher, - final WrappedCommitter wrappedCommitter, - final List segmentIdentifiers - ) - { - log.info("Pushing segments: [%s]", Joiner.on(", ").join(segmentIdentifiers)); - - return Futures.transform( - appenderator.push(segmentIdentifiers, wrappedCommitter), - (Function) segmentsAndMetadata -> { - // Sanity check - final Set pushedSegments = segmentsAndMetadata.getSegments().stream() - .map(SegmentIdentifier::fromDataSegment) - .collect(Collectors.toSet()); - if (!pushedSegments.equals(Sets.newHashSet(segmentIdentifiers))) { - throw new ISE( - "WTF?! Pushed different segments than requested. Pushed[%s], requested[%s].", - pushedSegments, - segmentIdentifiers - ); - } - - if (segmentsAndMetadata.getSegments().isEmpty()) { - log.info("Nothing to publish, skipping publish step."); - } else { - log.info( - "Publishing segments with commitMetadata[%s]: [%s]", - segmentsAndMetadata.getCommitMetadata(), - Joiner.on(", ").join(segmentsAndMetadata.getSegments()) - ); - - try { - final boolean published = publisher.publishSegments( - ImmutableSet.copyOf(segmentsAndMetadata.getSegments()), - ((AppenderatorDriverMetadata) segmentsAndMetadata.getCommitMetadata()).getCallerMetadata() - ); - - if (published) { - log.info("Published segments."); - } else { - log.info("Transaction failure while publishing segments, checking if someone else beat us to it."); - if (usedSegmentChecker.findUsedSegments(pushedSegments) - .equals(Sets.newHashSet(segmentsAndMetadata.getSegments()))) { - log.info("Our segments really do exist, awaiting handoff."); - } else { - throw new ISE("Failed to publish segments[%s]", segmentIdentifiers); - } - } - } - catch (IOException e) { - throw Throwables.propagate(e); - } - } - - return segmentsAndMetadata; - }, - publishExecutor - ); - } - - public ListenableFuture publishAndRegisterHandoff( - final TransactionalSegmentPublisher publisher, - final Committer committer, - final Collection sequenceNames - ) - { - return Futures.transform( - publish(publisher, committer, sequenceNames), - this::registerHandoff - ); - } - - private interface WrappedCommitter extends Committer - { - } - - private Supplier wrapCommitterSupplier(final Supplier committerSupplier) - { - return () -> wrapCommitter(committerSupplier.get()); - } - - private WrappedCommitter wrapCommitter(final Committer committer) - { - final AppenderatorDriverMetadata wrappedMetadata; - synchronized (segments) { - wrappedMetadata = new AppenderatorDriverMetadata( - ImmutableMap.copyOf( - Maps.transformValues( - segments, - (Function>, List>) input -> ImmutableList - .copyOf(input.values().stream().flatMap(Collection::stream).collect(Collectors.toList())) - ) - ), - ImmutableMap.copyOf(lastSegmentIds), - committer.getMetadata() - ); - } - - return new WrappedCommitter() - { - @Override - public Object getMetadata() - { - return wrappedMetadata; - } - - @Override - public void run() - { - committer.run(); - } - }; - } -} diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriverAddResult.java b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriverAddResult.java index 4838d58a227f..c5afa3a78205 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriverAddResult.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriverAddResult.java @@ -25,7 +25,7 @@ import javax.annotation.Nullable; /** - * Result of {@link AppenderatorDriver#add(InputRow, String, Supplier, boolean)}. It contains the identifier of the + * Result of {@link BaseAppenderatorDriver#add(InputRow, String, Supplier, boolean)}. It contains the identifier of the * segment which the InputRow is added to, the number of rows in that segment and if persist is required because either * maxRowsInMemory or intermediate persist period threshold is hit. */ diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriverMetadata.java b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriverMetadata.java index 0e8722c7d087..d1e002a66dab 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriverMetadata.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorDriverMetadata.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import io.druid.segment.realtime.appenderator.SegmentWithState.SegmentState; import java.util.ArrayList; import java.util.List; @@ -33,13 +34,13 @@ public class AppenderatorDriverMetadata { - private final Map> segments; + private final Map> segments; private final Map lastSegmentIds; private final Object callerMetadata; @JsonCreator public AppenderatorDriverMetadata( - @JsonProperty("segments") Map> segments, + @JsonProperty("segments") Map> segments, @JsonProperty("lastSegmentIds") Map lastSegmentIds, @JsonProperty("callerMetadata") Object callerMetadata, // Next two properties are for backwards compatibility, should be removed on versions greater than 0.12.x @@ -57,7 +58,7 @@ public AppenderatorDriverMetadata( ); if (segments == null) { // convert old metadata to new one - final Map> newMetadata = Maps.newHashMap(); + final Map> newMetadata = Maps.newHashMap(); final Set activeSegmentsAlreadySeen = Sets.newHashSet(); // temp data structure activeSegments.entrySet() @@ -67,10 +68,7 @@ public AppenderatorDriverMetadata( .stream() .map(segmentIdentifier -> { activeSegmentsAlreadySeen.add(segmentIdentifier.toString()); - return new AppenderatorDriver.SegmentWithState( - segmentIdentifier, - AppenderatorDriver.SegmentState.ACTIVE - ); + return SegmentWithState.newSegment(segmentIdentifier); }) .collect(Collectors.toList()) )); @@ -84,9 +82,9 @@ public AppenderatorDriverMetadata( .stream() .filter(segmentIdentifier -> !activeSegmentsAlreadySeen.contains( segmentIdentifier.toString())) - .map(segmentIdentifier -> new AppenderatorDriver.SegmentWithState( + .map(segmentIdentifier -> SegmentWithState.newSegment( segmentIdentifier, - AppenderatorDriver.SegmentState.INACTIVE + SegmentState.APPEND_FINISHED )) .collect(Collectors.toList()) )); @@ -99,7 +97,7 @@ public AppenderatorDriverMetadata( } public AppenderatorDriverMetadata( - Map> segments, + Map> segments, Map lastSegmentIds, Object callerMetadata ) @@ -108,7 +106,7 @@ public AppenderatorDriverMetadata( } @JsonProperty - public Map> getSegments() + public Map> getSegments() { return segments; } diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java index deb7bd7fb332..2f4a851bc57e 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java @@ -202,7 +202,7 @@ public Object startJob() public AppenderatorAddResult add( final SegmentIdentifier identifier, final InputRow row, - final Supplier committerSupplier, + @Nullable final Supplier committerSupplier, final boolean allowIncrementalPersists ) throws IndexSizeExceededException, SegmentNotWritableException { @@ -244,7 +244,7 @@ public AppenderatorAddResult add( || rowsCurrentlyInMemory.get() >= tuningConfig.getMaxRowsInMemory()) { if (allowIncrementalPersists) { // persistAll clears rowsCurrentlyInMemory, no need to update it. - persistAll(committerSupplier.get()); + persistAll(committerSupplier == null ? null : committerSupplier.get()); } else { isPersistRequired = true; } @@ -340,35 +340,37 @@ public void clear() throws InterruptedException // Drop commit metadata, then abandon all segments. try { - final ListenableFuture uncommitFuture = persistExecutor.submit( - new Callable() - { - @Override - public Object call() throws Exception + if (persistExecutor != null) { + final ListenableFuture uncommitFuture = persistExecutor.submit( + new Callable() { - try { - commitLock.lock(); - objectMapper.writeValue(computeCommitFile(), Committed.nil()); - } - finally { - commitLock.unlock(); + @Override + public Object call() throws Exception + { + try { + commitLock.lock(); + objectMapper.writeValue(computeCommitFile(), Committed.nil()); + } + finally { + commitLock.unlock(); + } + return null; } - return null; } - } - ); + ); - // Await uncommit. - uncommitFuture.get(); + // Await uncommit. + uncommitFuture.get(); - // Drop everything. - final List> futures = Lists.newArrayList(); - for (Map.Entry entry : sinks.entrySet()) { - futures.add(abandonSegment(entry.getKey(), entry.getValue(), true)); - } + // Drop everything. + final List> futures = Lists.newArrayList(); + for (Map.Entry entry : sinks.entrySet()) { + futures.add(abandonSegment(entry.getKey(), entry.getValue(), true)); + } - // Await dropping. - Futures.allAsList(futures).get(); + // Await dropping. + Futures.allAsList(futures).get(); + } } catch (ExecutionException e) { throw Throwables.propagate(e); @@ -387,7 +389,7 @@ public ListenableFuture drop(final SegmentIdentifier identifier) } @Override - public ListenableFuture persist(Collection identifiers, Committer committer) + public ListenableFuture persist(Collection identifiers, @Nullable Committer committer) { final Map currentHydrants = Maps.newHashMap(); final List> indexesToPersist = Lists.newArrayList(); @@ -418,7 +420,7 @@ public ListenableFuture persist(Collection identifier log.info("Submitting persist runnable for dataSource[%s]", schema.getDataSource()); final String threadName = StringUtils.format("%s-incremental-persist", schema.getDataSource()); - final Object commitMetadata = committer.getMetadata(); + final Object commitMetadata = committer == null ? null : committer.getMetadata(); final Stopwatch runExecStopwatch = Stopwatch.createStarted(); final Stopwatch persistStopwatch = Stopwatch.createStarted(); final ListenableFuture future = persistExecutor.submit( @@ -432,37 +434,39 @@ public Object doCall() metrics.incrementRowOutputCount(persistHydrant(pair.lhs, pair.rhs)); } - log.info( - "Committing metadata[%s] for sinks[%s].", commitMetadata, Joiner.on(", ").join( - currentHydrants.entrySet() - .stream() - .map(entry -> StringUtils.format( - "%s:%d", - entry.getKey(), - entry.getValue() - )) - .collect(Collectors.toList()) - ) - ); - - committer.run(); - - try { - commitLock.lock(); - final File commitFile = computeCommitFile(); - final Map commitHydrants = Maps.newHashMap(); - if (commitFile.exists()) { - // merge current hydrants with existing hydrants - final Committed oldCommitted = objectMapper.readValue(commitFile, Committed.class); - commitHydrants.putAll(oldCommitted.getHydrants()); + if (committer != null) { + log.info( + "Committing metadata[%s] for sinks[%s].", commitMetadata, Joiner.on(", ").join( + currentHydrants.entrySet() + .stream() + .map(entry -> StringUtils.format( + "%s:%d", + entry.getKey(), + entry.getValue() + )) + .collect(Collectors.toList()) + ) + ); + + committer.run(); + + try { + commitLock.lock(); + final Map commitHydrants = Maps.newHashMap(); + final Committed oldCommit = readCommit(); + if (oldCommit != null) { + // merge current hydrants with existing hydrants + commitHydrants.putAll(oldCommit.getHydrants()); + } + commitHydrants.putAll(currentHydrants); + writeCommit(new Committed(commitHydrants, commitMetadata)); + } + finally { + commitLock.unlock(); } - commitHydrants.putAll(currentHydrants); - objectMapper.writeValue(commitFile, new Committed(commitHydrants, commitMetadata)); - } - finally { - commitLock.unlock(); } + // return null if committer is null return commitMetadata; } catch (Exception e) { @@ -493,7 +497,7 @@ public Object doCall() } @Override - public ListenableFuture persistAll(final Committer committer) + public ListenableFuture persistAll(@Nullable final Committer committer) { // Submit persistAll task to the persistExecutor return persist(sinks.keySet(), committer); @@ -502,7 +506,7 @@ public ListenableFuture persistAll(final Committer committer) @Override public ListenableFuture push( final Collection identifiers, - final Committer committer + @Nullable final Committer committer ) { final Map theSinks = Maps.newHashMap(); @@ -707,6 +711,9 @@ public void close() intermediateTempExecutor == null || intermediateTempExecutor.awaitTermination(365, TimeUnit.DAYS), "intermediateTempExecutor not terminated" ); + persistExecutor = null; + pushExecutor = null; + intermediateTempExecutor = null; } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -746,6 +753,7 @@ public void closeNow() } try { shutdownExecutors(); + // We don't wait for pushExecutor to be terminated. See Javadoc for more details. Preconditions.checkState( persistExecutor == null || persistExecutor.awaitTermination(365, TimeUnit.DAYS), "persistExecutor not terminated" @@ -754,6 +762,8 @@ public void closeNow() intermediateTempExecutor == null || intermediateTempExecutor.awaitTermination(365, TimeUnit.DAYS), "intermediateTempExecutor not terminated" ); + persistExecutor = null; + intermediateTempExecutor = null; } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -1042,10 +1052,9 @@ public Object apply(@Nullable Object input) log.info("Removing commit metadata for segment[%s].", identifier); try { commitLock.lock(); - final File commitFile = computeCommitFile(); - if (commitFile.exists()) { - final Committed oldCommitted = objectMapper.readValue(commitFile, Committed.class); - objectMapper.writeValue(commitFile, oldCommitted.without(identifier.getIdentifierAsString())); + final Committed oldCommit = readCommit(); + if (oldCommit != null) { + writeCommit(oldCommit.without(identifier.getIdentifierAsString())); } } catch (Exception e) { @@ -1098,6 +1107,23 @@ public Object apply(@Nullable Object input) ); } + private Committed readCommit() throws IOException + { + final File commitFile = computeCommitFile(); + if (commitFile.exists()) { + // merge current hydrants with existing hydrants + return objectMapper.readValue(commitFile, Committed.class); + } else { + return null; + } + } + + private void writeCommit(Committed newCommit) throws IOException + { + final File commitFile = computeCommitFile(); + objectMapper.writeValue(commitFile, newCommit); + } + private File computeCommitFile() { return new File(tuningConfig.getBasePersistDirectory(), "commit.json"); diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/BaseAppenderatorDriver.java b/server/src/main/java/io/druid/segment/realtime/appenderator/BaseAppenderatorDriver.java new file mode 100644 index 000000000000..24482d19c96c --- /dev/null +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/BaseAppenderatorDriver.java @@ -0,0 +1,541 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.realtime.appenderator; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import io.druid.data.input.Committer; +import io.druid.data.input.InputRow; +import io.druid.java.util.common.ISE; +import io.druid.java.util.common.concurrent.Execs; +import io.druid.java.util.common.logger.Logger; +import io.druid.segment.realtime.appenderator.SegmentWithState.SegmentState; +import org.joda.time.DateTime; + +import javax.annotation.Nullable; +import java.io.Closeable; +import java.io.IOException; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NavigableMap; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * A BaseAppenderatorDriver drives an Appenderator to index a finite stream of data. This class does not help you + * index unbounded streams. All handoff is done at the end of indexing. + *

    + * This class helps with doing things that Appenderators don't, including deciding which segments to use (with a + * SegmentAllocator), publishing segments to the metadata store (with a SegmentPublisher). + *

    + * This class has two child classes, i.e., {@link BatchAppenderatorDriver} and {@link StreamAppenderatorDriver}, + * which are for batch and streaming ingestion, respectively. This class provides some fundamental methods for making + * the child classes' life easier like {@link #pushInBackground}, {@link #dropInBackground}, or + * {@link #publishInBackground}. The child classes can use these methods to achieve their goal. + *

    + * Note that the commit metadata stored by this class via the underlying Appenderator is not the same metadata as + * you pass in. It's wrapped in some extra metadata needed by the driver. + */ +public abstract class BaseAppenderatorDriver implements Closeable +{ + /** + * Allocated segments for a sequence + */ + static class SegmentsForSequence + { + // Interval Start millis -> List of Segments for this interval + // there might be multiple segments for a start interval, for example one segment + // can be in APPENDING state and others might be in PUBLISHING state + private final NavigableMap> intervalToSegmentStates; + + // most recently allocated segment + private String lastSegmentId; + + SegmentsForSequence() + { + this.intervalToSegmentStates = new TreeMap<>(); + } + + SegmentsForSequence( + NavigableMap> intervalToSegmentStates, + String lastSegmentId + ) + { + this.intervalToSegmentStates = intervalToSegmentStates; + this.lastSegmentId = lastSegmentId; + } + + void add(SegmentIdentifier identifier) + { + intervalToSegmentStates.computeIfAbsent(identifier.getInterval().getStartMillis(), k -> new LinkedList<>()) + .addFirst(SegmentWithState.newSegment(identifier)); + lastSegmentId = identifier.getIdentifierAsString(); + } + + Entry> floor(long timestamp) + { + return intervalToSegmentStates.floorEntry(timestamp); + } + + LinkedList get(long timestamp) + { + return intervalToSegmentStates.get(timestamp); + } + + Stream segmentStateStream() + { + return intervalToSegmentStates.values().stream().flatMap(Collection::stream); + } + } + + private static final Logger log = new Logger(BaseAppenderatorDriver.class); + + private final SegmentAllocator segmentAllocator; + private final UsedSegmentChecker usedSegmentChecker; + + protected final Appenderator appenderator; + // sequenceName -> segmentsForSequence + // This map should be locked with itself before accessing it. + // Note: BatchAppenderatorDriver currently doesn't need to lock this map because it doesn't do anything concurrently. + // However, it's desired to do some operations like indexing and pushing at the same time. Locking this map is also + // required in BatchAppenderatorDriver once this feature is supported. + protected final Map segments = new TreeMap<>(); + protected final ListeningExecutorService executor; + + BaseAppenderatorDriver( + Appenderator appenderator, + SegmentAllocator segmentAllocator, + UsedSegmentChecker usedSegmentChecker + ) + { + this.appenderator = Preconditions.checkNotNull(appenderator, "appenderator"); + this.segmentAllocator = Preconditions.checkNotNull(segmentAllocator, "segmentAllocator"); + this.usedSegmentChecker = Preconditions.checkNotNull(usedSegmentChecker, "usedSegmentChecker"); + this.executor = MoreExecutors.listeningDecorator(Execs.singleThreaded("publish-%d")); + } + + @VisibleForTesting + Map getSegments() + { + return segments; + } + + /** + * Perform any initial setup and return currently persisted commit metadata. + *

    + * Note that this method returns the same metadata you've passed in with your Committers, even though this class + * stores extra metadata on disk. + * + * @return currently persisted commit metadata + */ + @Nullable + public abstract Object startJob(); + + /** + * Find a segment in the {@link SegmentState#APPENDING} state for the given timestamp and sequenceName. + */ + private SegmentIdentifier getAppendableSegment(final DateTime timestamp, final String sequenceName) + { + synchronized (segments) { + final SegmentsForSequence segmentsForSequence = segments.get(sequenceName); + + if (segmentsForSequence == null) { + return null; + } + + final Map.Entry> candidateEntry = segmentsForSequence.floor( + timestamp.getMillis() + ); + if (candidateEntry != null + && candidateEntry.getValue().getFirst().getSegmentIdentifier().getInterval().contains(timestamp) + && candidateEntry.getValue().getFirst().getState() == SegmentState.APPENDING) { + return candidateEntry.getValue().getFirst().getSegmentIdentifier(); + } else { + return null; + } + } + } + + /** + * Return a segment usable for "timestamp". May return null if no segment can be allocated. + * + * @param row input row + * @param sequenceName sequenceName for potential segment allocation + * @param skipSegmentLineageCheck if false, perform lineage validation using previousSegmentId for this sequence. + * Should be set to false if replica tasks would index events in same order + * + * @return identifier, or null + * + * @throws IOException if an exception occurs while allocating a segment + */ + private SegmentIdentifier getSegment( + final InputRow row, + final String sequenceName, + final boolean skipSegmentLineageCheck + ) throws IOException + { + synchronized (segments) { + final DateTime timestamp = row.getTimestamp(); + final SegmentIdentifier existing = getAppendableSegment(timestamp, sequenceName); + if (existing != null) { + return existing; + } else { + // Allocate new segment. + final SegmentsForSequence segmentsForSequence = segments.get(sequenceName); + final SegmentIdentifier newSegment = segmentAllocator.allocate( + row, + sequenceName, + segmentsForSequence == null ? null : segmentsForSequence.lastSegmentId, + // send lastSegmentId irrespective of skipSegmentLineageCheck so that + // unique constraint for sequence_name_prev_id_sha1 does not fail for + // allocatePendingSegment in IndexerSQLMetadataStorageCoordinator + skipSegmentLineageCheck + ); + + if (newSegment != null) { + for (SegmentIdentifier identifier : appenderator.getSegments()) { + if (identifier.equals(newSegment)) { + throw new ISE( + "WTF?! Allocated segment[%s] which conflicts with existing segment[%s].", + newSegment, + identifier + ); + } + } + + log.info("New segment[%s] for row[%s] sequenceName[%s].", newSegment, row, sequenceName); + addSegment(sequenceName, newSegment); + } else { + // Well, we tried. + log.warn("Cannot allocate segment for timestamp[%s], sequenceName[%s]. ", timestamp, sequenceName); + } + + return newSegment; + } + } + } + + private void addSegment(String sequenceName, SegmentIdentifier identifier) + { + synchronized (segments) { + segments.computeIfAbsent(sequenceName, k -> new SegmentsForSequence()) + .add(identifier); + } + } + + /** + * Add a row. Must not be called concurrently from multiple threads. + * + * @param row the row to add + * @param sequenceName sequenceName for this row's segment + * @param committerSupplier supplier of a committer associated with all data that has been added, including this row + * if {@param allowIncrementalPersists} is set to false then this will not be used + * @param skipSegmentLineageCheck if true, perform lineage validation using previousSegmentId for this sequence. + * Should be set to false if replica tasks would index events in same order + * @param allowIncrementalPersists whether to allow persist to happen when maxRowsInMemory or intermediate persist period + * threshold is hit + * + * @return {@link AppenderatorDriverAddResult} + * + * @throws IOException if there is an I/O error while allocating or writing to a segment + */ + protected AppenderatorDriverAddResult append( + final InputRow row, + final String sequenceName, + @Nullable final Supplier committerSupplier, + final boolean skipSegmentLineageCheck, + final boolean allowIncrementalPersists + ) throws IOException + { + Preconditions.checkNotNull(row, "row"); + Preconditions.checkNotNull(sequenceName, "sequenceName"); + + final SegmentIdentifier identifier = getSegment(row, sequenceName, skipSegmentLineageCheck); + + if (identifier != null) { + try { + final Appenderator.AppenderatorAddResult result = appenderator.add( + identifier, + row, + committerSupplier == null ? null : wrapCommitterSupplier(committerSupplier), + allowIncrementalPersists + ); + return AppenderatorDriverAddResult.ok( + identifier, + result.getNumRowsInSegment(), + appenderator.getTotalRowCount(), + result.isPersistRequired() + ); + } + catch (SegmentNotWritableException e) { + throw new ISE(e, "WTF?! Segment[%s] not writable when it should have been.", identifier); + } + } else { + return AppenderatorDriverAddResult.fail(); + } + } + + /** + * Returns a stream of {@link SegmentWithState} for the given sequenceNames. + */ + Stream getSegmentWithStates(Collection sequenceNames) + { + synchronized (segments) { + return sequenceNames + .stream() + .map(segments::get) + .filter(Objects::nonNull) + .flatMap(segmentsForSequence -> segmentsForSequence.intervalToSegmentStates.values().stream()) + .flatMap(Collection::stream); + } + } + + /** + * Push the given segments in background. + * + * @param wrappedCommitter should not be null if you want to persist intermediate states + * @param segmentIdentifiers identifiers of the segments to be pushed + * + * @return a future for pushing segments + */ + ListenableFuture pushInBackground( + @Nullable final WrappedCommitter wrappedCommitter, + final Collection segmentIdentifiers + ) + { + log.info("Pushing segments in background: [%s]", Joiner.on(", ").join(segmentIdentifiers)); + + return Futures.transform( + appenderator.push(segmentIdentifiers, wrappedCommitter), + (Function) segmentsAndMetadata -> { + // Sanity check + final Set pushedSegments = segmentsAndMetadata.getSegments().stream() + .map(SegmentIdentifier::fromDataSegment) + .collect(Collectors.toSet()); + if (!pushedSegments.equals(Sets.newHashSet(segmentIdentifiers))) { + throw new ISE( + "WTF?! Pushed different segments than requested. Pushed[%s], requested[%s].", + pushedSegments, + segmentIdentifiers + ); + } + + return segmentsAndMetadata; + }, + executor + ); + } + + /** + * Drop segments in background. The segments should be pushed (in batch ingestion) or published (in streaming + * ingestion) before being dropped. + * + * @param segmentsAndMetadata result of pushing or publishing + * + * @return a future for dropping segments + */ + ListenableFuture dropInBackground(SegmentsAndMetadata segmentsAndMetadata) + { + log.info("Dropping segments[%s]", segmentsAndMetadata.getSegments()); + final ListenableFuture dropFuture = Futures.allAsList( + segmentsAndMetadata + .getSegments() + .stream() + .map(segment -> appenderator.drop(SegmentIdentifier.fromDataSegment(segment))) + .collect(Collectors.toList()) + ); + + return Futures.transform( + dropFuture, + (Function) x -> { + final Object metadata = segmentsAndMetadata.getCommitMetadata(); + return new SegmentsAndMetadata( + segmentsAndMetadata.getSegments(), + metadata == null ? null : ((AppenderatorDriverMetadata) metadata).getCallerMetadata() + ); + } + ); + } + + /** + * Publish segments in background. The segments should be dropped (in batch ingestion) or pushed (in streaming + * ingestion) before being published. + * + * @param segmentsAndMetadata result of dropping or pushing + * @param publisher transactional segment publisher + * + * @return a future for publishing segments + */ + ListenableFuture publishInBackground( + SegmentsAndMetadata segmentsAndMetadata, + TransactionalSegmentPublisher publisher + ) + { + return executor.submit( + () -> { + if (segmentsAndMetadata.getSegments().isEmpty()) { + log.info("Nothing to publish, skipping publish step."); + } else { + log.info( + "Publishing segments with commitMetadata[%s]: [%s]", + segmentsAndMetadata.getCommitMetadata(), + Joiner.on(", ").join(segmentsAndMetadata.getSegments()) + ); + + try { + final Object metadata = segmentsAndMetadata.getCommitMetadata(); + final boolean published = publisher.publishSegments( + ImmutableSet.copyOf(segmentsAndMetadata.getSegments()), + metadata == null ? null : ((AppenderatorDriverMetadata) metadata).getCallerMetadata() + ); + + if (published) { + log.info("Published segments."); + } else { + log.info("Transaction failure while publishing segments, checking if someone else beat us to it."); + final Set segmentsIdentifiers = segmentsAndMetadata + .getSegments() + .stream() + .map(SegmentIdentifier::fromDataSegment) + .collect(Collectors.toSet()); + if (usedSegmentChecker.findUsedSegments(segmentsIdentifiers) + .equals(Sets.newHashSet(segmentsAndMetadata.getSegments()))) { + log.info("Our segments really do exist, awaiting handoff."); + } else { + throw new ISE("Failed to publish segments[%s]", segmentsAndMetadata.getSegments()); + } + } + } + catch (IOException e) { + throw Throwables.propagate(e); + } + } + + return segmentsAndMetadata; + } + ); + } + + /** + * Clears out all our state and also calls {@link Appenderator#clear()} on the underlying Appenderator. + */ + public void clear() throws InterruptedException + { + synchronized (segments) { + segments.clear(); + } + appenderator.clear(); + } + + /** + * Closes this driver. Does not close the underlying Appenderator; you should do that yourself. + */ + @Override + public void close() + { + executor.shutdownNow(); + } + + /** + * Wrapped committer for BaseAppenderatorDriver. Used in only {@link StreamAppenderatorDriver} because batch ingestion + * doesn't need committing intermediate states. + */ + static class WrappedCommitter implements Committer + { + private final Committer delegate; + private final AppenderatorDriverMetadata metadata; + + WrappedCommitter(Committer delegate, AppenderatorDriverMetadata metadata) + { + this.delegate = delegate; + this.metadata = metadata; + } + + @Override + public Object getMetadata() + { + return metadata; + } + + @Override + public void run() + { + delegate.run(); + } + } + + WrappedCommitter wrapCommitter(final Committer committer) + { + final AppenderatorDriverMetadata wrappedMetadata; + final Map snapshot; + synchronized (segments) { + snapshot = ImmutableMap.copyOf(segments); + } + + wrappedMetadata = new AppenderatorDriverMetadata( + ImmutableMap.copyOf( + Maps.transformValues( + snapshot, + (Function>) input -> ImmutableList.copyOf( + input.intervalToSegmentStates.values() + .stream() + .flatMap(Collection::stream) + .collect(Collectors.toList()) + ) + ) + ), + snapshot.entrySet() + .stream() + .collect( + Collectors.toMap( + Entry::getKey, + e -> e.getValue().lastSegmentId + ) + ), + committer.getMetadata() + ); + + return new WrappedCommitter(committer, wrappedMetadata); + } + + private Supplier wrapCommitterSupplier(final Supplier committerSupplier) + { + return () -> wrapCommitter(committerSupplier.get()); + } +} diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriver.java b/server/src/main/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriver.java new file mode 100644 index 000000000000..c67b31054bf1 --- /dev/null +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriver.java @@ -0,0 +1,209 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.realtime.appenderator; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import io.druid.data.input.InputRow; +import io.druid.java.util.common.ISE; +import io.druid.segment.realtime.appenderator.SegmentWithState.SegmentState; +import io.druid.timeline.DataSegment; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; + +/** + * This class is specifialized for batch ingestion. In batch ingestion, the segment lifecycle is like: + * + * 

    + * APPENDING -> PUSHED_AND_DROPPED -> PUBLISHED
+ *
    + * + *
      + *
    • APPENDING: Segment is available for appending.
      • + *
    • PUSHED_AND_DROPPED: Segment is pushed to deep storage and dropped from the local storage.
      • + *
    • PUBLISHED: Segment's metadata is published to metastore.
      • + *
    + */ +public class BatchAppenderatorDriver extends BaseAppenderatorDriver +{ + /** + * Create a driver. + * + * @param appenderator appenderator + * @param segmentAllocator segment allocator + * @param usedSegmentChecker used segment checker + */ + public BatchAppenderatorDriver( + Appenderator appenderator, + SegmentAllocator segmentAllocator, + UsedSegmentChecker usedSegmentChecker + ) + { + super(appenderator, segmentAllocator, usedSegmentChecker); + } + + /** + * This method always returns null because batch ingestion doesn't support restoring tasks on failures. + + * @return always null + */ + @Override + @Nullable + public Object startJob() + { + final Object metadata = appenderator.startJob(); + if (metadata != null) { + throw new ISE("Metadata should be null because BatchAppenderatorDriver never persists it"); + } + return null; + } + + /** + * Add a row. Must not be called concurrently from multiple threads. + * + * @param row the row to add + * @param sequenceName sequenceName for this row's segment + * + * @return {@link AppenderatorDriverAddResult} + * + * @throws IOException if there is an I/O error while allocating or writing to a segment + */ + public AppenderatorDriverAddResult add( + InputRow row, + String sequenceName + ) throws IOException + { + return append(row, sequenceName, null, false, true); + } + + /** + * Push and drop all segments in the {@link SegmentState#APPENDING} state. + * + * @param pushAndClearTimeoutMs timeout for pushing and dropping segments + * + * @return {@link SegmentsAndMetadata} for pushed and dropped segments + */ + public SegmentsAndMetadata pushAllAndClear(long pushAndClearTimeoutMs) + throws InterruptedException, ExecutionException, TimeoutException + { + final Collection sequences; + synchronized (segments) { + sequences = ImmutableList.copyOf(segments.keySet()); + } + + return pushAndClear(sequences, pushAndClearTimeoutMs); + } + + private SegmentsAndMetadata pushAndClear( + Collection sequenceNames, + long pushAndClearTimeoutMs + ) throws InterruptedException, ExecutionException, TimeoutException + { + final List segmentIdentifierList = getSegmentWithStates(sequenceNames) + .filter(segmentWithState -> segmentWithState.getState() == SegmentState.APPENDING) + .map(SegmentWithState::getSegmentIdentifier) + .collect(Collectors.toList()); + + final ListenableFuture future = Futures.transform( + pushInBackground(null, segmentIdentifierList), + this::dropInBackground + ); + + final SegmentsAndMetadata segmentsAndMetadata = pushAndClearTimeoutMs == 0L ? + future.get() : + future.get(pushAndClearTimeoutMs, TimeUnit.MILLISECONDS); + + // Sanity check + final Map pushedSegmentIdToSegmentMap = segmentsAndMetadata + .getSegments() + .stream() + .collect(Collectors.toMap( + SegmentIdentifier::fromDataSegment, + dataSegment -> dataSegment + )); + + final Map requestedSegmentIdsForSequences = getSegmentWithStates(sequenceNames) + .filter(segmentWithState -> segmentWithState.getState() == SegmentState.APPENDING) + .collect(Collectors.toMap( + SegmentWithState::getSegmentIdentifier, + segmentWithState -> segmentWithState + )); + + if (!pushedSegmentIdToSegmentMap.keySet().equals(requestedSegmentIdsForSequences.keySet())) { + throw new ISE( + "Pushed segments[%s] are different from the requested ones[%s]", + pushedSegmentIdToSegmentMap.keySet(), + requestedSegmentIdsForSequences.keySet() + ); + } + + // State transition + requestedSegmentIdsForSequences.forEach( + (segmentId, segmentWithState) -> { + segmentWithState.pushAndDrop(pushedSegmentIdToSegmentMap.get(segmentId)); + } + ); + + return segmentsAndMetadata; + } + + /** + * Publish all segments. + * + * @param publisher segment publisher + * + * @return a {@link ListenableFuture} for the publish task + */ + public ListenableFuture publishAll(final TransactionalSegmentPublisher publisher) + { + final Map snapshot; + synchronized (segments) { + snapshot = ImmutableMap.copyOf(segments); + } + + return publishInBackground( + new SegmentsAndMetadata( + snapshot + .values() + .stream() + .flatMap(SegmentsForSequence::segmentStateStream) + .map(segmentWithState -> Preconditions.checkNotNull( + segmentWithState.getDataSegment(), + "dataSegment for segmentId[%s]", + segmentWithState.getSegmentIdentifier()) + ) + .collect(Collectors.toList()), + null + ), + publisher + ); + } +} diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/SegmentWithState.java b/server/src/main/java/io/druid/segment/realtime/appenderator/SegmentWithState.java new file mode 100644 index 000000000000..3a89c1593cf9 --- /dev/null +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/SegmentWithState.java @@ -0,0 +1,153 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.realtime.appenderator; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import io.druid.timeline.DataSegment; + +import javax.annotation.Nullable; +import java.util.Collection; +import java.util.List; + +public class SegmentWithState +{ + /** + * Segment state transition is different in {@link BatchAppenderatorDriver} and {@link StreamAppenderatorDriver}. + * When a new segment is created, its state is {@link #APPENDING}. + * + * - In stream ingestion, the state of some segments can be changed to the {@link #APPEND_FINISHED} state. Data is + * not appended to these segments anymore, and they are waiting for beging published. + * See {@link StreamAppenderatorDriver#moveSegmentOut(String, List)}. + * - In batch ingestion, the state of some segments can be changed to the {@link #PUSHED_AND_DROPPED} state. These + * segments are pushed and dropped from the local storage, but not published yet. + * See {@link BatchAppenderatorDriver#pushAndClear(Collection, long)}. + * + * Note: If you need to add more states which are used differently in batch and streaming ingestion, consider moving + * SegmentState to {@link BatchAppenderatorDriver} and {@link StreamAppenderatorDriver}. + */ + public enum SegmentState + { + APPENDING, + APPEND_FINISHED, // only used in StreamAppenderatorDriver + PUSHED_AND_DROPPED; // only used in BatchAppenderatorDriver + + @JsonCreator + public static SegmentState fromString(@JsonProperty String name) + { + if (name.equalsIgnoreCase("ACTIVE")) { + return APPENDING; + } else if (name.equalsIgnoreCase("INACTIVE")) { + return APPEND_FINISHED; + } else { + return SegmentState.valueOf(name); + } + } + } + + private final SegmentIdentifier segmentIdentifier; + private SegmentState state; + + /** + * This is to keep what dataSegment object was created for {@link #segmentIdentifier} when + * {@link BaseAppenderatorDriver#pushInBackground} is called. + */ + @Nullable private DataSegment dataSegment; + + static SegmentWithState newSegment(SegmentIdentifier segmentIdentifier) + { + return new SegmentWithState(segmentIdentifier, SegmentState.APPENDING, null); + } + + static SegmentWithState newSegment(SegmentIdentifier segmentIdentifier, SegmentState state) + { + return new SegmentWithState(segmentIdentifier, state, null); + } + + @JsonCreator + public SegmentWithState( + @JsonProperty("segmentIdentifier") SegmentIdentifier segmentIdentifier, + @JsonProperty("state") SegmentState state, + @JsonProperty("dataSegment") @Nullable DataSegment dataSegment) + { + this.segmentIdentifier = segmentIdentifier; + this.state = state; + this.dataSegment = dataSegment; + } + + public void setState(SegmentState state) + { + this.state = state; + } + + /** + * Change the segment state to {@link SegmentState#APPEND_FINISHED}. The current state should be + * {@link SegmentState#APPENDING}. + */ + public void finishAppending() + { + checkStateTransition(this.state, SegmentState.APPENDING, SegmentState.APPEND_FINISHED); + this.state = SegmentState.APPEND_FINISHED; + } + + /** + * Change the segment state to {@link SegmentState#PUSHED_AND_DROPPED}. The current state should be + * {@link SegmentState#APPENDING}. This method should be called after the segment of {@link #segmentIdentifier} is + * completely pushed and dropped. + * + * @param dataSegment pushed {@link DataSegment} + */ + public void pushAndDrop(DataSegment dataSegment) + { + checkStateTransition(this.state, SegmentState.APPENDING, SegmentState.PUSHED_AND_DROPPED); + this.state = SegmentState.PUSHED_AND_DROPPED; + this.dataSegment = dataSegment; + } + + @JsonProperty + public SegmentIdentifier getSegmentIdentifier() + { + return segmentIdentifier; + } + + @JsonProperty + public SegmentState getState() + { + return state; + } + + @JsonProperty + @Nullable + public DataSegment getDataSegment() + { + return dataSegment; + } + + private static void checkStateTransition(SegmentState actualFrom, SegmentState expectedFrom, SegmentState to) + { + Preconditions.checkState( + actualFrom == expectedFrom, + "Wrong state transition from [%s] to [%s]", + actualFrom, + to + ); + } +} diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/SegmentsAndMetadata.java b/server/src/main/java/io/druid/segment/realtime/appenderator/SegmentsAndMetadata.java index f5d7ae1e1902..72a7f8b325c4 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/SegmentsAndMetadata.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/SegmentsAndMetadata.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList; import io.druid.timeline.DataSegment; +import javax.annotation.Nullable; import java.util.List; import java.util.Objects; @@ -34,13 +35,14 @@ public class SegmentsAndMetadata public SegmentsAndMetadata( List segments, - Object commitMetadata + @Nullable Object commitMetadata ) { this.segments = ImmutableList.copyOf(segments); this.commitMetadata = commitMetadata; } + @Nullable public Object getCommitMetadata() { return commitMetadata; diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriver.java b/server/src/main/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriver.java new file mode 100644 index 000000000000..a5b1a3fbdd65 --- /dev/null +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriver.java @@ -0,0 +1,416 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.realtime.appenderator; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.google.common.base.Throwables; +import com.google.common.util.concurrent.AsyncFunction; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.SettableFuture; +import io.druid.data.input.Committer; +import io.druid.data.input.InputRow; +import io.druid.java.util.common.ISE; +import io.druid.java.util.common.logger.Logger; +import io.druid.query.SegmentDescriptor; +import io.druid.segment.realtime.FireDepartmentMetrics; +import io.druid.segment.realtime.appenderator.SegmentWithState.SegmentState; +import io.druid.segment.realtime.plumber.SegmentHandoffNotifier; +import io.druid.segment.realtime.plumber.SegmentHandoffNotifierFactory; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NavigableMap; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +/** + * This class is specialized for streaming ingestion. In streaming ingestion, the segment lifecycle is like: + * + * 
    + * APPENDING -> APPEND_FINISHED -> PUBLISHED
+ *
    + * + *
      + *
    • APPENDING: Segment is available for appending.
      • + *
    • APPEND_FINISHED: Segment cannot be updated (data cannot be added anymore) and is waiting for being published.
      • + *
    • PUBLISHED: Segment is pushed to deep storage, its metadata is published to metastore, and finally the segment is + * dropped from local storage
      • + *
    + */ +public class StreamAppenderatorDriver extends BaseAppenderatorDriver +{ + private static final Logger log = new Logger(StreamAppenderatorDriver.class); + + private final SegmentHandoffNotifier handoffNotifier; + private final FireDepartmentMetrics metrics; + private final ObjectMapper objectMapper; + + /** + * Create a driver. + * + * @param appenderator appenderator + * @param segmentAllocator segment allocator + * @param handoffNotifierFactory handoff notifier factory + * @param usedSegmentChecker used segment checker + * @param objectMapper object mapper, used for serde of commit metadata + * @param metrics Firedepartment metrics + */ + public StreamAppenderatorDriver( + Appenderator appenderator, + SegmentAllocator segmentAllocator, + SegmentHandoffNotifierFactory handoffNotifierFactory, + UsedSegmentChecker usedSegmentChecker, + ObjectMapper objectMapper, + FireDepartmentMetrics metrics + ) + { + super(appenderator, segmentAllocator, usedSegmentChecker); + + this.handoffNotifier = Preconditions.checkNotNull(handoffNotifierFactory, "handoffNotifierFactory") + .createSegmentHandoffNotifier(appenderator.getDataSource()); + this.metrics = Preconditions.checkNotNull(metrics, "metrics"); + this.objectMapper = Preconditions.checkNotNull(objectMapper, "objectMapper"); + } + + @Override + @Nullable + public Object startJob() + { + handoffNotifier.start(); + + final AppenderatorDriverMetadata metadata = objectMapper.convertValue( + appenderator.startJob(), + AppenderatorDriverMetadata.class + ); + + log.info("Restored metadata[%s].", metadata); + + if (metadata != null) { + synchronized (segments) { + final Map lastSegmentIds = metadata.getLastSegmentIds(); + Preconditions.checkState( + metadata.getSegments().keySet().equals(lastSegmentIds.keySet()), + "Sequences for segment states and last segment IDs are not same" + ); + + final Map builders = new TreeMap<>(); + + for (Entry> entry : metadata.getSegments().entrySet()) { + final String sequenceName = entry.getKey(); + final SegmentsForSequenceBuilder builder = new SegmentsForSequenceBuilder(lastSegmentIds.get(sequenceName)); + builders.put(sequenceName, builder); + entry.getValue().forEach(builder::add); + } + + builders.forEach((sequence, builder) -> segments.put(sequence, builder.build())); + } + + return metadata.getCallerMetadata(); + } else { + return null; + } + } + + /** + * Add a row. Must not be called concurrently from multiple threads. + * + * @param row the row to add + * @param sequenceName sequenceName for this row's segment + * @param committerSupplier supplier of a committer associated with all data that has been added, including this row + * if {@param allowIncrementalPersists} is set to false then this will not be used + * @param skipSegmentLineageCheck if true, perform lineage validation using previousSegmentId for this sequence. + * Should be set to false if replica tasks would index events in same order + * @param allowIncrementalPersists whether to allow persist to happen when maxRowsInMemory or intermediate persist period + * threshold is hit + * + * @return {@link AppenderatorDriverAddResult} + * + * @throws IOException if there is an I/O error while allocating or writing to a segment + */ + public AppenderatorDriverAddResult add( + final InputRow row, + final String sequenceName, + final Supplier committerSupplier, + final boolean skipSegmentLineageCheck, + final boolean allowIncrementalPersists + ) throws IOException + { + return append(row, sequenceName, committerSupplier, skipSegmentLineageCheck, allowIncrementalPersists); + } + + /** + * Move a set of identifiers out from "active", making way for newer segments. + * This method is to support KafkaIndexTask's legacy mode and will be removed in the future. + * See KakfaIndexTask.runLegacy(). + */ + public void moveSegmentOut(final String sequenceName, final List identifiers) + { + synchronized (segments) { + final SegmentsForSequence activeSegmentsForSequence = segments.get(sequenceName); + if (activeSegmentsForSequence == null) { + throw new ISE("WTF?! Asked to remove segments for sequenceName[%s] which doesn't exist...", sequenceName); + } + + for (final SegmentIdentifier identifier : identifiers) { + log.info("Moving segment[%s] out of active list.", identifier); + final long key = identifier.getInterval().getStartMillis(); + if (activeSegmentsForSequence.get(key) == null || activeSegmentsForSequence.get(key).stream().noneMatch( + segmentWithState -> { + if (segmentWithState.getSegmentIdentifier().equals(identifier)) { + segmentWithState.finishAppending(); + return true; + } else { + return false; + } + } + )) { + throw new ISE("WTF?! Asked to remove segment[%s] that didn't exist...", identifier); + } + } + } + } + + /** + * Persist all data indexed through this driver so far. Blocks until complete. + *

    + * Should be called after all data has been added through {@link #add(InputRow, String, Supplier, boolean, boolean)}. + * + * @param committer committer representing all data that has been added so far + * + * @return commitMetadata persisted + */ + public Object persist(final Committer committer) throws InterruptedException + { + try { + log.info("Persisting data."); + final long start = System.currentTimeMillis(); + final Object commitMetadata = appenderator.persistAll(wrapCommitter(committer)).get(); + log.info("Persisted pending data in %,dms.", System.currentTimeMillis() - start); + return commitMetadata; + } + catch (InterruptedException e) { + throw e; + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + + /** + * Persist all data indexed through this driver so far. Returns a future of persisted commitMetadata. + *

    + * Should be called after all data has been added through {@link #add(InputRow, String, Supplier, boolean, boolean)}. + * + * @param committer committer representing all data that has been added so far + * + * @return future containing commitMetadata persisted + */ + public ListenableFuture persistAsync(final Committer committer) + { + return appenderator.persistAll(wrapCommitter(committer)); + } + + /** + * Execute a task in background to publish all segments corresponding to the given sequence names. The task + * internally pushes the segments to the deep storage first, and then publishes the metadata to the metadata storage. + * + * @param publisher segment publisher + * @param committer committer + * @param sequenceNames a collection of sequence names to be published + * + * @return a {@link ListenableFuture} for the submitted task which removes published {@code sequenceNames} from + * {@code activeSegments} and {@code publishPendingSegments} + */ + public ListenableFuture publish( + final TransactionalSegmentPublisher publisher, + final Committer committer, + final Collection sequenceNames + ) + { + final List theSegments = getSegmentWithStates(sequenceNames) + .map(SegmentWithState::getSegmentIdentifier) + .collect(Collectors.toList()); + + final ListenableFuture publishFuture = Futures.transform( + pushInBackground(wrapCommitter(committer), theSegments), + (AsyncFunction) segmentsAndMetadata -> publishInBackground( + segmentsAndMetadata, + publisher + ) + ); + + return Futures.transform( + publishFuture, + (Function) segmentsAndMetadata -> { + synchronized (segments) { + sequenceNames.forEach(segments::remove); + } + return segmentsAndMetadata; + } + ); + } + + /** + * Register the segments in the given {@link SegmentsAndMetadata} to be handed off and execute a background task which + * waits until the hand off completes. + * + * @param segmentsAndMetadata the result segments and metadata of + * {@link #publish(TransactionalSegmentPublisher, Committer, Collection)} + * + * @return null if the input segmentsAndMetadata is null. Otherwise, a {@link ListenableFuture} for the submitted task + * which returns {@link SegmentsAndMetadata} containing the segments successfully handed off and the metadata + * of the caller of {@link AppenderatorDriverMetadata} + */ + public ListenableFuture registerHandoff(SegmentsAndMetadata segmentsAndMetadata) + { + if (segmentsAndMetadata == null) { + return Futures.immediateFuture(null); + + } else { + final List waitingSegmentIdList = segmentsAndMetadata.getSegments().stream() + .map(SegmentIdentifier::fromDataSegment) + .collect(Collectors.toList()); + final Object metadata = Preconditions.checkNotNull(segmentsAndMetadata.getCommitMetadata(), "commitMetadata"); + + if (waitingSegmentIdList.isEmpty()) { + return Futures.immediateFuture( + new SegmentsAndMetadata( + segmentsAndMetadata.getSegments(), + ((AppenderatorDriverMetadata) metadata).getCallerMetadata() + ) + ); + } + + log.info("Register handoff of segments: [%s]", waitingSegmentIdList); + + final SettableFuture resultFuture = SettableFuture.create(); + final AtomicInteger numRemainingHandoffSegments = new AtomicInteger(waitingSegmentIdList.size()); + + for (final SegmentIdentifier segmentIdentifier : waitingSegmentIdList) { + handoffNotifier.registerSegmentHandoffCallback( + new SegmentDescriptor( + segmentIdentifier.getInterval(), + segmentIdentifier.getVersion(), + segmentIdentifier.getShardSpec().getPartitionNum() + ), + MoreExecutors.sameThreadExecutor(), + () -> { + log.info("Segment[%s] successfully handed off, dropping.", segmentIdentifier); + metrics.incrementHandOffCount(); + + final ListenableFuture dropFuture = appenderator.drop(segmentIdentifier); + Futures.addCallback( + dropFuture, + new FutureCallback() + { + @Override + public void onSuccess(Object result) + { + if (numRemainingHandoffSegments.decrementAndGet() == 0) { + log.info("Successfully handed off [%d] segments.", segmentsAndMetadata.getSegments().size()); + resultFuture.set( + new SegmentsAndMetadata( + segmentsAndMetadata.getSegments(), + ((AppenderatorDriverMetadata) metadata).getCallerMetadata() + ) + ); + } + } + + @Override + public void onFailure(Throwable e) + { + log.warn(e, "Failed to drop segment[%s]?!", segmentIdentifier); + numRemainingHandoffSegments.decrementAndGet(); + resultFuture.setException(e); + } + } + ); + } + ); + } + + return resultFuture; + } + } + + public ListenableFuture publishAndRegisterHandoff( + final TransactionalSegmentPublisher publisher, + final Committer committer, + final Collection sequenceNames + ) + { + return Futures.transform( + publish(publisher, committer, sequenceNames), + this::registerHandoff + ); + } + + @Override + public void close() + { + super.close(); + handoffNotifier.close(); + } + + private static class SegmentsForSequenceBuilder + { + private final NavigableMap> intervalToSegmentStates; + private final String lastSegmentId; + + SegmentsForSequenceBuilder(String lastSegmentId) + { + this.intervalToSegmentStates = new TreeMap<>(); + this.lastSegmentId = lastSegmentId; + } + + void add(SegmentWithState segmentWithState) + { + final SegmentIdentifier identifier = segmentWithState.getSegmentIdentifier(); + final LinkedList segmentsInInterval = intervalToSegmentStates.computeIfAbsent( + identifier.getInterval().getStartMillis(), + k -> new LinkedList<>() + ); + // always keep APPENDING segments for an interval start millis in the front + if (segmentWithState.getState() == SegmentState.APPENDING) { + segmentsInInterval.addFirst(segmentWithState); + } else { + segmentsInInterval.addLast(segmentWithState); + } + } + + SegmentsForSequence build() + { + return new SegmentsForSequence(intervalToSegmentStates, lastSegmentId); + } + } +} diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/TransactionalSegmentPublisher.java b/server/src/main/java/io/druid/segment/realtime/appenderator/TransactionalSegmentPublisher.java index de74f58966ac..359708a78c0a 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/TransactionalSegmentPublisher.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/TransactionalSegmentPublisher.java @@ -21,6 +21,7 @@ import io.druid.timeline.DataSegment; +import javax.annotation.Nullable; import java.io.IOException; import java.util.Set; @@ -35,6 +36,6 @@ public interface TransactionalSegmentPublisher */ boolean publishSegments( Set segments, - Object commitMetadata + @Nullable Object commitMetadata ) throws IOException; } diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriverTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriverTest.java new file mode 100644 index 000000000000..c69c8e443a7d --- /dev/null +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriverTest.java @@ -0,0 +1,189 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.realtime.appenderator; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import io.druid.data.input.InputRow; +import io.druid.data.input.MapBasedInputRow; +import io.druid.java.util.common.DateTimes; +import io.druid.java.util.common.Intervals; +import io.druid.java.util.common.granularity.Granularities; +import io.druid.segment.realtime.appenderator.BaseAppenderatorDriver.SegmentsForSequence; +import io.druid.segment.realtime.appenderator.StreamAppenderatorDriverTest.TestSegmentAllocator; +import io.druid.segment.realtime.appenderator.SegmentWithState.SegmentState; +import io.druid.timeline.partition.NumberedShardSpec; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public class BatchAppenderatorDriverTest +{ + private static final String DATA_SOURCE = "foo"; + private static final String VERSION = "abc123"; + private static final int MAX_ROWS_IN_MEMORY = 100; + private static final long TIMEOUT = 1000; + + private static final List ROWS = Arrays.asList( + new MapBasedInputRow( + DateTimes.of("2000"), + ImmutableList.of("dim1"), + ImmutableMap.of("dim1", "foo", "met1", "1") + ), + new MapBasedInputRow( + DateTimes.of("2000T01"), + ImmutableList.of("dim1"), + ImmutableMap.of("dim1", "foo", "met1", 2.0) + ), + new MapBasedInputRow( + DateTimes.of("2000T01"), + ImmutableList.of("dim2"), + ImmutableMap.of("dim2", "bar", "met1", 2.0) + ) + ); + + private SegmentAllocator allocator; + private AppenderatorTester appenderatorTester; + private BatchAppenderatorDriver driver; + + @Before + public void setup() + { + appenderatorTester = new AppenderatorTester(MAX_ROWS_IN_MEMORY); + allocator = new TestSegmentAllocator(DATA_SOURCE, Granularities.HOUR); + driver = new BatchAppenderatorDriver( + appenderatorTester.getAppenderator(), + allocator, + new TestUsedSegmentChecker(appenderatorTester) + ); + } + + @After + public void tearDown() throws Exception + { + driver.clear(); + driver.close(); + } + + @Test + public void testSimple() throws Exception + { + Assert.assertNull(driver.startJob()); + + for (InputRow row : ROWS) { + Assert.assertTrue(driver.add(row, "dummy").isOk()); + } + + checkSegmentStates(2, SegmentState.APPENDING); + + driver.pushAllAndClear(TIMEOUT); + + checkSegmentStates(2, SegmentState.PUSHED_AND_DROPPED); + + final SegmentsAndMetadata published = driver.publishAll(makeOkPublisher()).get( + TIMEOUT, + TimeUnit.MILLISECONDS + ); + + Assert.assertEquals( + ImmutableSet.of( + new SegmentIdentifier(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)), + new SegmentIdentifier(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0)) + ), + published.getSegments() + .stream() + .map(SegmentIdentifier::fromDataSegment) + .collect(Collectors.toSet()) + ); + + Assert.assertNull(published.getCommitMetadata()); + } + + @Test + public void testIncrementalPush() throws Exception + { + Assert.assertNull(driver.startJob()); + + int i = 0; + for (InputRow row : ROWS) { + Assert.assertTrue(driver.add(row, "dummy").isOk()); + + checkSegmentStates(1, SegmentState.APPENDING); + checkSegmentStates(i, SegmentState.PUSHED_AND_DROPPED); + + driver.pushAllAndClear(TIMEOUT); + checkSegmentStates(0, SegmentState.APPENDING); + checkSegmentStates(++i, SegmentState.PUSHED_AND_DROPPED); + } + + final SegmentsAndMetadata published = driver.publishAll(makeOkPublisher()).get( + TIMEOUT, + TimeUnit.MILLISECONDS + ); + + Assert.assertEquals( + ImmutableSet.of( + new SegmentIdentifier(DATA_SOURCE, Intervals.of("2000/PT1H"), VERSION, new NumberedShardSpec(0, 0)), + new SegmentIdentifier(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(0, 0)), + new SegmentIdentifier(DATA_SOURCE, Intervals.of("2000T01/PT1H"), VERSION, new NumberedShardSpec(1, 0)) + ), + published.getSegments() + .stream() + .map(SegmentIdentifier::fromDataSegment) + .collect(Collectors.toSet()) + ); + + Assert.assertNull(published.getCommitMetadata()); + } + + @Test + public void testRestart() + { + Assert.assertNull(driver.startJob()); + driver.close(); + appenderatorTester.getAppenderator().close(); + + Assert.assertNull(driver.startJob()); + } + + private void checkSegmentStates(int expectedNumSegmentsInState, SegmentState expectedState) + { + final SegmentsForSequence segmentsForSequence = driver.getSegments().get("dummy"); + Assert.assertNotNull(segmentsForSequence); + final List segmentWithStates = segmentsForSequence + .segmentStateStream() + .filter(segmentWithState -> segmentWithState.getState() == expectedState) + .collect(Collectors.toList()); + + Assert.assertEquals(expectedNumSegmentsInState, segmentWithStates.size()); + } + + static TransactionalSegmentPublisher makeOkPublisher() + { + return (segments, commitMetadata) -> true; + } +} diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/SegmentWithStateTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/SegmentWithStateTest.java new file mode 100644 index 000000000000..569eebdc8a11 --- /dev/null +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/SegmentWithStateTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package io.druid.segment.realtime.appenderator; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.jackson.DefaultObjectMapper; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; + +public class SegmentWithStateTest +{ + @Test + public void testSerde() throws IOException + { + final ObjectMapper objectMapper = new DefaultObjectMapper(); + final byte[] bytes = objectMapper.writeValueAsBytes(SegmentWithState.SegmentState.APPEND_FINISHED); + Assert.assertEquals( + SegmentWithState.SegmentState.APPEND_FINISHED, + objectMapper.readValue(bytes, SegmentWithState.SegmentState.class) + ); + } + + @Test + public void testSerdeForBackwardCompatibility() throws IOException + { + final ObjectMapper objectMapper = new DefaultObjectMapper(); + Assert.assertEquals( + SegmentWithState.SegmentState.APPENDING, + objectMapper.readValue("\"ACTIVE\"", SegmentWithState.SegmentState.class) + ); + Assert.assertEquals( + SegmentWithState.SegmentState.APPEND_FINISHED, + objectMapper.readValue("\"INACTIVE\"", SegmentWithState.SegmentState.class) + ); + } +} diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorDriverFailTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverFailTest.java similarity index 94% rename from server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorDriverFailTest.java rename to server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverFailTest.java index 1cc19f4c1458..a249ccb7f088 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorDriverFailTest.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverFailTest.java @@ -40,9 +40,9 @@ import io.druid.query.SegmentDescriptor; import io.druid.segment.incremental.IndexSizeExceededException; import io.druid.segment.realtime.FireDepartmentMetrics; -import io.druid.segment.realtime.appenderator.AppenderatorDriverTest.TestCommitterSupplier; -import io.druid.segment.realtime.appenderator.AppenderatorDriverTest.TestSegmentAllocator; -import io.druid.segment.realtime.appenderator.AppenderatorDriverTest.TestSegmentHandoffNotifierFactory; +import io.druid.segment.realtime.appenderator.StreamAppenderatorDriverTest.TestCommitterSupplier; +import io.druid.segment.realtime.appenderator.StreamAppenderatorDriverTest.TestSegmentAllocator; +import io.druid.segment.realtime.appenderator.StreamAppenderatorDriverTest.TestSegmentHandoffNotifierFactory; import io.druid.timeline.DataSegment; import org.hamcrest.CoreMatchers; import org.joda.time.Interval; @@ -65,7 +65,7 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; -public class AppenderatorDriverFailTest +public class StreamAppenderatorDriverFailTest { private static final String DATA_SOURCE = "foo"; private static final ObjectMapper OBJECT_MAPPER = new DefaultObjectMapper(); @@ -91,7 +91,7 @@ public class AppenderatorDriverFailTest SegmentAllocator allocator; TestSegmentHandoffNotifierFactory segmentHandoffNotifierFactory; - AppenderatorDriver driver; + StreamAppenderatorDriver driver; @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -121,7 +121,7 @@ public void testFailDuringPersist() throws IOException, InterruptedException, Ti + "[[foo_2000-01-01T00:00:00.000Z_2000-01-01T01:00:00.000Z_abc123, " + "foo_2000-01-01T01:00:00.000Z_2000-01-01T02:00:00.000Z_abc123]]"); - driver = new AppenderatorDriver( + driver = new StreamAppenderatorDriver( createPersistFailAppenderator(), allocator, segmentHandoffNotifierFactory, @@ -139,11 +139,11 @@ public void testFailDuringPersist() throws IOException, InterruptedException, Ti for (int i = 0; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); - Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier).isOk()); + Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk()); } driver.publish( - AppenderatorDriverTest.makeOkPublisher(), + StreamAppenderatorDriverTest.makeOkPublisher(), committerSupplier.get(), ImmutableList.of("dummy") ).get(PUBLISH_TIMEOUT, TimeUnit.MILLISECONDS); @@ -158,7 +158,7 @@ public void testFailDuringPush() throws IOException, InterruptedException, Timeo + "[[foo_2000-01-01T00:00:00.000Z_2000-01-01T01:00:00.000Z_abc123, " + "foo_2000-01-01T01:00:00.000Z_2000-01-01T02:00:00.000Z_abc123]]"); - driver = new AppenderatorDriver( + driver = new StreamAppenderatorDriver( createPushFailAppenderator(), allocator, segmentHandoffNotifierFactory, @@ -176,11 +176,11 @@ public void testFailDuringPush() throws IOException, InterruptedException, Timeo for (int i = 0; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); - Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier).isOk()); + Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk()); } driver.publish( - AppenderatorDriverTest.makeOkPublisher(), + StreamAppenderatorDriverTest.makeOkPublisher(), committerSupplier.get(), ImmutableList.of("dummy") ).get(PUBLISH_TIMEOUT, TimeUnit.MILLISECONDS); @@ -195,7 +195,7 @@ public void testFailDuringDrop() throws IOException, InterruptedException, Timeo "Fail test while dropping segment[foo_2000-01-01T00:00:00.000Z_2000-01-01T01:00:00.000Z_abc123]" ); - driver = new AppenderatorDriver( + driver = new StreamAppenderatorDriver( createDropFailAppenderator(), allocator, segmentHandoffNotifierFactory, @@ -213,11 +213,11 @@ public void testFailDuringDrop() throws IOException, InterruptedException, Timeo for (int i = 0; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); - Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier).isOk()); + Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk()); } final SegmentsAndMetadata published = driver.publish( - AppenderatorDriverTest.makeOkPublisher(), + StreamAppenderatorDriverTest.makeOkPublisher(), committerSupplier.get(), ImmutableList.of("dummy") ).get(PUBLISH_TIMEOUT, TimeUnit.MILLISECONDS); diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorDriverTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverTest.java similarity index 90% rename from server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorDriverTest.java rename to server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverTest.java index d9681351c77e..ee92d2e2d7d0 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorDriverTest.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverTest.java @@ -27,8 +27,6 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; -import com.google.common.collect.Ordering; -import com.google.common.collect.Sets; import com.google.common.util.concurrent.ListenableFuture; import io.druid.data.input.Committer; import io.druid.data.input.InputRow; @@ -44,10 +42,7 @@ import io.druid.segment.realtime.plumber.SegmentHandoffNotifier; import io.druid.segment.realtime.plumber.SegmentHandoffNotifierFactory; import io.druid.timeline.DataSegment; -import io.druid.timeline.TimelineObjectHolder; -import io.druid.timeline.VersionedIntervalTimeline; import io.druid.timeline.partition.NumberedShardSpec; -import io.druid.timeline.partition.PartitionChunk; import org.joda.time.DateTime; import org.junit.After; import org.junit.Assert; @@ -66,7 +61,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -public class AppenderatorDriverTest +public class StreamAppenderatorDriverTest { private static final String DATA_SOURCE = "foo"; private static final String VERSION = "abc123"; @@ -94,10 +89,10 @@ public class AppenderatorDriverTest ) ); - SegmentAllocator allocator; - AppenderatorTester appenderatorTester; - TestSegmentHandoffNotifierFactory segmentHandoffNotifierFactory; - AppenderatorDriver driver; + private SegmentAllocator allocator; + private AppenderatorTester appenderatorTester; + private TestSegmentHandoffNotifierFactory segmentHandoffNotifierFactory; + private StreamAppenderatorDriver driver; @Before public void setUp() @@ -105,11 +100,11 @@ public void setUp() appenderatorTester = new AppenderatorTester(MAX_ROWS_IN_MEMORY); allocator = new TestSegmentAllocator(DATA_SOURCE, Granularities.HOUR); segmentHandoffNotifierFactory = new TestSegmentHandoffNotifierFactory(); - driver = new AppenderatorDriver( + driver = new StreamAppenderatorDriver( appenderatorTester.getAppenderator(), allocator, segmentHandoffNotifierFactory, - new TestUsedSegmentChecker(), + new TestUsedSegmentChecker(appenderatorTester), OBJECT_MAPPER, new FireDepartmentMetrics() ); @@ -131,7 +126,7 @@ public void testSimple() throws Exception for (int i = 0; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); - Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier).isOk()); + Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk()); } final SegmentsAndMetadata published = driver.publish( @@ -177,7 +172,7 @@ public void testMaxRowsPerSegment() throws Exception 2.0 ) ); - final AppenderatorDriverAddResult addResult = driver.add(row, "dummy", committerSupplier); + final AppenderatorDriverAddResult addResult = driver.add(row, "dummy", committerSupplier, false, true); Assert.assertTrue(addResult.isOk()); if (addResult.getNumRowsInSegment() > MAX_ROWS_PER_SEGMENT) { driver.moveSegmentOut("dummy", ImmutableList.of(addResult.getSegmentIdentifier())); @@ -210,7 +205,7 @@ public void testHandoffTimeout() throws Exception for (int i = 0; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); - Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier).isOk()); + Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk()); } final SegmentsAndMetadata published = driver.publish( @@ -236,7 +231,7 @@ public void testPublishPerRow() throws IOException, InterruptedException, Timeou // Add the first row and publish immediately { committerSupplier.setMetadata(1); - Assert.assertTrue(driver.add(ROWS.get(0), "dummy", committerSupplier).isOk()); + Assert.assertTrue(driver.add(ROWS.get(0), "dummy", committerSupplier, false, true).isOk()); final SegmentsAndMetadata segmentsAndMetadata = driver.publishAndRegisterHandoff( makeOkPublisher(), @@ -257,7 +252,7 @@ public void testPublishPerRow() throws IOException, InterruptedException, Timeou // Add the second and third rows and publish immediately for (int i = 1; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); - Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier).isOk()); + Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk()); final SegmentsAndMetadata segmentsAndMetadata = driver.publishAndRegisterHandoff( makeOkPublisher(), @@ -302,11 +297,11 @@ public void testIncrementalHandoff() throws Exception Assert.assertNull(driver.startJob()); committerSupplier.setMetadata(1); - Assert.assertTrue(driver.add(ROWS.get(0), "sequence_0", committerSupplier).isOk()); + Assert.assertTrue(driver.add(ROWS.get(0), "sequence_0", committerSupplier, false, true).isOk()); for (int i = 1; i < ROWS.size(); i++) { committerSupplier.setMetadata(i + 1); - Assert.assertTrue(driver.add(ROWS.get(i), "sequence_1", committerSupplier).isOk()); + Assert.assertTrue(driver.add(ROWS.get(i), "sequence_1", committerSupplier, false, true).isOk()); } final ListenableFuture futureForSequence0 = driver.publishAndRegisterHandoff( @@ -501,33 +496,4 @@ public void close() }; } } - - private class TestUsedSegmentChecker implements UsedSegmentChecker - { - @Override - public Set findUsedSegments(Set identifiers) throws IOException - { - final VersionedIntervalTimeline timeline = new VersionedIntervalTimeline<>(Ordering.natural()); - for (DataSegment dataSegment : appenderatorTester.getPushedSegments()) { - timeline.add( - dataSegment.getInterval(), - dataSegment.getVersion(), - dataSegment.getShardSpec().createChunk(dataSegment) - ); - } - - final Set retVal = Sets.newHashSet(); - for (SegmentIdentifier identifier : identifiers) { - for (TimelineObjectHolder holder : timeline.lookup(identifier.getInterval())) { - for (PartitionChunk chunk : holder.getObject()) { - if (identifiers.contains(SegmentIdentifier.fromDataSegment(chunk.getObject()))) { - retVal.add(chunk.getObject()); - } - } - } - } - - return retVal; - } - } } diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/TestUsedSegmentChecker.java b/server/src/test/java/io/druid/segment/realtime/appenderator/TestUsedSegmentChecker.java new file mode 100644 index 000000000000..e2e7cd620801 --- /dev/null +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/TestUsedSegmentChecker.java @@ -0,0 +1,66 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment.realtime.appenderator; + +import com.google.common.collect.Ordering; +import com.google.common.collect.Sets; +import io.druid.timeline.DataSegment; +import io.druid.timeline.TimelineObjectHolder; +import io.druid.timeline.VersionedIntervalTimeline; +import io.druid.timeline.partition.PartitionChunk; + +import java.io.IOException; +import java.util.Set; + +public class TestUsedSegmentChecker implements UsedSegmentChecker +{ + private final AppenderatorTester appenderatorTester; + + public TestUsedSegmentChecker(AppenderatorTester appenderatorTester) + { + this.appenderatorTester = appenderatorTester; + } + + @Override + public Set findUsedSegments(Set identifiers) throws IOException + { + final VersionedIntervalTimeline timeline = new VersionedIntervalTimeline<>(Ordering.natural()); + for (DataSegment dataSegment : appenderatorTester.getPushedSegments()) { + timeline.add( + dataSegment.getInterval(), + dataSegment.getVersion(), + dataSegment.getShardSpec().createChunk(dataSegment) + ); + } + + final Set retVal = Sets.newHashSet(); + for (SegmentIdentifier identifier : identifiers) { + for (TimelineObjectHolder holder : timeline.lookup(identifier.getInterval())) { + for (PartitionChunk chunk : holder.getObject()) { + if (identifiers.contains(SegmentIdentifier.fromDataSegment(chunk.getObject()))) { + retVal.add(chunk.getObject()); + } + } + } + } + + return retVal; + } +} From 8c46900dacd1230de881a1c7d24a7bc68d3e7ac1 Mon Sep 17 00:00:00 2001 From: jon-wei Date: Fri, 16 Feb 2018 13:51:16 -0800 Subject: [PATCH 30/82] [maven-release-plugin] prepare release druid-0.12.0-rc2 --- api/pom.xml | 2 +- aws-common/pom.xml | 2 +- benchmarks/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- extendedset/pom.xml | 2 +- extensions-contrib/ambari-metrics-emitter/pom.xml | 2 +- extensions-contrib/azure-extensions/pom.xml | 2 +- extensions-contrib/cassandra-storage/pom.xml | 2 +- extensions-contrib/cloudfiles-extensions/pom.xml | 2 +- extensions-contrib/distinctcount/pom.xml | 2 +- extensions-contrib/druid-rocketmq/pom.xml | 2 +- extensions-contrib/google-extensions/pom.xml | 2 +- extensions-contrib/graphite-emitter/pom.xml | 2 +- extensions-contrib/kafka-eight-simpleConsumer/pom.xml | 2 +- extensions-contrib/kafka-emitter/pom.xml | 2 +- extensions-contrib/orc-extensions/pom.xml | 2 +- extensions-contrib/parquet-extensions/pom.xml | 2 +- extensions-contrib/rabbitmq/pom.xml | 2 +- extensions-contrib/redis-cache/pom.xml | 2 +- extensions-contrib/sqlserver-metadata-storage/pom.xml | 2 +- extensions-contrib/statsd-emitter/pom.xml | 2 +- extensions-contrib/thrift-extensions/pom.xml | 2 +- extensions-contrib/time-min-max/pom.xml | 2 +- extensions-contrib/virtual-columns/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 2 +- extensions-core/datasketches/pom.xml | 2 +- extensions-core/druid-basic-security/pom.xml | 2 +- extensions-core/druid-kerberos/pom.xml | 2 +- extensions-core/hdfs-storage/pom.xml | 2 +- extensions-core/histogram/pom.xml | 2 +- extensions-core/kafka-eight/pom.xml | 2 +- extensions-core/kafka-extraction-namespace/pom.xml | 2 +- extensions-core/kafka-indexing-service/pom.xml | 2 +- extensions-core/lookups-cached-global/pom.xml | 2 +- extensions-core/lookups-cached-single/pom.xml | 2 +- extensions-core/mysql-metadata-storage/pom.xml | 2 +- extensions-core/postgresql-metadata-storage/pom.xml | 2 +- extensions-core/protobuf-extensions/pom.xml | 2 +- extensions-core/s3-extensions/pom.xml | 2 +- extensions-core/simple-client-sslcontext/pom.xml | 2 +- extensions-core/stats/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- integration-tests/pom.xml | 2 +- java-util/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- sql/pom.xml | 2 +- 53 files changed, 54 insertions(+), 54 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 1c682410c835..28e4d551e2de 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 diff --git a/aws-common/pom.xml b/aws-common/pom.xml index 93461b80a807..f5b9077f4980 100644 --- a/aws-common/pom.xml +++ b/aws-common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 2c008c6cb96b..f355a56ab429 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 diff --git a/common/pom.xml b/common/pom.xml index fbe104a2f7e4..7230c4983145 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 diff --git a/distribution/pom.xml b/distribution/pom.xml index a4e07724ab93..abe1e0460eed 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -28,7 +28,7 @@ druid io.druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 diff --git a/examples/pom.xml b/examples/pom.xml index 15a23eb80697..9a4afdae2051 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 diff --git a/extendedset/pom.xml b/extendedset/pom.xml index f055966c5380..ee9d5a76ab77 100755 --- a/extendedset/pom.xml +++ b/extendedset/pom.xml @@ -31,7 +31,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index 98e86fcb2491..e6a4c099535b 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-contrib/azure-extensions/pom.xml b/extensions-contrib/azure-extensions/pom.xml index d45421bca1a2..41e028b54920 100644 --- a/extensions-contrib/azure-extensions/pom.xml +++ b/extensions-contrib/azure-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-contrib/cassandra-storage/pom.xml b/extensions-contrib/cassandra-storage/pom.xml index 8f984ab912e7..beac93fe28e2 100644 --- a/extensions-contrib/cassandra-storage/pom.xml +++ b/extensions-contrib/cassandra-storage/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index e12f3a6625fb..ead1958fa3da 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-contrib/distinctcount/pom.xml b/extensions-contrib/distinctcount/pom.xml index 04c15bc285b4..fabb1aa6bf00 100644 --- a/extensions-contrib/distinctcount/pom.xml +++ b/extensions-contrib/distinctcount/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-contrib/druid-rocketmq/pom.xml b/extensions-contrib/druid-rocketmq/pom.xml index faf5ca728bc6..48b5663f8d9f 100644 --- a/extensions-contrib/druid-rocketmq/pom.xml +++ b/extensions-contrib/druid-rocketmq/pom.xml @@ -22,7 +22,7 @@ druid io.druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-contrib/google-extensions/pom.xml b/extensions-contrib/google-extensions/pom.xml index 86fc2a529aea..a3a5bd5eda76 100644 --- a/extensions-contrib/google-extensions/pom.xml +++ b/extensions-contrib/google-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index a2029f1e185b..acee17053641 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml index 1804c8267904..84becf7fb1ab 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml +++ b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index a70ac2a9c6da..6a7ce7dbcd4e 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-contrib/orc-extensions/pom.xml b/extensions-contrib/orc-extensions/pom.xml index cce83a1f00db..bba6b3e60cff 100644 --- a/extensions-contrib/orc-extensions/pom.xml +++ b/extensions-contrib/orc-extensions/pom.xml @@ -24,7 +24,7 @@ druid io.druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/parquet-extensions/pom.xml b/extensions-contrib/parquet-extensions/pom.xml index 22ae9f2fc121..d991ced259d4 100644 --- a/extensions-contrib/parquet-extensions/pom.xml +++ b/extensions-contrib/parquet-extensions/pom.xml @@ -8,7 +8,7 @@ druid io.druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/rabbitmq/pom.xml b/extensions-contrib/rabbitmq/pom.xml index 74980e39ce24..9f4467669fe0 100644 --- a/extensions-contrib/rabbitmq/pom.xml +++ b/extensions-contrib/rabbitmq/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-contrib/redis-cache/pom.xml b/extensions-contrib/redis-cache/pom.xml index 2875c9b79d6f..9aa2a2a34ba4 100644 --- a/extensions-contrib/redis-cache/pom.xml +++ b/extensions-contrib/redis-cache/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-contrib/sqlserver-metadata-storage/pom.xml b/extensions-contrib/sqlserver-metadata-storage/pom.xml index e6abb94e2f9a..f2f7ea74cb5c 100644 --- a/extensions-contrib/sqlserver-metadata-storage/pom.xml +++ b/extensions-contrib/sqlserver-metadata-storage/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index 7badd860cade..731fe56e6d37 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index d76e4f1568bc..ae4b40340019 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -9,7 +9,7 @@ druid io.druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/time-min-max/pom.xml b/extensions-contrib/time-min-max/pom.xml index a91bc4b97b8f..f0ecb5ea13fe 100644 --- a/extensions-contrib/time-min-max/pom.xml +++ b/extensions-contrib/time-min-max/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/virtual-columns/pom.xml b/extensions-contrib/virtual-columns/pom.xml index d2aa4cfe127e..d5457b2ed365 100644 --- a/extensions-contrib/virtual-columns/pom.xml +++ b/extensions-contrib/virtual-columns/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index 20499946fe6e..f96395b9b016 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index 6b3916845b3e..63e4268cd3bb 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-core/druid-basic-security/pom.xml b/extensions-core/druid-basic-security/pom.xml index 7e4b8b4faf99..79481bc1ad5a 100644 --- a/extensions-core/druid-basic-security/pom.xml +++ b/extensions-core/druid-basic-security/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index acb82457ef0f..b1008d5fac8e 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index ccbe4daf484a..d96f033e39d5 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-core/histogram/pom.xml b/extensions-core/histogram/pom.xml index e08cb5d29d11..b2398cd659cd 100644 --- a/extensions-core/histogram/pom.xml +++ b/extensions-core/histogram/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-core/kafka-eight/pom.xml b/extensions-core/kafka-eight/pom.xml index 47ccdd340f13..bd66795495b9 100644 --- a/extensions-core/kafka-eight/pom.xml +++ b/extensions-core/kafka-eight/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index 1ae74d56f254..f46c98469e41 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index 9c0ba11c0a88..65eb0acb20cb 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-core/lookups-cached-global/pom.xml b/extensions-core/lookups-cached-global/pom.xml index 7c75dee17106..346907c859a8 100644 --- a/extensions-core/lookups-cached-global/pom.xml +++ b/extensions-core/lookups-cached-global/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-core/lookups-cached-single/pom.xml b/extensions-core/lookups-cached-single/pom.xml index 28614129f85d..9ff022cca428 100644 --- a/extensions-core/lookups-cached-single/pom.xml +++ b/extensions-core/lookups-cached-single/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index 33ec83e9adfd..9b9bc72558c8 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-core/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml index a8fd60ce036d..708677c81538 100644 --- a/extensions-core/postgresql-metadata-storage/pom.xml +++ b/extensions-core/postgresql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-core/protobuf-extensions/pom.xml b/extensions-core/protobuf-extensions/pom.xml index 85fc52cd6342..87606274cc37 100644 --- a/extensions-core/protobuf-extensions/pom.xml +++ b/extensions-core/protobuf-extensions/pom.xml @@ -27,7 +27,7 @@ druid io.druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index 49ca28462df5..e0b90c3db85b 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/extensions-core/simple-client-sslcontext/pom.xml b/extensions-core/simple-client-sslcontext/pom.xml index 16eed3db9e10..61e2ed5b0aef 100644 --- a/extensions-core/simple-client-sslcontext/pom.xml +++ b/extensions-core/simple-client-sslcontext/pom.xml @@ -3,7 +3,7 @@ druid io.druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml 4.0.0 diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml index d666981362f2..3bc5fd03e05c 100644 --- a/extensions-core/stats/pom.xml +++ b/extensions-core/stats/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 ../../pom.xml diff --git a/hll/pom.xml b/hll/pom.xml index 700775d48295..8cdd7092564a 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 druid-hll diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index e947275bb56f..6f6e0f7f5a9e 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index e5161b7963f3..a51da46693fa 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index a69881914c1a..45f7c27a84f0 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 diff --git a/java-util/pom.xml b/java-util/pom.xml index 71fc76a9c7e9..0dc01e4dd12e 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 java-util diff --git a/pom.xml b/pom.xml index 273441a4c183..06e86fc46ebd 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 pom ${project.groupId}:${project.artifactId} @@ -53,7 +53,7 @@ scm:git:ssh://git@github.com/druid-io/druid.git scm:git:ssh://git@github.com/druid-io/druid.git https://github.com/druid-io/druid.git - 0.12.0-SNAPSHOT + druid-0.12.0-rc2 diff --git a/processing/pom.xml b/processing/pom.xml index 92ddda9e04ec..ba1e3592ff32 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 diff --git a/server/pom.xml b/server/pom.xml index 3a48aab63228..c85b267b28e4 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 diff --git a/services/pom.xml b/services/pom.xml index 62ac7d2e17ec..de9c6af05e45 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -25,7 +25,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 diff --git a/sql/pom.xml b/sql/pom.xml index f1776868e1d6..7730acf48257 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc2-SNAPSHOT + 0.12.0-rc2 From ebb0592948293ce78d93e46b1e0bed62b36cca17 Mon Sep 17 00:00:00 2001 From: jon-wei Date: Fri, 16 Feb 2018 13:51:50 -0800 Subject: [PATCH 31/82] [maven-release-plugin] prepare for next development iteration --- api/pom.xml | 2 +- aws-common/pom.xml | 2 +- benchmarks/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- extendedset/pom.xml | 2 +- extensions-contrib/ambari-metrics-emitter/pom.xml | 2 +- extensions-contrib/azure-extensions/pom.xml | 2 +- extensions-contrib/cassandra-storage/pom.xml | 2 +- extensions-contrib/cloudfiles-extensions/pom.xml | 2 +- extensions-contrib/distinctcount/pom.xml | 2 +- extensions-contrib/druid-rocketmq/pom.xml | 2 +- extensions-contrib/google-extensions/pom.xml | 2 +- extensions-contrib/graphite-emitter/pom.xml | 2 +- extensions-contrib/kafka-eight-simpleConsumer/pom.xml | 2 +- extensions-contrib/kafka-emitter/pom.xml | 2 +- extensions-contrib/orc-extensions/pom.xml | 2 +- extensions-contrib/parquet-extensions/pom.xml | 2 +- extensions-contrib/rabbitmq/pom.xml | 2 +- extensions-contrib/redis-cache/pom.xml | 2 +- extensions-contrib/sqlserver-metadata-storage/pom.xml | 2 +- extensions-contrib/statsd-emitter/pom.xml | 2 +- extensions-contrib/thrift-extensions/pom.xml | 2 +- extensions-contrib/time-min-max/pom.xml | 2 +- extensions-contrib/virtual-columns/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 2 +- extensions-core/datasketches/pom.xml | 2 +- extensions-core/druid-basic-security/pom.xml | 2 +- extensions-core/druid-kerberos/pom.xml | 2 +- extensions-core/hdfs-storage/pom.xml | 2 +- extensions-core/histogram/pom.xml | 2 +- extensions-core/kafka-eight/pom.xml | 2 +- extensions-core/kafka-extraction-namespace/pom.xml | 2 +- extensions-core/kafka-indexing-service/pom.xml | 2 +- extensions-core/lookups-cached-global/pom.xml | 2 +- extensions-core/lookups-cached-single/pom.xml | 2 +- extensions-core/mysql-metadata-storage/pom.xml | 2 +- extensions-core/postgresql-metadata-storage/pom.xml | 2 +- extensions-core/protobuf-extensions/pom.xml | 2 +- extensions-core/s3-extensions/pom.xml | 2 +- extensions-core/simple-client-sslcontext/pom.xml | 2 +- extensions-core/stats/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- integration-tests/pom.xml | 2 +- java-util/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- sql/pom.xml | 2 +- 53 files changed, 54 insertions(+), 54 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 28e4d551e2de..2cf586b00371 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT diff --git a/aws-common/pom.xml b/aws-common/pom.xml index f5b9077f4980..ab010106cc7c 100644 --- a/aws-common/pom.xml +++ b/aws-common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index f355a56ab429..527610703a40 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 7230c4983145..af218d57caa2 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT diff --git a/distribution/pom.xml b/distribution/pom.xml index abe1e0460eed..cc6b8e98b568 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -28,7 +28,7 @@ druid io.druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 9a4afdae2051..77a1f215b676 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT diff --git a/extendedset/pom.xml b/extendedset/pom.xml index ee9d5a76ab77..7433a4447d3f 100755 --- a/extendedset/pom.xml +++ b/extendedset/pom.xml @@ -31,7 +31,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index e6a4c099535b..94e602b308d3 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/azure-extensions/pom.xml b/extensions-contrib/azure-extensions/pom.xml index 41e028b54920..b7bda794a19e 100644 --- a/extensions-contrib/azure-extensions/pom.xml +++ b/extensions-contrib/azure-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/cassandra-storage/pom.xml b/extensions-contrib/cassandra-storage/pom.xml index beac93fe28e2..1e0bcaf84504 100644 --- a/extensions-contrib/cassandra-storage/pom.xml +++ b/extensions-contrib/cassandra-storage/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index ead1958fa3da..58673b9ba120 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/distinctcount/pom.xml b/extensions-contrib/distinctcount/pom.xml index fabb1aa6bf00..19131347bffd 100644 --- a/extensions-contrib/distinctcount/pom.xml +++ b/extensions-contrib/distinctcount/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/druid-rocketmq/pom.xml b/extensions-contrib/druid-rocketmq/pom.xml index 48b5663f8d9f..7f639861eb73 100644 --- a/extensions-contrib/druid-rocketmq/pom.xml +++ b/extensions-contrib/druid-rocketmq/pom.xml @@ -22,7 +22,7 @@ druid io.druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/google-extensions/pom.xml b/extensions-contrib/google-extensions/pom.xml index a3a5bd5eda76..6fb09057ec27 100644 --- a/extensions-contrib/google-extensions/pom.xml +++ b/extensions-contrib/google-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index acee17053641..4234ba25bded 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml index 84becf7fb1ab..0e9c0b883d71 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml +++ b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index 6a7ce7dbcd4e..6a5aef66763f 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/orc-extensions/pom.xml b/extensions-contrib/orc-extensions/pom.xml index bba6b3e60cff..ba0489dd61c4 100644 --- a/extensions-contrib/orc-extensions/pom.xml +++ b/extensions-contrib/orc-extensions/pom.xml @@ -24,7 +24,7 @@ druid io.druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/parquet-extensions/pom.xml b/extensions-contrib/parquet-extensions/pom.xml index d991ced259d4..a0c01c1ba01e 100644 --- a/extensions-contrib/parquet-extensions/pom.xml +++ b/extensions-contrib/parquet-extensions/pom.xml @@ -8,7 +8,7 @@ druid io.druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/rabbitmq/pom.xml b/extensions-contrib/rabbitmq/pom.xml index 9f4467669fe0..4bac9fe31fef 100644 --- a/extensions-contrib/rabbitmq/pom.xml +++ b/extensions-contrib/rabbitmq/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/redis-cache/pom.xml b/extensions-contrib/redis-cache/pom.xml index 9aa2a2a34ba4..ab894f852593 100644 --- a/extensions-contrib/redis-cache/pom.xml +++ b/extensions-contrib/redis-cache/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/sqlserver-metadata-storage/pom.xml b/extensions-contrib/sqlserver-metadata-storage/pom.xml index f2f7ea74cb5c..74928bca6cff 100644 --- a/extensions-contrib/sqlserver-metadata-storage/pom.xml +++ b/extensions-contrib/sqlserver-metadata-storage/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index 731fe56e6d37..8adc85219edc 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index ae4b40340019..f7ba36093556 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -9,7 +9,7 @@ druid io.druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/time-min-max/pom.xml b/extensions-contrib/time-min-max/pom.xml index f0ecb5ea13fe..ab3400e6513b 100644 --- a/extensions-contrib/time-min-max/pom.xml +++ b/extensions-contrib/time-min-max/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/virtual-columns/pom.xml b/extensions-contrib/virtual-columns/pom.xml index d5457b2ed365..636b1ed4c55c 100644 --- a/extensions-contrib/virtual-columns/pom.xml +++ b/extensions-contrib/virtual-columns/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index f96395b9b016..e5420b9a14e0 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index 63e4268cd3bb..7661fddb8b8c 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-basic-security/pom.xml b/extensions-core/druid-basic-security/pom.xml index 79481bc1ad5a..b077a9036082 100644 --- a/extensions-core/druid-basic-security/pom.xml +++ b/extensions-core/druid-basic-security/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index b1008d5fac8e..ed449ac35888 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index d96f033e39d5..69d4a444c8f4 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/histogram/pom.xml b/extensions-core/histogram/pom.xml index b2398cd659cd..edcf0ba53ff6 100644 --- a/extensions-core/histogram/pom.xml +++ b/extensions-core/histogram/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-eight/pom.xml b/extensions-core/kafka-eight/pom.xml index bd66795495b9..276430945370 100644 --- a/extensions-core/kafka-eight/pom.xml +++ b/extensions-core/kafka-eight/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index f46c98469e41..595a07d0b428 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index 65eb0acb20cb..3c5446606783 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/lookups-cached-global/pom.xml b/extensions-core/lookups-cached-global/pom.xml index 346907c859a8..0376aaac2d23 100644 --- a/extensions-core/lookups-cached-global/pom.xml +++ b/extensions-core/lookups-cached-global/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/lookups-cached-single/pom.xml b/extensions-core/lookups-cached-single/pom.xml index 9ff022cca428..08681cf6fa05 100644 --- a/extensions-core/lookups-cached-single/pom.xml +++ b/extensions-core/lookups-cached-single/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index 9b9bc72558c8..760cd6ea701e 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml index 708677c81538..20c938d256f0 100644 --- a/extensions-core/postgresql-metadata-storage/pom.xml +++ b/extensions-core/postgresql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/protobuf-extensions/pom.xml b/extensions-core/protobuf-extensions/pom.xml index 87606274cc37..77f8212d4274 100644 --- a/extensions-core/protobuf-extensions/pom.xml +++ b/extensions-core/protobuf-extensions/pom.xml @@ -27,7 +27,7 @@ druid io.druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index e0b90c3db85b..4bd13f050fca 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/simple-client-sslcontext/pom.xml b/extensions-core/simple-client-sslcontext/pom.xml index 61e2ed5b0aef..5fa2168b8fc8 100644 --- a/extensions-core/simple-client-sslcontext/pom.xml +++ b/extensions-core/simple-client-sslcontext/pom.xml @@ -3,7 +3,7 @@ druid io.druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml index 3bc5fd03e05c..70ec638e82df 100644 --- a/extensions-core/stats/pom.xml +++ b/extensions-core/stats/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT ../../pom.xml diff --git a/hll/pom.xml b/hll/pom.xml index 8cdd7092564a..62d74159d78b 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT druid-hll diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 6f6e0f7f5a9e..b3261aca6d25 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index a51da46693fa..56954b1b29be 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 45f7c27a84f0..42241c3943c3 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT diff --git a/java-util/pom.xml b/java-util/pom.xml index 0dc01e4dd12e..3902705ac2e3 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT java-util diff --git a/pom.xml b/pom.xml index 06e86fc46ebd..19eacf381b4f 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT pom ${project.groupId}:${project.artifactId} @@ -53,7 +53,7 @@ scm:git:ssh://git@github.com/druid-io/druid.git scm:git:ssh://git@github.com/druid-io/druid.git https://github.com/druid-io/druid.git - druid-0.12.0-rc2 + 0.12.0-SNAPSHOT diff --git a/processing/pom.xml b/processing/pom.xml index ba1e3592ff32..0e0adae75efc 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index c85b267b28e4..d4658e1cb262 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index de9c6af05e45..6a0df8cdf377 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -25,7 +25,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT diff --git a/sql/pom.xml b/sql/pom.xml index 7730acf48257..1c727bc1c6eb 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc2 + 0.12.0-rc3-SNAPSHOT From db31e2a0748f2a394c8001c228327b5ef0098eaa Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Fri, 16 Feb 2018 17:00:49 -0800 Subject: [PATCH 32/82] [Backport] Fix DefaultLimitSpec to respect sortByDimsFirst (#5398) * Fix DefaultLimitSpec to respect sortByDimsFirst * fix build --- .../io/druid/query/groupby/GroupByQuery.java | 9 +- .../groupby/orderby/DefaultLimitSpec.java | 35 +++++--- .../query/groupby/orderby/LimitSpec.java | 13 ++- .../query/groupby/orderby/NoopLimitSpec.java | 5 +- .../query/groupby/GroupByQueryRunnerTest.java | 82 +++++++++++++++++-- .../groupby/orderby/DefaultLimitSpecTest.java | 70 ++++++++++++++-- 6 files changed, 186 insertions(+), 28 deletions(-) diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java index 9043ad6a1e12..ea48985cedb7 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java @@ -136,8 +136,13 @@ public GroupByQuery( private Function, Sequence> makePostProcessingFn() { - Function, Sequence> postProcessingFn = - limitSpec.build(dimensions, aggregatorSpecs, postAggregatorSpecs); + Function, Sequence> postProcessingFn = limitSpec.build( + dimensions, + aggregatorSpecs, + postAggregatorSpecs, + getGranularity(), + getContextSortByDimsFirst() + ); if (havingSpec != null) { postProcessingFn = Functions.compose( diff --git a/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java b/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java index cc3d9e6014b4..f3c31ae8325a 100644 --- a/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java @@ -32,6 +32,8 @@ import com.google.common.primitives.Longs; import io.druid.data.input.Row; import io.druid.java.util.common.ISE; +import io.druid.java.util.common.granularity.Granularities; +import io.druid.java.util.common.granularity.Granularity; import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; import io.druid.query.aggregation.AggregatorFactory; @@ -119,16 +121,14 @@ public boolean isLimited() public Function, Sequence> build( List dimensions, List aggs, - List postAggs + List postAggs, + Granularity granularity, + boolean sortByDimsFirst ) { // Can avoid re-sorting if the natural ordering is good enough. - boolean sortingNeeded = false; - - if (dimensions.size() < columns.size()) { - sortingNeeded = true; - } + boolean sortingNeeded = dimensions.size() < columns.size(); final Set aggAndPostAggNames = Sets.newHashSet(); for (AggregatorFactory agg : aggs) { @@ -167,12 +167,17 @@ public Function, Sequence> build( } } + if (!sortingNeeded) { + // If granularity is ALL, sortByDimsFirst doesn't change the sorting order. + sortingNeeded = !granularity.equals(Granularities.ALL) && sortByDimsFirst; + } + if (!sortingNeeded) { return isLimited() ? new LimitingFn(limit) : Functions.identity(); } // Materialize the Comparator first for fast-fail error checking. - final Ordering ordering = makeComparator(dimensions, aggs, postAggs); + final Ordering ordering = makeComparator(dimensions, aggs, postAggs, sortByDimsFirst); if (isLimited()) { return new TopNFunction(ordering, limit); @@ -199,10 +204,13 @@ private ValueType getOrderByType(final OrderByColumnSpec columnSpec, final List< } private Ordering makeComparator( - List dimensions, List aggs, List postAggs + List dimensions, + List aggs, + List postAggs, + boolean sortByDimsFirst ) { - Ordering ordering = new Ordering() + Ordering timeOrdering = new Ordering() { @Override public int compare(Row left, Row right) @@ -226,6 +234,7 @@ public int compare(Row left, Row right) postAggregatorsMap.put(postAgg.getName(), postAgg); } + Ordering ordering = null; for (OrderByColumnSpec columnSpec : columns) { String columnName = columnSpec.getDimension(); Ordering nextOrdering = null; @@ -246,7 +255,13 @@ public int compare(Row left, Row right) nextOrdering = nextOrdering.reverse(); } - ordering = ordering.compound(nextOrdering); + ordering = ordering == null ? nextOrdering : ordering.compound(nextOrdering); + } + + if (ordering != null) { + ordering = sortByDimsFirst ? ordering.compound(timeOrdering) : timeOrdering.compound(ordering); + } else { + ordering = timeOrdering; } return ordering; diff --git a/processing/src/main/java/io/druid/query/groupby/orderby/LimitSpec.java b/processing/src/main/java/io/druid/query/groupby/orderby/LimitSpec.java index 4638e6a49dc4..2e0e489ce1e7 100644 --- a/processing/src/main/java/io/druid/query/groupby/orderby/LimitSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/orderby/LimitSpec.java @@ -24,6 +24,7 @@ import com.google.common.base.Function; import io.druid.data.input.Row; import io.druid.java.util.common.Cacheable; +import io.druid.java.util.common.granularity.Granularity; import io.druid.java.util.common.guava.Sequence; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; @@ -48,16 +49,20 @@ static LimitSpec nullToNoopLimitSpec(@Nullable LimitSpec limitSpec) /** * Returns a function that applies a limit to an input sequence that is assumed to be sorted on dimensions. * - * @param dimensions query dimensions - * @param aggs query aggregators - * @param postAggs query postAggregators + * @param dimensions query dimensions + * @param aggs query aggregators + * @param postAggs query postAggregators + * @param granularity query granularity + * @param sortByDimsFirst 'sortByDimsFirst' value in queryContext * * @return limit function */ Function, Sequence> build( List dimensions, List aggs, - List postAggs + List postAggs, + Granularity granularity, + boolean sortByDimsFirst ); LimitSpec merge(LimitSpec other); diff --git a/processing/src/main/java/io/druid/query/groupby/orderby/NoopLimitSpec.java b/processing/src/main/java/io/druid/query/groupby/orderby/NoopLimitSpec.java index 2ba458a6755d..835c1d0a006c 100644 --- a/processing/src/main/java/io/druid/query/groupby/orderby/NoopLimitSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/orderby/NoopLimitSpec.java @@ -23,6 +23,7 @@ import com.google.common.base.Function; import com.google.common.base.Functions; import io.druid.data.input.Row; +import io.druid.java.util.common.granularity.Granularity; import io.druid.java.util.common.guava.Sequence; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.PostAggregator; @@ -52,7 +53,9 @@ private NoopLimitSpec() public Function, Sequence> build( List dimensions, List aggs, - List postAggs + List postAggs, + Granularity granularity, + boolean sortByDimsFirst ) { return Functions.identity(); diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index ce586d576c3e..906f1095d69e 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -3533,6 +3533,68 @@ public void testGroupByWithAlphaNumericDimensionOrder() TestHelper.assertExpectedObjects(expectedResults, results, ""); } + @Test + public void testGroupByWithLookupAndLimitAndSortByDimsFirst() + { + Map map = new HashMap<>(); + map.put("automotive", "9"); + map.put("business", "8"); + map.put("entertainment", "7"); + map.put("health", "6"); + map.put("mezzanine", "5"); + map.put("news", "4"); + map.put("premium", "3"); + map.put("technology", "2"); + map.put("travel", "1"); + + GroupByQuery query = GroupByQuery + .builder() + .setDataSource(QueryRunnerTestHelper.dataSource) + .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) + .setDimensions( + Lists.newArrayList( + new ExtractionDimensionSpec( + "quality", + "alias", + new LookupExtractionFn(new MapLookupExtractor(map, false), false, null, false, false) + ) + ) + ) + .setAggregatorSpecs( + Arrays.asList( + QueryRunnerTestHelper.rowsCount, + new LongSumAggregatorFactory("idx", "index") + ) + ) + .setLimitSpec(new DefaultLimitSpec(Lists.newArrayList( + new OrderByColumnSpec("alias", null, StringComparators.ALPHANUMERIC)), 11)) + .setGranularity(QueryRunnerTestHelper.dayGran) + .setContext(ImmutableMap.of("sortByDimsFirst", true)) + .build(); + + List expectedResults = Arrays.asList( + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "1", "rows", 1L, "idx", 119L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "1", "rows", 1L, "idx", 126L), + + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "2", "rows", 1L, "idx", 78L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "2", "rows", 1L, "idx", 97L), + + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "3", "rows", 3L, "idx", 2900L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "3", "rows", 3L, "idx", 2505L), + + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "4", "rows", 1L, "idx", 121L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "4", "rows", 1L, "idx", 114L), + + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "5", "rows", 3L, "idx", 2870L), + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "5", "rows", 3L, "idx", 2447L), + + GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "6", "rows", 1L, "idx", 120L) + ); + + Iterable results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query); + TestHelper.assertExpectedObjects(expectedResults, results, ""); + } + @Ignore @Test // This is a test to verify per limit groupings, but Druid currently does not support this functionality. At a point @@ -7438,7 +7500,9 @@ public void testGroupByLongColumn() query.getLimitSpec().build( query.getDimensions(), query.getAggregatorSpecs(), - query.getPostAggregatorSpecs() + query.getPostAggregatorSpecs(), + query.getGranularity(), + query.getContextSortByDimsFirst() ) ); @@ -7499,7 +7563,9 @@ public void testGroupByLongColumnDescending() query.getLimitSpec().build( query.getDimensions(), query.getAggregatorSpecs(), - query.getPostAggregatorSpecs() + query.getPostAggregatorSpecs(), + query.getGranularity(), + query.getContextSortByDimsFirst() ) ); @@ -7702,7 +7768,9 @@ public void testGroupByFloatColumn() query.getLimitSpec().build( query.getDimensions(), query.getAggregatorSpecs(), - query.getPostAggregatorSpecs() + query.getPostAggregatorSpecs(), + query.getGranularity(), + query.getContextSortByDimsFirst() ) ); @@ -7764,7 +7832,9 @@ public void testGroupByFloatColumnDescending() query.getLimitSpec().build( query.getDimensions(), query.getAggregatorSpecs(), - query.getPostAggregatorSpecs() + query.getPostAggregatorSpecs(), + query.getGranularity(), + query.getContextSortByDimsFirst() ) ); @@ -7825,7 +7895,9 @@ public void testGroupByDoubleColumnDescending() query.getLimitSpec().build( query.getDimensions(), query.getAggregatorSpecs(), - query.getPostAggregatorSpecs() + query.getPostAggregatorSpecs(), + query.getGranularity(), + query.getContextSortByDimsFirst() ) ); diff --git a/processing/src/test/java/io/druid/query/groupby/orderby/DefaultLimitSpecTest.java b/processing/src/test/java/io/druid/query/groupby/orderby/DefaultLimitSpecTest.java index c4bbebfe9c27..cd442acfddb1 100644 --- a/processing/src/test/java/io/druid/query/groupby/orderby/DefaultLimitSpecTest.java +++ b/processing/src/test/java/io/druid/query/groupby/orderby/DefaultLimitSpecTest.java @@ -27,6 +27,7 @@ import io.druid.data.input.MapBasedRow; import io.druid.data.input.Row; import io.druid.java.util.common.DateTimes; +import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; import io.druid.query.aggregation.AggregatorFactory; @@ -40,6 +41,7 @@ import io.druid.query.expression.TestExprMacroTable; import io.druid.query.ordering.StringComparators; import io.druid.segment.TestHelper; +import io.druid.segment.column.ValueType; import org.junit.Assert; import org.junit.Test; @@ -160,7 +162,9 @@ public void testBuildSimple() Function, Sequence> limitFn = limitSpec.build( ImmutableList.of(), ImmutableList.of(), - ImmutableList.of() + ImmutableList.of(), + Granularities.NONE, + false ); Assert.assertEquals( @@ -169,6 +173,50 @@ public void testBuildSimple() ); } + @Test + public void testWithAllGranularity() + { + DefaultLimitSpec limitSpec = new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec("k1", OrderByColumnSpec.Direction.ASCENDING, StringComparators.NUMERIC)), + 2 + ); + + Function, Sequence> limitFn = limitSpec.build( + ImmutableList.of(new DefaultDimensionSpec("k1", "k1", ValueType.DOUBLE)), + ImmutableList.of(), + ImmutableList.of(), + Granularities.ALL, + true + ); + + Assert.assertEquals( + ImmutableList.of(testRowsList.get(0), testRowsList.get(1)), + Sequences.toList(limitFn.apply(testRowsSequence), new ArrayList<>()) + ); + } + + @Test + public void testWithSortByDimsFirst() + { + DefaultLimitSpec limitSpec = new DefaultLimitSpec( + ImmutableList.of(new OrderByColumnSpec("k1", OrderByColumnSpec.Direction.ASCENDING, StringComparators.NUMERIC)), + 2 + ); + + Function, Sequence> limitFn = limitSpec.build( + ImmutableList.of(new DefaultDimensionSpec("k1", "k1", ValueType.DOUBLE)), + ImmutableList.of(), + ImmutableList.of(), + Granularities.NONE, + true + ); + + Assert.assertEquals( + ImmutableList.of(testRowsList.get(2), testRowsList.get(0)), + Sequences.toList(limitFn.apply(testRowsSequence), new ArrayList<>()) + ); + } + @Test public void testSortDimensionDescending() { @@ -180,7 +228,9 @@ public void testSortDimensionDescending() Function, Sequence> limitFn = limitSpec.build( ImmutableList.of(new DefaultDimensionSpec("k1", "k1")), ImmutableList.of(), - ImmutableList.of() + ImmutableList.of(), + Granularities.NONE, + false ); // Note: This test encodes the fact that limitSpec sorts numbers like strings; we might want to change this @@ -210,7 +260,9 @@ public void testBuildWithExplicitOrder() ), ImmutableList.of( new ConstantPostAggregator("k3", 1L) - ) + ), + Granularities.NONE, + false ); Assert.assertEquals( ImmutableList.of(testRowsList.get(0), testRowsList.get(1)), @@ -227,7 +279,9 @@ public void testBuildWithExplicitOrder() ), ImmutableList.of( new ConstantPostAggregator("k3", 1L) - ) + ), + Granularities.NONE, + false ); Assert.assertEquals( ImmutableList.of(testRowsList.get(2), testRowsList.get(0)), @@ -250,7 +304,9 @@ public void testBuildWithExplicitOrder() new ConstantPostAggregator("x", 1), new ConstantPostAggregator("y", 1)) ) - ) + ), + Granularities.NONE, + false ); Assert.assertEquals( (List) ImmutableList.of(testRowsList.get(2), testRowsList.get(0)), @@ -261,7 +317,9 @@ public void testBuildWithExplicitOrder() limitFn = limitSpec.build( ImmutableList.of(new DefaultDimensionSpec("k1", "k1")), ImmutableList.of(new LongSumAggregatorFactory("k2", "k2")), - ImmutableList.of(new ExpressionPostAggregator("k1", "1 + 1", null, TestExprMacroTable.INSTANCE)) + ImmutableList.of(new ExpressionPostAggregator("k1", "1 + 1", null, TestExprMacroTable.INSTANCE)), + Granularities.NONE, + false ); Assert.assertEquals( (List) ImmutableList.of(testRowsList.get(2), testRowsList.get(0)), From 3e849796afae00231c114d883a4a0b7d53a71a4d Mon Sep 17 00:00:00 2001 From: jon-wei Date: Fri, 16 Feb 2018 18:11:08 -0800 Subject: [PATCH 33/82] [maven-release-plugin] prepare release druid-0.12.0-rc3 --- api/pom.xml | 2 +- aws-common/pom.xml | 2 +- benchmarks/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- extendedset/pom.xml | 2 +- extensions-contrib/ambari-metrics-emitter/pom.xml | 2 +- extensions-contrib/azure-extensions/pom.xml | 2 +- extensions-contrib/cassandra-storage/pom.xml | 2 +- extensions-contrib/cloudfiles-extensions/pom.xml | 2 +- extensions-contrib/distinctcount/pom.xml | 2 +- extensions-contrib/druid-rocketmq/pom.xml | 2 +- extensions-contrib/google-extensions/pom.xml | 2 +- extensions-contrib/graphite-emitter/pom.xml | 2 +- extensions-contrib/kafka-eight-simpleConsumer/pom.xml | 2 +- extensions-contrib/kafka-emitter/pom.xml | 2 +- extensions-contrib/orc-extensions/pom.xml | 2 +- extensions-contrib/parquet-extensions/pom.xml | 2 +- extensions-contrib/rabbitmq/pom.xml | 2 +- extensions-contrib/redis-cache/pom.xml | 2 +- extensions-contrib/sqlserver-metadata-storage/pom.xml | 2 +- extensions-contrib/statsd-emitter/pom.xml | 2 +- extensions-contrib/thrift-extensions/pom.xml | 2 +- extensions-contrib/time-min-max/pom.xml | 2 +- extensions-contrib/virtual-columns/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 2 +- extensions-core/datasketches/pom.xml | 2 +- extensions-core/druid-basic-security/pom.xml | 2 +- extensions-core/druid-kerberos/pom.xml | 2 +- extensions-core/hdfs-storage/pom.xml | 2 +- extensions-core/histogram/pom.xml | 2 +- extensions-core/kafka-eight/pom.xml | 2 +- extensions-core/kafka-extraction-namespace/pom.xml | 2 +- extensions-core/kafka-indexing-service/pom.xml | 2 +- extensions-core/lookups-cached-global/pom.xml | 2 +- extensions-core/lookups-cached-single/pom.xml | 2 +- extensions-core/mysql-metadata-storage/pom.xml | 2 +- extensions-core/postgresql-metadata-storage/pom.xml | 2 +- extensions-core/protobuf-extensions/pom.xml | 2 +- extensions-core/s3-extensions/pom.xml | 2 +- extensions-core/simple-client-sslcontext/pom.xml | 2 +- extensions-core/stats/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- integration-tests/pom.xml | 2 +- java-util/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- sql/pom.xml | 2 +- 53 files changed, 54 insertions(+), 54 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 2cf586b00371..d39d6252bbd0 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 diff --git a/aws-common/pom.xml b/aws-common/pom.xml index ab010106cc7c..d50a415ba13d 100644 --- a/aws-common/pom.xml +++ b/aws-common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 527610703a40..6ad9033eaa60 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 diff --git a/common/pom.xml b/common/pom.xml index af218d57caa2..e995d70ec91f 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 diff --git a/distribution/pom.xml b/distribution/pom.xml index cc6b8e98b568..d70e600a9412 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -28,7 +28,7 @@ druid io.druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 diff --git a/examples/pom.xml b/examples/pom.xml index 77a1f215b676..2881ecd6d169 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 diff --git a/extendedset/pom.xml b/extendedset/pom.xml index 7433a4447d3f..08ca138e34b5 100755 --- a/extendedset/pom.xml +++ b/extendedset/pom.xml @@ -31,7 +31,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index 94e602b308d3..8b78e6ee625c 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-contrib/azure-extensions/pom.xml b/extensions-contrib/azure-extensions/pom.xml index b7bda794a19e..dae8229a31e1 100644 --- a/extensions-contrib/azure-extensions/pom.xml +++ b/extensions-contrib/azure-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-contrib/cassandra-storage/pom.xml b/extensions-contrib/cassandra-storage/pom.xml index 1e0bcaf84504..31b25595f282 100644 --- a/extensions-contrib/cassandra-storage/pom.xml +++ b/extensions-contrib/cassandra-storage/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index 58673b9ba120..c8c23fad5e0d 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-contrib/distinctcount/pom.xml b/extensions-contrib/distinctcount/pom.xml index 19131347bffd..e1ce0db82aab 100644 --- a/extensions-contrib/distinctcount/pom.xml +++ b/extensions-contrib/distinctcount/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-contrib/druid-rocketmq/pom.xml b/extensions-contrib/druid-rocketmq/pom.xml index 7f639861eb73..4d24afbc20c3 100644 --- a/extensions-contrib/druid-rocketmq/pom.xml +++ b/extensions-contrib/druid-rocketmq/pom.xml @@ -22,7 +22,7 @@ druid io.druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-contrib/google-extensions/pom.xml b/extensions-contrib/google-extensions/pom.xml index 6fb09057ec27..65ca4d6020f4 100644 --- a/extensions-contrib/google-extensions/pom.xml +++ b/extensions-contrib/google-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index 4234ba25bded..328564c0c1b8 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml index 0e9c0b883d71..34b821b5688a 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml +++ b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index 6a5aef66763f..227381e1c9fd 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-contrib/orc-extensions/pom.xml b/extensions-contrib/orc-extensions/pom.xml index ba0489dd61c4..47fdb9b05d31 100644 --- a/extensions-contrib/orc-extensions/pom.xml +++ b/extensions-contrib/orc-extensions/pom.xml @@ -24,7 +24,7 @@ druid io.druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/parquet-extensions/pom.xml b/extensions-contrib/parquet-extensions/pom.xml index a0c01c1ba01e..7c5b996e6053 100644 --- a/extensions-contrib/parquet-extensions/pom.xml +++ b/extensions-contrib/parquet-extensions/pom.xml @@ -8,7 +8,7 @@ druid io.druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/rabbitmq/pom.xml b/extensions-contrib/rabbitmq/pom.xml index 4bac9fe31fef..f5bfcdb8c53e 100644 --- a/extensions-contrib/rabbitmq/pom.xml +++ b/extensions-contrib/rabbitmq/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-contrib/redis-cache/pom.xml b/extensions-contrib/redis-cache/pom.xml index ab894f852593..f8d0fcc61003 100644 --- a/extensions-contrib/redis-cache/pom.xml +++ b/extensions-contrib/redis-cache/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-contrib/sqlserver-metadata-storage/pom.xml b/extensions-contrib/sqlserver-metadata-storage/pom.xml index 74928bca6cff..babd86f0b38b 100644 --- a/extensions-contrib/sqlserver-metadata-storage/pom.xml +++ b/extensions-contrib/sqlserver-metadata-storage/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index 8adc85219edc..ed64e68b37f0 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index f7ba36093556..97b5f1455e1c 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -9,7 +9,7 @@ druid io.druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/time-min-max/pom.xml b/extensions-contrib/time-min-max/pom.xml index ab3400e6513b..8a670a6a952f 100644 --- a/extensions-contrib/time-min-max/pom.xml +++ b/extensions-contrib/time-min-max/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/virtual-columns/pom.xml b/extensions-contrib/virtual-columns/pom.xml index 636b1ed4c55c..07e4793063e1 100644 --- a/extensions-contrib/virtual-columns/pom.xml +++ b/extensions-contrib/virtual-columns/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index e5420b9a14e0..ebe8f25506c8 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index 7661fddb8b8c..16a86841bb9c 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-core/druid-basic-security/pom.xml b/extensions-core/druid-basic-security/pom.xml index b077a9036082..0cf293e51fe0 100644 --- a/extensions-core/druid-basic-security/pom.xml +++ b/extensions-core/druid-basic-security/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index ed449ac35888..f2ce8ad6da6b 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index 69d4a444c8f4..c6289ee663ec 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-core/histogram/pom.xml b/extensions-core/histogram/pom.xml index edcf0ba53ff6..76a4960aa52d 100644 --- a/extensions-core/histogram/pom.xml +++ b/extensions-core/histogram/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-core/kafka-eight/pom.xml b/extensions-core/kafka-eight/pom.xml index 276430945370..bf50e0a0c62d 100644 --- a/extensions-core/kafka-eight/pom.xml +++ b/extensions-core/kafka-eight/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index 595a07d0b428..2527b07461fb 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index 3c5446606783..c2bc2a524ce4 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-core/lookups-cached-global/pom.xml b/extensions-core/lookups-cached-global/pom.xml index 0376aaac2d23..bbd31280e87c 100644 --- a/extensions-core/lookups-cached-global/pom.xml +++ b/extensions-core/lookups-cached-global/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-core/lookups-cached-single/pom.xml b/extensions-core/lookups-cached-single/pom.xml index 08681cf6fa05..5d8ac2acb684 100644 --- a/extensions-core/lookups-cached-single/pom.xml +++ b/extensions-core/lookups-cached-single/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index 760cd6ea701e..5c6628016d91 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-core/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml index 20c938d256f0..7e3946bde2d7 100644 --- a/extensions-core/postgresql-metadata-storage/pom.xml +++ b/extensions-core/postgresql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-core/protobuf-extensions/pom.xml b/extensions-core/protobuf-extensions/pom.xml index 77f8212d4274..3c4dbdbc8351 100644 --- a/extensions-core/protobuf-extensions/pom.xml +++ b/extensions-core/protobuf-extensions/pom.xml @@ -27,7 +27,7 @@ druid io.druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index 4bd13f050fca..ae315baa1bc4 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/extensions-core/simple-client-sslcontext/pom.xml b/extensions-core/simple-client-sslcontext/pom.xml index 5fa2168b8fc8..fd8f0afd19ab 100644 --- a/extensions-core/simple-client-sslcontext/pom.xml +++ b/extensions-core/simple-client-sslcontext/pom.xml @@ -3,7 +3,7 @@ druid io.druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml 4.0.0 diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml index 70ec638e82df..c3180f285671 100644 --- a/extensions-core/stats/pom.xml +++ b/extensions-core/stats/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 ../../pom.xml diff --git a/hll/pom.xml b/hll/pom.xml index 62d74159d78b..a14c3f6826a2 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 druid-hll diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index b3261aca6d25..c932f230705d 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 56954b1b29be..1c2a341f38b9 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 42241c3943c3..ed0603fe977d 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 diff --git a/java-util/pom.xml b/java-util/pom.xml index 3902705ac2e3..a6ad6c177403 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 java-util diff --git a/pom.xml b/pom.xml index 19eacf381b4f..0528cc0c8f57 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 pom ${project.groupId}:${project.artifactId} @@ -53,7 +53,7 @@ scm:git:ssh://git@github.com/druid-io/druid.git scm:git:ssh://git@github.com/druid-io/druid.git https://github.com/druid-io/druid.git - 0.12.0-SNAPSHOT + druid-0.12.0-rc3 diff --git a/processing/pom.xml b/processing/pom.xml index 0e0adae75efc..ac09f3f21592 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 diff --git a/server/pom.xml b/server/pom.xml index d4658e1cb262..69ccbe905c69 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 diff --git a/services/pom.xml b/services/pom.xml index 6a0df8cdf377..e5d83bc1805a 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -25,7 +25,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 diff --git a/sql/pom.xml b/sql/pom.xml index 1c727bc1c6eb..014cdae72dc8 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc3-SNAPSHOT + 0.12.0-rc3 From ce8220cf88f6b6c2a71b71c14f9db19368e9299f Mon Sep 17 00:00:00 2001 From: jon-wei Date: Fri, 16 Feb 2018 18:11:21 -0800 Subject: [PATCH 34/82] [maven-release-plugin] prepare for next development iteration --- api/pom.xml | 2 +- aws-common/pom.xml | 2 +- benchmarks/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- extendedset/pom.xml | 2 +- extensions-contrib/ambari-metrics-emitter/pom.xml | 2 +- extensions-contrib/azure-extensions/pom.xml | 2 +- extensions-contrib/cassandra-storage/pom.xml | 2 +- extensions-contrib/cloudfiles-extensions/pom.xml | 2 +- extensions-contrib/distinctcount/pom.xml | 2 +- extensions-contrib/druid-rocketmq/pom.xml | 2 +- extensions-contrib/google-extensions/pom.xml | 2 +- extensions-contrib/graphite-emitter/pom.xml | 2 +- extensions-contrib/kafka-eight-simpleConsumer/pom.xml | 2 +- extensions-contrib/kafka-emitter/pom.xml | 2 +- extensions-contrib/orc-extensions/pom.xml | 2 +- extensions-contrib/parquet-extensions/pom.xml | 2 +- extensions-contrib/rabbitmq/pom.xml | 2 +- extensions-contrib/redis-cache/pom.xml | 2 +- extensions-contrib/sqlserver-metadata-storage/pom.xml | 2 +- extensions-contrib/statsd-emitter/pom.xml | 2 +- extensions-contrib/thrift-extensions/pom.xml | 2 +- extensions-contrib/time-min-max/pom.xml | 2 +- extensions-contrib/virtual-columns/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 2 +- extensions-core/datasketches/pom.xml | 2 +- extensions-core/druid-basic-security/pom.xml | 2 +- extensions-core/druid-kerberos/pom.xml | 2 +- extensions-core/hdfs-storage/pom.xml | 2 +- extensions-core/histogram/pom.xml | 2 +- extensions-core/kafka-eight/pom.xml | 2 +- extensions-core/kafka-extraction-namespace/pom.xml | 2 +- extensions-core/kafka-indexing-service/pom.xml | 2 +- extensions-core/lookups-cached-global/pom.xml | 2 +- extensions-core/lookups-cached-single/pom.xml | 2 +- extensions-core/mysql-metadata-storage/pom.xml | 2 +- extensions-core/postgresql-metadata-storage/pom.xml | 2 +- extensions-core/protobuf-extensions/pom.xml | 2 +- extensions-core/s3-extensions/pom.xml | 2 +- extensions-core/simple-client-sslcontext/pom.xml | 2 +- extensions-core/stats/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- integration-tests/pom.xml | 2 +- java-util/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- sql/pom.xml | 2 +- 53 files changed, 54 insertions(+), 54 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index d39d6252bbd0..166602edcbdf 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT diff --git a/aws-common/pom.xml b/aws-common/pom.xml index d50a415ba13d..c02c00107d1b 100644 --- a/aws-common/pom.xml +++ b/aws-common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 6ad9033eaa60..3f4939d260ae 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index e995d70ec91f..4509d9c7e89e 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT diff --git a/distribution/pom.xml b/distribution/pom.xml index d70e600a9412..ef6224f4c327 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -28,7 +28,7 @@ druid io.druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 2881ecd6d169..5624d3c6c59b 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT diff --git a/extendedset/pom.xml b/extendedset/pom.xml index 08ca138e34b5..82e00b4b777c 100755 --- a/extendedset/pom.xml +++ b/extendedset/pom.xml @@ -31,7 +31,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index 8b78e6ee625c..f654651e6062 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/azure-extensions/pom.xml b/extensions-contrib/azure-extensions/pom.xml index dae8229a31e1..3c37203f7f2d 100644 --- a/extensions-contrib/azure-extensions/pom.xml +++ b/extensions-contrib/azure-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/cassandra-storage/pom.xml b/extensions-contrib/cassandra-storage/pom.xml index 31b25595f282..89e3a38ced1a 100644 --- a/extensions-contrib/cassandra-storage/pom.xml +++ b/extensions-contrib/cassandra-storage/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index c8c23fad5e0d..e5da29541e23 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/distinctcount/pom.xml b/extensions-contrib/distinctcount/pom.xml index e1ce0db82aab..a4728e929e44 100644 --- a/extensions-contrib/distinctcount/pom.xml +++ b/extensions-contrib/distinctcount/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/druid-rocketmq/pom.xml b/extensions-contrib/druid-rocketmq/pom.xml index 4d24afbc20c3..4792b201ad1c 100644 --- a/extensions-contrib/druid-rocketmq/pom.xml +++ b/extensions-contrib/druid-rocketmq/pom.xml @@ -22,7 +22,7 @@ druid io.druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/google-extensions/pom.xml b/extensions-contrib/google-extensions/pom.xml index 65ca4d6020f4..6bab9543e8bc 100644 --- a/extensions-contrib/google-extensions/pom.xml +++ b/extensions-contrib/google-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index 328564c0c1b8..e67d5e426dbd 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml index 34b821b5688a..39f413552d92 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml +++ b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index 227381e1c9fd..d269b186070c 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/orc-extensions/pom.xml b/extensions-contrib/orc-extensions/pom.xml index 47fdb9b05d31..69d006a2d8c3 100644 --- a/extensions-contrib/orc-extensions/pom.xml +++ b/extensions-contrib/orc-extensions/pom.xml @@ -24,7 +24,7 @@ druid io.druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/parquet-extensions/pom.xml b/extensions-contrib/parquet-extensions/pom.xml index 7c5b996e6053..3c088199630c 100644 --- a/extensions-contrib/parquet-extensions/pom.xml +++ b/extensions-contrib/parquet-extensions/pom.xml @@ -8,7 +8,7 @@ druid io.druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/rabbitmq/pom.xml b/extensions-contrib/rabbitmq/pom.xml index f5bfcdb8c53e..6949cfdbd42e 100644 --- a/extensions-contrib/rabbitmq/pom.xml +++ b/extensions-contrib/rabbitmq/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/redis-cache/pom.xml b/extensions-contrib/redis-cache/pom.xml index f8d0fcc61003..dfbb156bae54 100644 --- a/extensions-contrib/redis-cache/pom.xml +++ b/extensions-contrib/redis-cache/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/sqlserver-metadata-storage/pom.xml b/extensions-contrib/sqlserver-metadata-storage/pom.xml index babd86f0b38b..a561e7e1076a 100644 --- a/extensions-contrib/sqlserver-metadata-storage/pom.xml +++ b/extensions-contrib/sqlserver-metadata-storage/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index ed64e68b37f0..33d9ab14ba32 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index 97b5f1455e1c..1a00ace020ec 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -9,7 +9,7 @@ druid io.druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/time-min-max/pom.xml b/extensions-contrib/time-min-max/pom.xml index 8a670a6a952f..797aad39523f 100644 --- a/extensions-contrib/time-min-max/pom.xml +++ b/extensions-contrib/time-min-max/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/virtual-columns/pom.xml b/extensions-contrib/virtual-columns/pom.xml index 07e4793063e1..a04814f69242 100644 --- a/extensions-contrib/virtual-columns/pom.xml +++ b/extensions-contrib/virtual-columns/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index ebe8f25506c8..b5278dbdc552 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index 16a86841bb9c..9e625c426bd8 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-basic-security/pom.xml b/extensions-core/druid-basic-security/pom.xml index 0cf293e51fe0..439d3dc9f0b9 100644 --- a/extensions-core/druid-basic-security/pom.xml +++ b/extensions-core/druid-basic-security/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index f2ce8ad6da6b..9c48e63b175e 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index c6289ee663ec..08c1c63fba0f 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-core/histogram/pom.xml b/extensions-core/histogram/pom.xml index 76a4960aa52d..73b635db21a8 100644 --- a/extensions-core/histogram/pom.xml +++ b/extensions-core/histogram/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-eight/pom.xml b/extensions-core/kafka-eight/pom.xml index bf50e0a0c62d..b4eef456d031 100644 --- a/extensions-core/kafka-eight/pom.xml +++ b/extensions-core/kafka-eight/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index 2527b07461fb..5e61ca2905b0 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index c2bc2a524ce4..ffe49a5952ac 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-core/lookups-cached-global/pom.xml b/extensions-core/lookups-cached-global/pom.xml index bbd31280e87c..8bd07a744ac2 100644 --- a/extensions-core/lookups-cached-global/pom.xml +++ b/extensions-core/lookups-cached-global/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-core/lookups-cached-single/pom.xml b/extensions-core/lookups-cached-single/pom.xml index 5d8ac2acb684..073d8844c0b9 100644 --- a/extensions-core/lookups-cached-single/pom.xml +++ b/extensions-core/lookups-cached-single/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index 5c6628016d91..3fba241f0e50 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-core/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml index 7e3946bde2d7..0839ac68b9cc 100644 --- a/extensions-core/postgresql-metadata-storage/pom.xml +++ b/extensions-core/postgresql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-core/protobuf-extensions/pom.xml b/extensions-core/protobuf-extensions/pom.xml index 3c4dbdbc8351..79ca5c08e7f9 100644 --- a/extensions-core/protobuf-extensions/pom.xml +++ b/extensions-core/protobuf-extensions/pom.xml @@ -27,7 +27,7 @@ druid io.druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index ae315baa1bc4..ed7da285bc36 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/extensions-core/simple-client-sslcontext/pom.xml b/extensions-core/simple-client-sslcontext/pom.xml index fd8f0afd19ab..5df770662c1c 100644 --- a/extensions-core/simple-client-sslcontext/pom.xml +++ b/extensions-core/simple-client-sslcontext/pom.xml @@ -3,7 +3,7 @@ druid io.druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml index c3180f285671..385b00810313 100644 --- a/extensions-core/stats/pom.xml +++ b/extensions-core/stats/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT ../../pom.xml diff --git a/hll/pom.xml b/hll/pom.xml index a14c3f6826a2..4981ce3b510d 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT druid-hll diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index c932f230705d..9cb3f42f9068 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 1c2a341f38b9..b6a88d1c0cd8 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index ed0603fe977d..eb6684dfcc6b 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT diff --git a/java-util/pom.xml b/java-util/pom.xml index a6ad6c177403..ee4e67c84d17 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT java-util diff --git a/pom.xml b/pom.xml index 0528cc0c8f57..142c53bc56e4 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT pom ${project.groupId}:${project.artifactId} @@ -53,7 +53,7 @@ scm:git:ssh://git@github.com/druid-io/druid.git scm:git:ssh://git@github.com/druid-io/druid.git https://github.com/druid-io/druid.git - druid-0.12.0-rc3 + 0.12.0-SNAPSHOT diff --git a/processing/pom.xml b/processing/pom.xml index ac09f3f21592..e4550631d869 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 69ccbe905c69..fd71f67160ae 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index e5d83bc1805a..3bc7c71cbd66 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -25,7 +25,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT diff --git a/sql/pom.xml b/sql/pom.xml index 014cdae72dc8..e61c1333bdfc 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc3 + 0.12.0-rc4-SNAPSHOT From 5c73cbae3fff1fc7a9e08b8fb5ae9ef62fc545fe Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Thu, 22 Feb 2018 16:50:03 -0800 Subject: [PATCH 35/82] Automatically adjust com.metamx.metrics Monitor class references (#5412) (#5417) * Automatically adjust com.metamx.metrics monitor class references * Log warning for old class names --- .../druid/server/metrics/MonitorsConfig.java | 53 +++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/io/druid/server/metrics/MonitorsConfig.java b/server/src/main/java/io/druid/server/metrics/MonitorsConfig.java index 08d48a1d466f..05ccdc8a03fe 100644 --- a/server/src/main/java/io/druid/server/metrics/MonitorsConfig.java +++ b/server/src/main/java/io/druid/server/metrics/MonitorsConfig.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.druid.java.util.common.logger.Logger; import io.druid.java.util.metrics.Monitor; import io.druid.query.DruidMetrics; @@ -35,17 +36,37 @@ */ public class MonitorsConfig { - public final static String METRIC_DIMENSION_PREFIX = "druid.metrics.emitter.dimension."; + private static final Logger log = new Logger(MonitorsConfig.class); + + public static final String METRIC_DIMENSION_PREFIX = "druid.metrics.emitter.dimension."; + + /** + * Prior to 0.12.0, Druid used Monitor classes from the `com.metamx.metrics` package. + * In 0.12.0, these Monitor classes were moved into Druid under `io.druid.java.util.metrics`. + * See https://github.com/druid-io/druid/pull/5289 for details. + * + * We automatically adjust old `com.metamx.metrics` package references to `io.druid.java.util.metrics` + * for backwards compatibility purposes, easing the upgrade process for users. + */ + public static final String OLD_METAMX_PACKAGE_NAME = "com.metamx.metrics"; + public static final String NEW_DRUID_PACKAGE_NAME = "io.druid.java.util.metrics"; @JsonProperty("monitors") @NotNull - private List> monitors = Lists.newArrayList(); + private List> monitors; public List> getMonitors() { return monitors; } + public MonitorsConfig( + @JsonProperty("monitors") List monitorNames + ) + { + monitors = getMonitorsFromNames(monitorNames); + } + @Override public String toString() { @@ -54,7 +75,6 @@ public String toString() '}'; } - public static Map mapOfDatasourceAndTaskID( final String datasource, final String taskId @@ -86,4 +106,31 @@ public static Map extractDimensions(Properties props, List> getMonitorsFromNames(List monitorNames) + { + List> monitors = Lists.newArrayList(); + if (monitorNames == null) { + return monitors; + } + try { + for (String monitorName : monitorNames) { + String effectiveMonitorName = monitorName.replace(OLD_METAMX_PACKAGE_NAME, NEW_DRUID_PACKAGE_NAME); + if (!effectiveMonitorName.equals(monitorName)) { + log.warn( + "Deprecated Monitor class name [%s] found, please use package %s instead of %s", + monitorName, + NEW_DRUID_PACKAGE_NAME, + OLD_METAMX_PACKAGE_NAME + ); + } + Class monitorClass = (Class) Class.forName(effectiveMonitorName); + monitors.add(monitorClass); + } + return monitors; + } + catch (ClassNotFoundException cnfe) { + throw new RuntimeException(cnfe); + } + } } From 0be7c552db442e8c9308f937aa734863aac4523a Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Wed, 28 Feb 2018 15:52:34 -0800 Subject: [PATCH 36/82] Skip normal authentication for JDBC requests in Router (#5435) (#5442) * Skip normal authentication for JDBC requests in Router * Add integration test * PR comments --- integration-tests/docker/broker.conf | 2 + integration-tests/docker/router.conf | 2 + integration-tests/pom.xml | 10 +++ .../ITBasicAuthConfigurationTest.java | 67 +++++++++++++++++++ .../cli/RouterJettyServerInitializer.java | 7 +- .../sql/avatica/DruidAvaticaHandler.java | 2 +- 6 files changed, 88 insertions(+), 2 deletions(-) diff --git a/integration-tests/docker/broker.conf b/integration-tests/docker/broker.conf index e7e744d5db8a..d791e82b70f5 100644 --- a/integration-tests/docker/broker.conf +++ b/integration-tests/docker/broker.conf @@ -34,6 +34,8 @@ command=java -Ddruid.escalator.authorizerName=basic -Ddruid.auth.authorizers="[\"basic\"]" -Ddruid.auth.authorizer.basic.type=basic + -Ddruid.sql.enable=true + -Ddruid.sql.avatica.enable=true -cp /shared/docker/lib/* io.druid.cli.Main server broker redirect_stderr=true diff --git a/integration-tests/docker/router.conf b/integration-tests/docker/router.conf index 3222c18db36b..06af3aab2656 100644 --- a/integration-tests/docker/router.conf +++ b/integration-tests/docker/router.conf @@ -23,6 +23,8 @@ command=java -Ddruid.escalator.authorizerName=basic -Ddruid.auth.authorizers="[\"basic\"]" -Ddruid.auth.authorizer.basic.type=basic + -Ddruid.sql.enable=true + -Ddruid.sql.avatica.enable=true -cp /shared/docker/lib/* io.druid.cli.Main server router redirect_stderr=true diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index eb6684dfcc6b..521570abae7a 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -98,6 +98,16 @@ + + org.apache.calcite.avatica + avatica + 1.10.0 + + + org.apache.calcite.avatica + avatica-server + 1.10.0 + org.testng testng diff --git a/integration-tests/src/test/java/io/druid/tests/security/ITBasicAuthConfigurationTest.java b/integration-tests/src/test/java/io/druid/tests/security/ITBasicAuthConfigurationTest.java index b8ecd4f71f36..dd9344846f11 100644 --- a/integration-tests/src/test/java/io/druid/tests/security/ITBasicAuthConfigurationTest.java +++ b/integration-tests/src/test/java/io/druid/tests/security/ITBasicAuthConfigurationTest.java @@ -39,8 +39,10 @@ import io.druid.server.security.Resource; import io.druid.server.security.ResourceAction; import io.druid.server.security.ResourceType; +import io.druid.sql.avatica.DruidAvaticaHandler; import io.druid.testing.IntegrationTestingConfig; import io.druid.testing.guice.DruidTestModuleFactory; +import org.apache.calcite.avatica.AvaticaSqlException; import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpResponseStatus; import org.testng.Assert; @@ -49,9 +51,14 @@ import javax.ws.rs.core.MediaType; import java.net.URL; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Properties; @Guice(moduleFactory = DruidTestModuleFactory.class) public class ITBasicAuthConfigurationTest @@ -197,8 +204,68 @@ public void testAuthConfiguration() throws Exception LOG.info("Checking access for user druid99."); checkNodeAccess(newUser99Client); + + String brokerUrl = "jdbc:avatica:remote:url=" + config.getBrokerUrl() + DruidAvaticaHandler.AVATICA_PATH; + String routerUrl = "jdbc:avatica:remote:url=" + config.getRouterUrl() + DruidAvaticaHandler.AVATICA_PATH; + + LOG.info("Checking Avatica query on broker."); + testAvaticaQuery(brokerUrl); + + LOG.info("Checking Avatica query on router."); + testAvaticaQuery(routerUrl); + + LOG.info("Testing Avatica query on broker with incorrect credentials."); + testAvaticaAuthFailure(brokerUrl); + + LOG.info("Testing Avatica query on router with incorrect credentials."); + testAvaticaAuthFailure(routerUrl); } + private void testAvaticaQuery(String url) + { + LOG.info("URL: " + url); + try { + Properties connectionProperties = new Properties(); + connectionProperties.put("user", "admin"); + connectionProperties.put("password", "priest"); + Connection connection = DriverManager.getConnection(url, connectionProperties); + Statement statement = connection.createStatement(); + statement.setMaxRows(450); + String query = "SELECT * FROM INFORMATION_SCHEMA.COLUMNS"; + ResultSet resultSet = statement.executeQuery(query); + Assert.assertTrue(resultSet.next()); + statement.close(); + connection.close(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void testAvaticaAuthFailure(String url) throws Exception + { + LOG.info("URL: " + url); + try { + Properties connectionProperties = new Properties(); + connectionProperties.put("user", "admin"); + connectionProperties.put("password", "wrongpassword"); + Connection connection = DriverManager.getConnection(url, connectionProperties); + Statement statement = connection.createStatement(); + statement.setMaxRows(450); + String query = "SELECT * FROM INFORMATION_SCHEMA.COLUMNS"; + statement.executeQuery(query); + } + catch (AvaticaSqlException ase) { + Assert.assertEquals( + ase.getErrorMessage(), + "Error while executing SQL \"SELECT * FROM INFORMATION_SCHEMA.COLUMNS\": Remote driver error: ForbiddenException: Authentication failed." + ); + return; + } + Assert.fail("Test failed, did not get AvaticaSqlException."); + } + + private void checkNodeAccess(HttpClient httpClient) { makeRequest(httpClient, HttpMethod.GET, config.getCoordinatorUrl() + "/status", null); diff --git a/services/src/main/java/io/druid/cli/RouterJettyServerInitializer.java b/services/src/main/java/io/druid/cli/RouterJettyServerInitializer.java index fc00ca644cb6..abe20a2d66e2 100644 --- a/services/src/main/java/io/druid/cli/RouterJettyServerInitializer.java +++ b/services/src/main/java/io/druid/cli/RouterJettyServerInitializer.java @@ -36,6 +36,7 @@ import io.druid.server.security.AuthenticationUtils; import io.druid.server.security.Authenticator; import io.druid.server.security.AuthenticatorMapper; +import io.druid.sql.avatica.DruidAvaticaHandler; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.HandlerList; @@ -52,7 +53,11 @@ public class RouterJettyServerInitializer implements JettyServerInitializer private static Logger log = new Logger(RouterJettyServerInitializer.class); private static List UNSECURED_PATHS = Lists.newArrayList( - "/status/health" + "/status/health", + // JDBC authentication uses the JDBC connection context instead of HTTP headers, skip the normal auth checks. + // The router will keep the connection context in the forwarded message, and the broker is responsible for + // performing the auth checks. + DruidAvaticaHandler.AVATICA_PATH ); private final AsyncQueryForwardingServlet asyncQueryForwardingServlet; diff --git a/sql/src/main/java/io/druid/sql/avatica/DruidAvaticaHandler.java b/sql/src/main/java/io/druid/sql/avatica/DruidAvaticaHandler.java index 471efce6a904..5e37d2a6a105 100644 --- a/sql/src/main/java/io/druid/sql/avatica/DruidAvaticaHandler.java +++ b/sql/src/main/java/io/druid/sql/avatica/DruidAvaticaHandler.java @@ -35,7 +35,7 @@ public class DruidAvaticaHandler extends AvaticaJsonHandler { - static final String AVATICA_PATH = "/druid/v2/sql/avatica/"; + public static final String AVATICA_PATH = "/druid/v2/sql/avatica/"; @Inject public DruidAvaticaHandler( From d094f56883c4b5f110c27c45f725429c27cbd348 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 1 Mar 2018 00:52:19 -0800 Subject: [PATCH 37/82] add task priority for kafka indexing (#5446) --- .../main/java/io/druid/indexing/kafka/KafkaIndexTask.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java index b80ec2493288..17d742a7589c 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java +++ b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java @@ -62,6 +62,7 @@ import io.druid.indexing.common.task.AbstractTask; import io.druid.indexing.common.task.RealtimeIndexTask; import io.druid.indexing.common.task.TaskResource; +import io.druid.indexing.common.task.Tasks; import io.druid.indexing.kafka.supervisor.KafkaSupervisor; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.ISE; @@ -306,6 +307,12 @@ private static String makeTaskId(String dataSource, int randomBits) return Joiner.on("_").join(TYPE, dataSource, suffix); } + @Override + public int getPriority() + { + return getContextValue(Tasks.PRIORITY_KEY, Tasks.DEFAULT_REALTIME_TASK_PRIORITY); + } + @Override public String getType() { From 5a11b097c0e67160dbff38e07473d7a60a369060 Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Fri, 2 Mar 2018 13:46:35 -0800 Subject: [PATCH 38/82] [Backport] Fix GroupBy limit push down descending sorting on numeric columns (#5453) (#5454) * Fix GroupBy limit push down descending sorting on numeric columns (#5453) * Compile fix --- .../io/druid/query/groupby/GroupByQuery.java | 18 +- ...roupByLimitPushDownMultiNodeMergeTest.java | 207 +++++++++++++++++- 2 files changed, 205 insertions(+), 20 deletions(-) diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java index ea48985cedb7..1f9b45e62eb0 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java @@ -574,22 +574,14 @@ private static int compareDimsForLimitPushDown( final StringComparator comparator = comparators.get(i); final int dimCompare; - - Object lhsObj; - Object rhsObj; - if (needsReverseList.get(i)) { - lhsObj = rhs.getRaw(fieldName); - rhsObj = lhs.getRaw(fieldName); - } else { - lhsObj = lhs.getRaw(fieldName); - rhsObj = rhs.getRaw(fieldName); - } + final Object lhsObj = lhs.getRaw(fieldName); + final Object rhsObj = rhs.getRaw(fieldName); if (isNumericField.get(i)) { if (comparator.equals(StringComparators.NUMERIC)) { dimCompare = ((Ordering) Comparators.naturalNullsFirst()).compare( - lhs.getRaw(fieldName), - rhs.getRaw(fieldName) + lhsObj, + rhsObj ); } else { dimCompare = comparator.compare(String.valueOf(lhsObj), String.valueOf(rhsObj)); @@ -599,7 +591,7 @@ private static int compareDimsForLimitPushDown( } if (dimCompare != 0) { - return dimCompare; + return needsReverseList.get(i) ? -dimCompare : dimCompare; } } return 0; diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java index d11f8e62d3f0..e15f076dbfe5 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByLimitPushDownMultiNodeMergeTest.java @@ -49,6 +49,9 @@ import io.druid.java.util.common.guava.Sequences; import io.druid.java.util.common.logger.Logger; import io.druid.math.expr.ExprMacroTable; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.expression.TestExprMacroTable; +import io.druid.segment.virtual.ExpressionVirtualColumn; import io.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import io.druid.query.BySegmentQueryRunner; import io.druid.query.DruidProcessingConfig; @@ -240,7 +243,78 @@ public void setup() throws Exception ); QueryableIndex qindexB = INDEX_IO.loadIndex(fileB); - groupByIndices = Arrays.asList(qindexA, qindexB); + final IncrementalIndex indexC = makeIncIndex(false); + incrementalIndices.add(indexC); + + event = new HashMap<>(); + event.put("dimA", "pomegranate"); + event.put("metA", 2395L); + row = new MapBasedInputRow(1505260800000L, dimNames, event); + indexC.add(row); + + event = new HashMap<>(); + event.put("dimA", "mango"); + event.put("metA", 8L); + row = new MapBasedInputRow(1605260800000L, dimNames, event); + indexC.add(row); + + event = new HashMap<>(); + event.put("dimA", "pomegranate"); + event.put("metA", 5028L); + row = new MapBasedInputRow(1705264400000L, dimNames, event); + indexC.add(row); + + event = new HashMap<>(); + event.put("dimA", "mango"); + event.put("metA", 7L); + row = new MapBasedInputRow(1805264400000L, dimNames, event); + indexC.add(row); + + final File fileC = INDEX_MERGER_V9.persist( + indexC, + new File(tmpDir, "C"), + new IndexSpec(), + null + ); + QueryableIndex qindexC = INDEX_IO.loadIndex(fileC); + + + final IncrementalIndex indexD = makeIncIndex(false); + incrementalIndices.add(indexD); + + event = new HashMap<>(); + event.put("dimA", "pomegranate"); + event.put("metA", 4718L); + row = new MapBasedInputRow(1505260800000L, dimNames, event); + indexD.add(row); + + event = new HashMap<>(); + event.put("dimA", "mango"); + event.put("metA", 18L); + row = new MapBasedInputRow(1605260800000L, dimNames, event); + indexD.add(row); + + event = new HashMap<>(); + event.put("dimA", "pomegranate"); + event.put("metA", 2698L); + row = new MapBasedInputRow(1705264400000L, dimNames, event); + indexD.add(row); + + event = new HashMap<>(); + event.put("dimA", "mango"); + event.put("metA", 3L); + row = new MapBasedInputRow(1805264400000L, dimNames, event); + indexD.add(row); + + final File fileD = INDEX_MERGER_V9.persist( + indexD, + new File(tmpDir, "D"), + new IndexSpec(), + null + ); + QueryableIndex qindexD = INDEX_IO.loadIndex(fileD); + + groupByIndices = Arrays.asList(qindexA, qindexB, qindexC, qindexD); setupGroupByFactory(); } @@ -376,6 +450,125 @@ public void tearDown() throws Exception } } + @Test + public void testDescendingNumerics() throws Exception + { + QueryToolChest toolChest = groupByFactory.getToolchest(); + QueryRunner theRunner = new FinalizeResultsQueryRunner<>( + toolChest.mergeResults( + groupByFactory.mergeRunners(executorService, getRunner1(2)) + ), + (QueryToolChest) toolChest + ); + + QueryRunner theRunner2 = new FinalizeResultsQueryRunner<>( + toolChest.mergeResults( + groupByFactory2.mergeRunners(executorService, getRunner2(3)) + ), + (QueryToolChest) toolChest + ); + + QueryRunner finalRunner = new FinalizeResultsQueryRunner<>( + toolChest.mergeResults( + new QueryRunner() + { + @Override + public Sequence run(QueryPlus queryPlus, Map responseContext) + { + return Sequences + .simple( + ImmutableList.of( + theRunner.run(queryPlus, responseContext), + theRunner2.run(queryPlus, responseContext) + ) + ) + .flatMerge(Function.identity(), queryPlus.getQuery().getResultOrdering()); + } + } + ), + (QueryToolChest) toolChest + ); + + QuerySegmentSpec intervalSpec = new MultipleIntervalSegmentSpec( + Collections.singletonList(Intervals.utc(1500000000000L, 1900000000000L)) + ); + + DefaultLimitSpec ls2 = new DefaultLimitSpec( + Arrays.asList( + new OrderByColumnSpec("d0", OrderByColumnSpec.Direction.DESCENDING, StringComparators.NUMERIC), + new OrderByColumnSpec("d1", OrderByColumnSpec.Direction.DESCENDING, StringComparators.NUMERIC), + new OrderByColumnSpec("d2", OrderByColumnSpec.Direction.DESCENDING, StringComparators.NUMERIC) + ), + 100 + ); + + GroupByQuery query = GroupByQuery + .builder() + .setDataSource("blah") + .setQuerySegmentSpec(intervalSpec) + .setVirtualColumns( + new ExpressionVirtualColumn("d0:v", "timestamp_extract(\"__time\",'YEAR','UTC')", ValueType.LONG, TestExprMacroTable.INSTANCE), + new ExpressionVirtualColumn("d1:v", "timestamp_extract(\"__time\",'MONTH','UTC')", ValueType.LONG, TestExprMacroTable.INSTANCE), + new ExpressionVirtualColumn("d2:v", "timestamp_extract(\"__time\",'DAY','UTC')", ValueType.LONG, TestExprMacroTable.INSTANCE) + ) + .setDimensions(Lists.newArrayList( + new DefaultDimensionSpec("d0:v", "d0", ValueType.LONG), + new DefaultDimensionSpec("d1:v", "d1", ValueType.LONG), + new DefaultDimensionSpec("d2:v", "d2", ValueType.LONG) + )) + .setAggregatorSpecs( + Arrays.asList(new CountAggregatorFactory("a0")) + ) + .setLimitSpec( + ls2 + ) + .setContext( + ImmutableMap.of( + GroupByQueryConfig.CTX_KEY_APPLY_LIMIT_PUSH_DOWN, true + ) + ) + .setGranularity(Granularities.ALL) + .build(); + + Sequence queryResult = finalRunner.run(QueryPlus.wrap(query), Maps.newHashMap()); + List results = Sequences.toList(queryResult, Lists.newArrayList()); + + Row expectedRow0 = GroupByQueryRunnerTestHelper.createExpectedRow( + "2017-07-14T02:40:00.000Z", + "d0", 2027L, + "d1", 3L, + "d2", 17L, + "a0", 2L + ); + Row expectedRow1 = GroupByQueryRunnerTestHelper.createExpectedRow( + "2017-07-14T02:40:00.000Z", + "d0", 2024L, + "d1", 1L, + "d2", 14L, + "a0", 2L + ); + Row expectedRow2 = GroupByQueryRunnerTestHelper.createExpectedRow( + "2017-07-14T02:40:00.000Z", + "d0", 2020L, + "d1", 11L, + "d2", 13L, + "a0", 2L + ); + Row expectedRow3 = GroupByQueryRunnerTestHelper.createExpectedRow( + "2017-07-14T02:40:00.000Z", + "d0", 2017L, + "d1", 9L, + "d2", 13L, + "a0", 2L + ); + + Assert.assertEquals(4, results.size()); + Assert.assertEquals(expectedRow0, results.get(0)); + Assert.assertEquals(expectedRow1, results.get(1)); + Assert.assertEquals(expectedRow2, results.get(2)); + Assert.assertEquals(expectedRow3, results.get(3)); + } + @Test public void testPartialLimitPushDownMerge() throws Exception { @@ -384,14 +577,14 @@ public void testPartialLimitPushDownMerge() throws Exception QueryToolChest toolChest = groupByFactory.getToolchest(); QueryRunner theRunner = new FinalizeResultsQueryRunner<>( toolChest.mergeResults( - groupByFactory.mergeRunners(executorService, getRunner1()) + groupByFactory.mergeRunners(executorService, getRunner1(0)) ), (QueryToolChest) toolChest ); QueryRunner theRunner2 = new FinalizeResultsQueryRunner<>( toolChest.mergeResults( - groupByFactory2.mergeRunners(executorService, getRunner2()) + groupByFactory2.mergeRunners(executorService, getRunner2(1)) ), (QueryToolChest) toolChest ); @@ -495,10 +688,10 @@ public Sequence run(QueryPlus queryPlus, Map responseC Assert.assertEquals(expectedRow3, results.get(3)); } - private List> getRunner1() + private List> getRunner1(int qIndexNumber) { List> runners = Lists.newArrayList(); - QueryableIndex index = groupByIndices.get(0); + QueryableIndex index = groupByIndices.get(qIndexNumber); QueryRunner runner = makeQueryRunner( groupByFactory, index.toString(), @@ -508,10 +701,10 @@ private List> getRunner1() return runners; } - private List> getRunner2() + private List> getRunner2(int qIndexNumber) { List> runners = Lists.newArrayList(); - QueryableIndex index2 = groupByIndices.get(1); + QueryableIndex index2 = groupByIndices.get(qIndexNumber); QueryRunner tooSmallRunner = makeQueryRunner( groupByFactory2, index2.toString(), From c9e5f1b462278ddf591b8fe85a38467cc305ecf6 Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Fri, 2 Mar 2018 16:03:31 -0800 Subject: [PATCH 39/82] Fix authorization check in supervisor history API (#5460) (#5463) [Backport] Fix authorization check in supervisor history API (#5460) --- .../supervisor/SupervisorResource.java | 2 +- .../supervisor/SupervisorResourceTest.java | 89 ++++++++++++++++++- 2 files changed, 88 insertions(+), 3 deletions(-) diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorResource.java b/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorResource.java index c23e028e6f10..b7cefa3fcd85 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorResource.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorResource.java @@ -235,7 +235,7 @@ public boolean apply(String id) } } ); - return Response.ok(supervisorHistory).build(); + return Response.ok(authorizedSupervisorHistory).build(); } } ); diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorResourceTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorResourceTest.java index 2c34ef83a28c..961f1e91069e 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorResourceTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorResourceTest.java @@ -28,9 +28,13 @@ import io.druid.indexing.overlord.DataSourceMetadata; import io.druid.indexing.overlord.TaskMaster; import io.druid.java.util.common.DateTimes; +import io.druid.server.security.Access; +import io.druid.server.security.Action; import io.druid.server.security.AuthConfig; -import io.druid.server.security.AuthTestUtils; import io.druid.server.security.AuthenticationResult; +import io.druid.server.security.Authorizer; +import io.druid.server.security.AuthorizerMapper; +import io.druid.server.security.Resource; import org.easymock.Capture; import org.easymock.EasyMock; import org.easymock.EasyMockRunner; @@ -64,7 +68,34 @@ public class SupervisorResourceTest extends EasyMockSupport @Before public void setUp() throws Exception { - supervisorResource = new SupervisorResource(taskMaster, new AuthConfig(), AuthTestUtils.TEST_AUTHORIZER_MAPPER); + supervisorResource = new SupervisorResource( + taskMaster, + new AuthConfig(), + new AuthorizerMapper(null) { + @Override + public Authorizer getAuthorizer(String name) + { + return new Authorizer() + { + @Override + public Access authorize( + AuthenticationResult authenticationResult, Resource resource, Action action + ) + { + if (authenticationResult.getIdentity().equals("druid")) { + return Access.OK; + } else { + if (resource.getName().equals("datasource2")) { + return new Access(false, "not authorized."); + } else { + return Access.OK; + } + } + } + }; + } + } + ); } @Test @@ -303,6 +334,60 @@ public List getDataSources() Assert.assertEquals(503, response.getStatus()); } + @Test + public void testSpecGetAllHistoryWithAuthFailureFiltering() throws Exception + { + Map> history = Maps.newHashMap(); + history.put("id1", null); + history.put("id2", null); + + EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.of(supervisorManager)).times(2); + EasyMock.expect(supervisorManager.getSupervisorHistory()).andReturn(history); + SupervisorSpec spec1 = new TestSupervisorSpec("id1", null) { + + @Override + public List getDataSources() + { + return Lists.newArrayList("datasource1"); + } + }; + SupervisorSpec spec2 = new TestSupervisorSpec("id2", null) { + + @Override + public List getDataSources() + { + return Lists.newArrayList("datasource2"); + } + }; + EasyMock.expect(supervisorManager.getSupervisorSpec("id1")).andReturn(Optional.of(spec1)).atLeastOnce(); + EasyMock.expect(supervisorManager.getSupervisorSpec("id2")).andReturn(Optional.of(spec2)).atLeastOnce(); + EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).atLeastOnce(); + EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( + new AuthenticationResult("wronguser", "druid", null) + ).atLeastOnce(); + request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); + EasyMock.expectLastCall().anyTimes(); + replayAll(); + + Response response = supervisorResource.specGetAllHistory(request); + + Map> filteredHistory = Maps.newHashMap(); + filteredHistory.put("id1", null); + + Assert.assertEquals(200, response.getStatus()); + Assert.assertEquals(filteredHistory, response.getEntity()); + + resetAll(); + + EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.absent()); + replayAll(); + + response = supervisorResource.specGetAllHistory(request); + verifyAll(); + + Assert.assertEquals(503, response.getStatus()); + } + @Test public void testSpecGetHistory() throws Exception { From 8265f44bd68793a503cf69ab0631cd8db5894994 Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Mon, 5 Mar 2018 22:36:41 -0800 Subject: [PATCH 40/82] [Backport] Fix JSON serde for taskStatusPlus (#5469) (#5472) * Fix JSON serde for taskStatusPlus (#5469) * Fix JSON serde for taskStatusPlus * add newline * fix to statusCode * fix fobidden api check * remove debugging code * Compile fix --- .../java/io/druid/indexer/TaskStatusPlus.java | 42 ++++++++- .../io/druid/indexer/TaskStatusPlusTest.java | 87 +++++++++++++++++++ 2 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 api/src/test/java/io/druid/indexer/TaskStatusPlusTest.java diff --git a/api/src/main/java/io/druid/indexer/TaskStatusPlus.java b/api/src/main/java/io/druid/indexer/TaskStatusPlus.java index a45a9a7865a9..0560206f213a 100644 --- a/api/src/main/java/io/druid/indexer/TaskStatusPlus.java +++ b/api/src/main/java/io/druid/indexer/TaskStatusPlus.java @@ -25,6 +25,7 @@ import org.joda.time.DateTime; import javax.annotation.Nullable; +import java.util.Objects; public class TaskStatusPlus { @@ -40,7 +41,7 @@ public TaskStatusPlus( @JsonProperty("id") String id, @JsonProperty("createdTime") DateTime createdTime, @JsonProperty("queueInsertionTime") DateTime queueInsertionTime, - @JsonProperty("state") @Nullable TaskState state, + @JsonProperty("statusCode") @Nullable TaskState state, @JsonProperty("duration") @Nullable Long duration, @JsonProperty("location") TaskLocation location ) @@ -74,7 +75,8 @@ public DateTime getQueueInsertionTime() return queueInsertionTime; } - @JsonProperty + @Nullable + @JsonProperty("statusCode") public TaskState getState() { return state; @@ -91,4 +93,40 @@ public TaskLocation getLocation() { return location; } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + final TaskStatusPlus that = (TaskStatusPlus) o; + if (!id.equals(that.id)) { + return false; + } + if (!createdTime.equals(that.createdTime)) { + return false; + } + if (!queueInsertionTime.equals(that.queueInsertionTime)) { + return false; + } + if (!Objects.equals(state, that.state)) { + return false; + } + if (!Objects.equals(duration, that.duration)) { + return false; + } + return location.equals(that.location); + } + + @Override + public int hashCode() + { + return Objects.hash(id, createdTime, queueInsertionTime, state, duration, location); + } } diff --git a/api/src/test/java/io/druid/indexer/TaskStatusPlusTest.java b/api/src/test/java/io/druid/indexer/TaskStatusPlusTest.java new file mode 100644 index 000000000000..a4f3460e65a2 --- /dev/null +++ b/api/src/test/java/io/druid/indexer/TaskStatusPlusTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.indexer; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; +import io.druid.java.util.common.DateTimes; +import org.joda.time.DateTime; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; + +public class TaskStatusPlusTest +{ + @Test + public void testSerde() throws IOException + { + final ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule( + new SimpleModule() + .addDeserializer(DateTime.class, new DateTimeDeserializer()) + .addSerializer(DateTime.class, ToStringSerializer.instance) + ); + final TaskStatusPlus status = new TaskStatusPlus( + "testId", + DateTimes.nowUtc(), + DateTimes.nowUtc(), + TaskState.RUNNING, + 1000L, + TaskLocation.create("testHost", 1010, -1) + ); + final String json = mapper.writeValueAsString(status); + Assert.assertEquals(status, mapper.readValue(json, TaskStatusPlus.class)); + } + + // Copied from io.druid.jackson.JodaStuff + private static class DateTimeDeserializer extends StdDeserializer + { + public DateTimeDeserializer() + { + super(DateTime.class); + } + + @Override + public DateTime deserialize(JsonParser jp, DeserializationContext ctxt) + throws IOException, JsonProcessingException + { + JsonToken t = jp.getCurrentToken(); + if (t == JsonToken.VALUE_NUMBER_INT) { + return DateTimes.utc(jp.getLongValue()); + } + if (t == JsonToken.VALUE_STRING) { + String str = jp.getText().trim(); + if (str.length() == 0) { // [JACKSON-360] + return null; + } + // make sure to preserve time zone information when parsing timestamps + return DateTimes.ISO_DATE_OR_TIME_WITH_OFFSET.parse(str); + } + throw ctxt.mappingException(getValueClass()); + } + } +} From 125552a91ce826977b7f3a2841af019de6f94088 Mon Sep 17 00:00:00 2001 From: jon-wei Date: Tue, 6 Mar 2018 11:35:17 -0800 Subject: [PATCH 41/82] [maven-release-plugin] prepare release druid-0.12.0 --- api/pom.xml | 2 +- aws-common/pom.xml | 2 +- benchmarks/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- extendedset/pom.xml | 2 +- extensions-contrib/ambari-metrics-emitter/pom.xml | 2 +- extensions-contrib/azure-extensions/pom.xml | 2 +- extensions-contrib/cassandra-storage/pom.xml | 2 +- extensions-contrib/cloudfiles-extensions/pom.xml | 2 +- extensions-contrib/distinctcount/pom.xml | 2 +- extensions-contrib/druid-rocketmq/pom.xml | 2 +- extensions-contrib/google-extensions/pom.xml | 2 +- extensions-contrib/graphite-emitter/pom.xml | 2 +- extensions-contrib/kafka-eight-simpleConsumer/pom.xml | 2 +- extensions-contrib/kafka-emitter/pom.xml | 2 +- extensions-contrib/orc-extensions/pom.xml | 2 +- extensions-contrib/parquet-extensions/pom.xml | 2 +- extensions-contrib/rabbitmq/pom.xml | 2 +- extensions-contrib/redis-cache/pom.xml | 2 +- extensions-contrib/sqlserver-metadata-storage/pom.xml | 2 +- extensions-contrib/statsd-emitter/pom.xml | 2 +- extensions-contrib/thrift-extensions/pom.xml | 2 +- extensions-contrib/time-min-max/pom.xml | 2 +- extensions-contrib/virtual-columns/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 2 +- extensions-core/datasketches/pom.xml | 2 +- extensions-core/druid-basic-security/pom.xml | 2 +- extensions-core/druid-kerberos/pom.xml | 2 +- extensions-core/hdfs-storage/pom.xml | 2 +- extensions-core/histogram/pom.xml | 2 +- extensions-core/kafka-eight/pom.xml | 2 +- extensions-core/kafka-extraction-namespace/pom.xml | 2 +- extensions-core/kafka-indexing-service/pom.xml | 2 +- extensions-core/lookups-cached-global/pom.xml | 2 +- extensions-core/lookups-cached-single/pom.xml | 2 +- extensions-core/mysql-metadata-storage/pom.xml | 2 +- extensions-core/postgresql-metadata-storage/pom.xml | 2 +- extensions-core/protobuf-extensions/pom.xml | 2 +- extensions-core/s3-extensions/pom.xml | 2 +- extensions-core/simple-client-sslcontext/pom.xml | 2 +- extensions-core/stats/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- integration-tests/pom.xml | 2 +- java-util/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- sql/pom.xml | 2 +- 53 files changed, 54 insertions(+), 54 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 166602edcbdf..e372ef260992 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 diff --git a/aws-common/pom.xml b/aws-common/pom.xml index c02c00107d1b..e9c5b4ce95a7 100644 --- a/aws-common/pom.xml +++ b/aws-common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 3f4939d260ae..f17278db8d44 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 diff --git a/common/pom.xml b/common/pom.xml index 4509d9c7e89e..67daecaacaca 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 diff --git a/distribution/pom.xml b/distribution/pom.xml index ef6224f4c327..8eb92fed8377 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -28,7 +28,7 @@ druid io.druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 diff --git a/examples/pom.xml b/examples/pom.xml index 5624d3c6c59b..a0869b12dc67 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 diff --git a/extendedset/pom.xml b/extendedset/pom.xml index 82e00b4b777c..eecb33b46d06 100755 --- a/extendedset/pom.xml +++ b/extendedset/pom.xml @@ -31,7 +31,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index f654651e6062..1b9bc72f66a8 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-contrib/azure-extensions/pom.xml b/extensions-contrib/azure-extensions/pom.xml index 3c37203f7f2d..bbe693efcc9c 100644 --- a/extensions-contrib/azure-extensions/pom.xml +++ b/extensions-contrib/azure-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-contrib/cassandra-storage/pom.xml b/extensions-contrib/cassandra-storage/pom.xml index 89e3a38ced1a..e801a4ba0aba 100644 --- a/extensions-contrib/cassandra-storage/pom.xml +++ b/extensions-contrib/cassandra-storage/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index e5da29541e23..3b9ee7355bc0 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-contrib/distinctcount/pom.xml b/extensions-contrib/distinctcount/pom.xml index a4728e929e44..ca34cfebf548 100644 --- a/extensions-contrib/distinctcount/pom.xml +++ b/extensions-contrib/distinctcount/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-contrib/druid-rocketmq/pom.xml b/extensions-contrib/druid-rocketmq/pom.xml index 4792b201ad1c..6122af5f5284 100644 --- a/extensions-contrib/druid-rocketmq/pom.xml +++ b/extensions-contrib/druid-rocketmq/pom.xml @@ -22,7 +22,7 @@ druid io.druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-contrib/google-extensions/pom.xml b/extensions-contrib/google-extensions/pom.xml index 6bab9543e8bc..13e53102d9eb 100644 --- a/extensions-contrib/google-extensions/pom.xml +++ b/extensions-contrib/google-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index e67d5e426dbd..5cf5e6c18ee1 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml index 39f413552d92..e55de99898bd 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml +++ b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index d269b186070c..65b47f4fd486 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-contrib/orc-extensions/pom.xml b/extensions-contrib/orc-extensions/pom.xml index 69d006a2d8c3..42f88cc6ef57 100644 --- a/extensions-contrib/orc-extensions/pom.xml +++ b/extensions-contrib/orc-extensions/pom.xml @@ -24,7 +24,7 @@ druid io.druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/parquet-extensions/pom.xml b/extensions-contrib/parquet-extensions/pom.xml index 3c088199630c..9d2bb10608af 100644 --- a/extensions-contrib/parquet-extensions/pom.xml +++ b/extensions-contrib/parquet-extensions/pom.xml @@ -8,7 +8,7 @@ druid io.druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/rabbitmq/pom.xml b/extensions-contrib/rabbitmq/pom.xml index 6949cfdbd42e..d7be6ae42723 100644 --- a/extensions-contrib/rabbitmq/pom.xml +++ b/extensions-contrib/rabbitmq/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-contrib/redis-cache/pom.xml b/extensions-contrib/redis-cache/pom.xml index dfbb156bae54..c76b87faa64a 100644 --- a/extensions-contrib/redis-cache/pom.xml +++ b/extensions-contrib/redis-cache/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-contrib/sqlserver-metadata-storage/pom.xml b/extensions-contrib/sqlserver-metadata-storage/pom.xml index a561e7e1076a..73464c40f61c 100644 --- a/extensions-contrib/sqlserver-metadata-storage/pom.xml +++ b/extensions-contrib/sqlserver-metadata-storage/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index 33d9ab14ba32..ef28c0aafb71 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index 1a00ace020ec..e231e95d8ee9 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -9,7 +9,7 @@ druid io.druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/time-min-max/pom.xml b/extensions-contrib/time-min-max/pom.xml index 797aad39523f..d2707b1c1430 100644 --- a/extensions-contrib/time-min-max/pom.xml +++ b/extensions-contrib/time-min-max/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/virtual-columns/pom.xml b/extensions-contrib/virtual-columns/pom.xml index a04814f69242..e671c223e478 100644 --- a/extensions-contrib/virtual-columns/pom.xml +++ b/extensions-contrib/virtual-columns/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index b5278dbdc552..d1d7fb360773 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index 9e625c426bd8..a322e5f62c53 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-core/druid-basic-security/pom.xml b/extensions-core/druid-basic-security/pom.xml index 439d3dc9f0b9..c214081573e5 100644 --- a/extensions-core/druid-basic-security/pom.xml +++ b/extensions-core/druid-basic-security/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index 9c48e63b175e..2d4f5b110d46 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index 08c1c63fba0f..64ffbb6c0f6a 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-core/histogram/pom.xml b/extensions-core/histogram/pom.xml index 73b635db21a8..63d3c6665dfb 100644 --- a/extensions-core/histogram/pom.xml +++ b/extensions-core/histogram/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-core/kafka-eight/pom.xml b/extensions-core/kafka-eight/pom.xml index b4eef456d031..9204cd5ead71 100644 --- a/extensions-core/kafka-eight/pom.xml +++ b/extensions-core/kafka-eight/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index 5e61ca2905b0..1b4eeaa60efd 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index ffe49a5952ac..9c91250daa9c 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-core/lookups-cached-global/pom.xml b/extensions-core/lookups-cached-global/pom.xml index 8bd07a744ac2..9eadea80bb0f 100644 --- a/extensions-core/lookups-cached-global/pom.xml +++ b/extensions-core/lookups-cached-global/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-core/lookups-cached-single/pom.xml b/extensions-core/lookups-cached-single/pom.xml index 073d8844c0b9..17bfe7a2e6ef 100644 --- a/extensions-core/lookups-cached-single/pom.xml +++ b/extensions-core/lookups-cached-single/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index 3fba241f0e50..c24ac06570f6 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-core/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml index 0839ac68b9cc..3a08bbd9254a 100644 --- a/extensions-core/postgresql-metadata-storage/pom.xml +++ b/extensions-core/postgresql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-core/protobuf-extensions/pom.xml b/extensions-core/protobuf-extensions/pom.xml index 79ca5c08e7f9..f7d444c0b4eb 100644 --- a/extensions-core/protobuf-extensions/pom.xml +++ b/extensions-core/protobuf-extensions/pom.xml @@ -27,7 +27,7 @@ druid io.druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index ed7da285bc36..2c2f184a7c6a 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/extensions-core/simple-client-sslcontext/pom.xml b/extensions-core/simple-client-sslcontext/pom.xml index 5df770662c1c..1fecaa1a56c7 100644 --- a/extensions-core/simple-client-sslcontext/pom.xml +++ b/extensions-core/simple-client-sslcontext/pom.xml @@ -3,7 +3,7 @@ druid io.druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml 4.0.0 diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml index 385b00810313..20182abeae04 100644 --- a/extensions-core/stats/pom.xml +++ b/extensions-core/stats/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 ../../pom.xml diff --git a/hll/pom.xml b/hll/pom.xml index 4981ce3b510d..9bcc1d937cd2 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 druid-hll diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 9cb3f42f9068..02b5cfa1ad56 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index b6a88d1c0cd8..850c6801dcee 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 521570abae7a..138132a06e6e 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 diff --git a/java-util/pom.xml b/java-util/pom.xml index ee4e67c84d17..a945a9743b56 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 java-util diff --git a/pom.xml b/pom.xml index 142c53bc56e4..bfb120e2b752 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 pom ${project.groupId}:${project.artifactId} @@ -53,7 +53,7 @@ scm:git:ssh://git@github.com/druid-io/druid.git scm:git:ssh://git@github.com/druid-io/druid.git https://github.com/druid-io/druid.git - 0.12.0-SNAPSHOT + druid-0.12.0 diff --git a/processing/pom.xml b/processing/pom.xml index e4550631d869..5a66331e3a43 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 diff --git a/server/pom.xml b/server/pom.xml index fd71f67160ae..016d9d129269 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 diff --git a/services/pom.xml b/services/pom.xml index 3bc7c71cbd66..b1b8223929aa 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -25,7 +25,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 diff --git a/sql/pom.xml b/sql/pom.xml index e61c1333bdfc..54b603079d01 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0-rc4-SNAPSHOT + 0.12.0 From f650a1a618e6073ca733ccb284f516e34a044be5 Mon Sep 17 00:00:00 2001 From: jon-wei Date: Tue, 6 Mar 2018 11:35:31 -0800 Subject: [PATCH 42/82] [maven-release-plugin] prepare for next development iteration --- api/pom.xml | 2 +- aws-common/pom.xml | 2 +- benchmarks/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- extendedset/pom.xml | 2 +- extensions-contrib/ambari-metrics-emitter/pom.xml | 2 +- extensions-contrib/azure-extensions/pom.xml | 2 +- extensions-contrib/cassandra-storage/pom.xml | 2 +- extensions-contrib/cloudfiles-extensions/pom.xml | 2 +- extensions-contrib/distinctcount/pom.xml | 2 +- extensions-contrib/druid-rocketmq/pom.xml | 2 +- extensions-contrib/google-extensions/pom.xml | 2 +- extensions-contrib/graphite-emitter/pom.xml | 2 +- extensions-contrib/kafka-eight-simpleConsumer/pom.xml | 2 +- extensions-contrib/kafka-emitter/pom.xml | 2 +- extensions-contrib/orc-extensions/pom.xml | 2 +- extensions-contrib/parquet-extensions/pom.xml | 2 +- extensions-contrib/rabbitmq/pom.xml | 2 +- extensions-contrib/redis-cache/pom.xml | 2 +- extensions-contrib/sqlserver-metadata-storage/pom.xml | 2 +- extensions-contrib/statsd-emitter/pom.xml | 2 +- extensions-contrib/thrift-extensions/pom.xml | 2 +- extensions-contrib/time-min-max/pom.xml | 2 +- extensions-contrib/virtual-columns/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 2 +- extensions-core/datasketches/pom.xml | 2 +- extensions-core/druid-basic-security/pom.xml | 2 +- extensions-core/druid-kerberos/pom.xml | 2 +- extensions-core/hdfs-storage/pom.xml | 2 +- extensions-core/histogram/pom.xml | 2 +- extensions-core/kafka-eight/pom.xml | 2 +- extensions-core/kafka-extraction-namespace/pom.xml | 2 +- extensions-core/kafka-indexing-service/pom.xml | 2 +- extensions-core/lookups-cached-global/pom.xml | 2 +- extensions-core/lookups-cached-single/pom.xml | 2 +- extensions-core/mysql-metadata-storage/pom.xml | 2 +- extensions-core/postgresql-metadata-storage/pom.xml | 2 +- extensions-core/protobuf-extensions/pom.xml | 2 +- extensions-core/s3-extensions/pom.xml | 2 +- extensions-core/simple-client-sslcontext/pom.xml | 2 +- extensions-core/stats/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- integration-tests/pom.xml | 2 +- java-util/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- sql/pom.xml | 2 +- 53 files changed, 54 insertions(+), 54 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index e372ef260992..0e8333e27709 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT diff --git a/aws-common/pom.xml b/aws-common/pom.xml index e9c5b4ce95a7..9ce40e3951ea 100644 --- a/aws-common/pom.xml +++ b/aws-common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index f17278db8d44..539d3df54524 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 67daecaacaca..234f21af5bb2 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT diff --git a/distribution/pom.xml b/distribution/pom.xml index 8eb92fed8377..d5d046f1b336 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -28,7 +28,7 @@ druid io.druid - 0.12.0 + 0.13.0-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index a0869b12dc67..a5804484bedf 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT diff --git a/extendedset/pom.xml b/extendedset/pom.xml index eecb33b46d06..aac97e228602 100755 --- a/extendedset/pom.xml +++ b/extendedset/pom.xml @@ -31,7 +31,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index 1b9bc72f66a8..fdad1c65963c 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/azure-extensions/pom.xml b/extensions-contrib/azure-extensions/pom.xml index bbe693efcc9c..9bdf4a8a6a1a 100644 --- a/extensions-contrib/azure-extensions/pom.xml +++ b/extensions-contrib/azure-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/cassandra-storage/pom.xml b/extensions-contrib/cassandra-storage/pom.xml index e801a4ba0aba..56c425c4520b 100644 --- a/extensions-contrib/cassandra-storage/pom.xml +++ b/extensions-contrib/cassandra-storage/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index 3b9ee7355bc0..5369f088ce3d 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/distinctcount/pom.xml b/extensions-contrib/distinctcount/pom.xml index ca34cfebf548..22be879de80b 100644 --- a/extensions-contrib/distinctcount/pom.xml +++ b/extensions-contrib/distinctcount/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/druid-rocketmq/pom.xml b/extensions-contrib/druid-rocketmq/pom.xml index 6122af5f5284..9bc68287152e 100644 --- a/extensions-contrib/druid-rocketmq/pom.xml +++ b/extensions-contrib/druid-rocketmq/pom.xml @@ -22,7 +22,7 @@ druid io.druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/google-extensions/pom.xml b/extensions-contrib/google-extensions/pom.xml index 13e53102d9eb..889fd343d1a6 100644 --- a/extensions-contrib/google-extensions/pom.xml +++ b/extensions-contrib/google-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index 5cf5e6c18ee1..c3b32e7bc072 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml index e55de99898bd..ec933db08366 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml +++ b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index 65b47f4fd486..fcc78fc7eebb 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/orc-extensions/pom.xml b/extensions-contrib/orc-extensions/pom.xml index 42f88cc6ef57..0ebc6fd8d19c 100644 --- a/extensions-contrib/orc-extensions/pom.xml +++ b/extensions-contrib/orc-extensions/pom.xml @@ -24,7 +24,7 @@ druid io.druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/parquet-extensions/pom.xml b/extensions-contrib/parquet-extensions/pom.xml index 9d2bb10608af..ead3c3bf78b1 100644 --- a/extensions-contrib/parquet-extensions/pom.xml +++ b/extensions-contrib/parquet-extensions/pom.xml @@ -8,7 +8,7 @@ druid io.druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/rabbitmq/pom.xml b/extensions-contrib/rabbitmq/pom.xml index d7be6ae42723..dd68599a48ba 100644 --- a/extensions-contrib/rabbitmq/pom.xml +++ b/extensions-contrib/rabbitmq/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/redis-cache/pom.xml b/extensions-contrib/redis-cache/pom.xml index c76b87faa64a..47be36eb3d73 100644 --- a/extensions-contrib/redis-cache/pom.xml +++ b/extensions-contrib/redis-cache/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/sqlserver-metadata-storage/pom.xml b/extensions-contrib/sqlserver-metadata-storage/pom.xml index 73464c40f61c..7d91c2d88207 100644 --- a/extensions-contrib/sqlserver-metadata-storage/pom.xml +++ b/extensions-contrib/sqlserver-metadata-storage/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index ef28c0aafb71..0fc55b102427 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index e231e95d8ee9..e815d046b13f 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -9,7 +9,7 @@ druid io.druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/time-min-max/pom.xml b/extensions-contrib/time-min-max/pom.xml index d2707b1c1430..1b79eb519f7b 100644 --- a/extensions-contrib/time-min-max/pom.xml +++ b/extensions-contrib/time-min-max/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/virtual-columns/pom.xml b/extensions-contrib/virtual-columns/pom.xml index e671c223e478..6095f37a87b5 100644 --- a/extensions-contrib/virtual-columns/pom.xml +++ b/extensions-contrib/virtual-columns/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index d1d7fb360773..d57142176726 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index a322e5f62c53..5db3f194b20a 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-basic-security/pom.xml b/extensions-core/druid-basic-security/pom.xml index c214081573e5..5367bfb93ef9 100644 --- a/extensions-core/druid-basic-security/pom.xml +++ b/extensions-core/druid-basic-security/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index 2d4f5b110d46..6f06a47b3d1b 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index 64ffbb6c0f6a..f737da39c3e1 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/histogram/pom.xml b/extensions-core/histogram/pom.xml index 63d3c6665dfb..ef4604a257a2 100644 --- a/extensions-core/histogram/pom.xml +++ b/extensions-core/histogram/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-eight/pom.xml b/extensions-core/kafka-eight/pom.xml index 9204cd5ead71..155aae49763b 100644 --- a/extensions-core/kafka-eight/pom.xml +++ b/extensions-core/kafka-eight/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index 1b4eeaa60efd..cc158c8d9a9f 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index 9c91250daa9c..13b15ca16478 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/lookups-cached-global/pom.xml b/extensions-core/lookups-cached-global/pom.xml index 9eadea80bb0f..80c84328b87a 100644 --- a/extensions-core/lookups-cached-global/pom.xml +++ b/extensions-core/lookups-cached-global/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/lookups-cached-single/pom.xml b/extensions-core/lookups-cached-single/pom.xml index 17bfe7a2e6ef..c0bfd3abfb87 100644 --- a/extensions-core/lookups-cached-single/pom.xml +++ b/extensions-core/lookups-cached-single/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index c24ac06570f6..a3b8fa7cd80a 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml index 3a08bbd9254a..05591967d3c0 100644 --- a/extensions-core/postgresql-metadata-storage/pom.xml +++ b/extensions-core/postgresql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/protobuf-extensions/pom.xml b/extensions-core/protobuf-extensions/pom.xml index f7d444c0b4eb..0c9f4cad1693 100644 --- a/extensions-core/protobuf-extensions/pom.xml +++ b/extensions-core/protobuf-extensions/pom.xml @@ -27,7 +27,7 @@ druid io.druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index 2c2f184a7c6a..dd84f21a182e 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/extensions-core/simple-client-sslcontext/pom.xml b/extensions-core/simple-client-sslcontext/pom.xml index 1fecaa1a56c7..c9037c4bde02 100644 --- a/extensions-core/simple-client-sslcontext/pom.xml +++ b/extensions-core/simple-client-sslcontext/pom.xml @@ -3,7 +3,7 @@ druid io.druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml index 20182abeae04..2e989c1c07f7 100644 --- a/extensions-core/stats/pom.xml +++ b/extensions-core/stats/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT ../../pom.xml diff --git a/hll/pom.xml b/hll/pom.xml index 9bcc1d937cd2..676a4eb92b7b 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT druid-hll diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 02b5cfa1ad56..e2bf002c3d55 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 850c6801dcee..cd15dc607b2d 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 138132a06e6e..7e1de402aec4 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT diff --git a/java-util/pom.xml b/java-util/pom.xml index a945a9743b56..1efd28e039ac 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT java-util diff --git a/pom.xml b/pom.xml index bfb120e2b752..b25f1c7d5252 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ druid - 0.12.0 + 0.13.0-SNAPSHOT pom ${project.groupId}:${project.artifactId} @@ -53,7 +53,7 @@ scm:git:ssh://git@github.com/druid-io/druid.git scm:git:ssh://git@github.com/druid-io/druid.git https://github.com/druid-io/druid.git - druid-0.12.0 + 0.12.0-SNAPSHOT diff --git a/processing/pom.xml b/processing/pom.xml index 5a66331e3a43..220abc4192f3 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 016d9d129269..0b9a07a3b9ef 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index b1b8223929aa..4444c2ad9297 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -25,7 +25,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT diff --git a/sql/pom.xml b/sql/pom.xml index 54b603079d01..d4eae7224b84 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.0 + 0.13.0-SNAPSHOT From f2534f004156a91480abc5299464c4e9266d39ba Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Mon, 9 Apr 2018 17:35:03 -0700 Subject: [PATCH 43/82] SQL: Throttle metadata refreshes when they fail. (#5328) (#5599) --- .../java/io/druid/sql/calcite/schema/DruidSchema.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sql/src/main/java/io/druid/sql/calcite/schema/DruidSchema.java b/sql/src/main/java/io/druid/sql/calcite/schema/DruidSchema.java index a24b13424ffb..b946afea7774 100644 --- a/sql/src/main/java/io/druid/sql/calcite/schema/DruidSchema.java +++ b/sql/src/main/java/io/druid/sql/calcite/schema/DruidSchema.java @@ -28,7 +28,6 @@ import com.google.common.collect.Sets; import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; -import io.druid.java.util.emitter.EmittingLogger; import io.druid.client.ServerView; import io.druid.client.TimelineServerView; import io.druid.guice.ManageLifecycle; @@ -40,6 +39,7 @@ import io.druid.java.util.common.guava.Yielders; import io.druid.java.util.common.lifecycle.LifecycleStart; import io.druid.java.util.common.lifecycle.LifecycleStop; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.query.TableDataSource; import io.druid.query.metadata.metadata.AllColumnIncluderator; import io.druid.query.metadata.metadata.ColumnAnalysis; @@ -121,6 +121,7 @@ public class DruidSchema extends AbstractSchema private boolean refreshImmediately = false; private long lastRefresh = 0L; + private long lastFailure = 0L; private boolean isServerViewInitialized = false; @Inject @@ -197,7 +198,13 @@ public void run() final long nextRefresh = nextRefreshNoFuzz + (long) ((nextRefreshNoFuzz - lastRefresh) * 0.10); while (true) { + // Do not refresh if it's too soon after a failure (to avoid rapid cycles of failure). + final boolean wasRecentFailure = DateTimes.utc(lastFailure) + .plus(config.getMetadataRefreshPeriod()) + .isAfterNow(); + if (isServerViewInitialized && + !wasRecentFailure && (!segmentsNeedingRefresh.isEmpty() || !dataSourcesNeedingRebuild.isEmpty()) && (refreshImmediately || nextRefresh < System.currentTimeMillis())) { break; @@ -211,6 +218,7 @@ public void run() // Mutable segments need a refresh every period, since new columns could be added dynamically. segmentsNeedingRefresh.addAll(mutableSegments); + lastFailure = 0L; lastRefresh = System.currentTimeMillis(); refreshImmediately = false; } @@ -258,6 +266,7 @@ public void run() // Add our segments and dataSources back to their refresh and rebuild lists. segmentsNeedingRefresh.addAll(segmentsToRefresh); dataSourcesNeedingRebuild.addAll(dataSourcesToRebuild); + lastFailure = System.currentTimeMillis(); lock.notifyAll(); } } From 42467796d26314578098c821d7b2ff7b646812bd Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Mon, 9 Apr 2018 17:35:17 -0700 Subject: [PATCH 44/82] Respect forceHashAggregation in queryContext (#5533) (#5609) * Respect forceHashAggregation in queryContext * address comment --- .../epinephelinae/GroupByQueryEngineV2.java | 24 +++++++++---------- .../groupby/strategy/GroupByStrategyV2.java | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java index 6db22e1efb19..3e3330cfefdb 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java @@ -91,7 +91,7 @@ public static Sequence process( final GroupByQuery query, final StorageAdapter storageAdapter, final NonBlockingPool intermediateResultsBufferPool, - final GroupByQueryConfig config + final GroupByQueryConfig querySpecificConfig ) { if (storageAdapter == null) { @@ -150,10 +150,10 @@ public GroupByEngineIterator make() final ByteBuffer buffer = bufferHolder.get(); // Check array-based aggregation is applicable - if (isArrayAggregateApplicable(config, query, dims, storageAdapter, buffer)) { + if (isArrayAggregateApplicable(querySpecificConfig, query, dims, storageAdapter, buffer)) { return new ArrayAggregateIterator( query, - config, + querySpecificConfig, cursor, buffer, fudgeTimestamp, @@ -165,7 +165,7 @@ public GroupByEngineIterator make() } else { return new HashAggregateIterator( query, - config, + querySpecificConfig, cursor, buffer, fudgeTimestamp, @@ -186,14 +186,14 @@ public void cleanup(GroupByEngineIterator iterFromMake) } private static boolean isArrayAggregateApplicable( - GroupByQueryConfig config, + GroupByQueryConfig querySpecificConfig, GroupByQuery query, GroupByColumnSelectorPlus[] dims, StorageAdapter storageAdapter, ByteBuffer buffer ) { - if (config.isForceHashAggregation()) { + if (querySpecificConfig.isForceHashAggregation()) { return false; } @@ -276,7 +276,7 @@ private abstract static class GroupByEngineIterator implements Iterator public GroupByEngineIterator( final GroupByQuery query, - final GroupByQueryConfig config, + final GroupByQueryConfig querySpecificConfig, final Cursor cursor, final ByteBuffer buffer, final DateTime fudgeTimestamp, @@ -285,7 +285,7 @@ public GroupByEngineIterator( ) { this.query = query; - this.querySpecificConfig = config.withOverrides(query); + this.querySpecificConfig = querySpecificConfig; this.cursor = cursor; this.buffer = buffer; this.keySerde = new GroupByEngineKeySerde(dims); @@ -413,7 +413,7 @@ private static class HashAggregateIterator extends GroupByEngineIterator process( StorageAdapter storageAdapter ) { - return GroupByQueryEngineV2.process(query, storageAdapter, bufferPool, configSupplier.get()); + return GroupByQueryEngineV2.process(query, storageAdapter, bufferPool, configSupplier.get().withOverrides(query)); } } From ca43968d29996935cee56a37c37bc223d8eaf00e Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Mon, 9 Apr 2018 17:35:31 -0700 Subject: [PATCH 45/82] Add overlord unsecured paths to coordinator when using combined service (#5579) (#5606) * Add overlord unsecured paths to coordinator when using combined service * PR comment --- services/src/main/java/io/druid/cli/CliOverlord.java | 3 +-- .../java/io/druid/cli/CoordinatorJettyServerInitializer.java | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/services/src/main/java/io/druid/cli/CliOverlord.java b/services/src/main/java/io/druid/cli/CliOverlord.java index ae075abd4b84..f070c8b5cdbb 100644 --- a/services/src/main/java/io/druid/cli/CliOverlord.java +++ b/services/src/main/java/io/druid/cli/CliOverlord.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; import com.google.inject.Binder; import com.google.inject.Injector; import com.google.inject.Key; @@ -116,7 +115,7 @@ public class CliOverlord extends ServerRunnable { private static Logger log = new Logger(CliOverlord.class); - private static List UNSECURED_PATHS = Lists.newArrayList( + protected static List UNSECURED_PATHS = ImmutableList.of( "/", "/console.html", "/old-console/*", diff --git a/services/src/main/java/io/druid/cli/CoordinatorJettyServerInitializer.java b/services/src/main/java/io/druid/cli/CoordinatorJettyServerInitializer.java index f47644f13478..5aefbd4182c8 100644 --- a/services/src/main/java/io/druid/cli/CoordinatorJettyServerInitializer.java +++ b/services/src/main/java/io/druid/cli/CoordinatorJettyServerInitializer.java @@ -118,6 +118,10 @@ public void initialize(Server server, Injector injector) // perform no-op authorization for these resources AuthenticationUtils.addNoopAuthorizationFilters(root, UNSECURED_PATHS); + if (beOverlord) { + AuthenticationUtils.addNoopAuthorizationFilters(root, CliOverlord.UNSECURED_PATHS); + } + authenticators = authenticatorMapper.getAuthenticatorChain(); AuthenticationUtils.addAuthenticationFilterChain(root, authenticators); From d10c9caf1c7665d24f4da39df32cb9f588bc2a5e Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Mon, 9 Apr 2018 17:35:42 -0700 Subject: [PATCH 46/82] pass configuration from context into JobConf for determining DatasourceInputFormat splits (#5408) (#5601) --- .../indexer/hadoop/DatasourceInputFormat.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/DatasourceInputFormat.java b/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/DatasourceInputFormat.java index b4856a925fca..a93459f61637 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/DatasourceInputFormat.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/hadoop/DatasourceInputFormat.java @@ -29,7 +29,6 @@ import io.druid.indexer.JobHelper; import io.druid.java.util.common.ISE; import io.druid.java.util.common.logger.Logger; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -65,7 +64,7 @@ public class DatasourceInputFormat extends InputFormat @Override public List getSplits(JobContext context) throws IOException, InterruptedException { - Configuration conf = context.getConfiguration(); + JobConf conf = new JobConf(context.getConfiguration()); String segmentsStr = Preconditions.checkNotNull( conf.get(CONF_INPUT_SEGMENTS), @@ -89,7 +88,7 @@ public List getSplits(JobContext context) throws IOException, Interr for (WindowedDataSegment segment : segments) { totalSize += segment.getSegment().getSize(); } - int mapTask = ((JobConf) conf).getNumMapTasks(); + int mapTask = conf.getNumMapTasks(); if (mapTask > 0) { maxSize = totalSize / mapTask; } @@ -116,11 +115,10 @@ public int compare(WindowedDataSegment s1, WindowedDataSegment s2) List list = new ArrayList<>(); long size = 0; - JobConf dummyConf = new JobConf(); org.apache.hadoop.mapred.InputFormat fio = supplier.get(); for (WindowedDataSegment segment : segments) { if (size + segment.getSegment().getSize() > maxSize && size > 0) { - splits.add(toDataSourceSplit(list, fio, dummyConf)); + splits.add(toDataSourceSplit(list, fio, conf)); list = Lists.newArrayList(); size = 0; } @@ -130,7 +128,7 @@ public int compare(WindowedDataSegment s1, WindowedDataSegment s2) } if (list.size() > 0) { - splits.add(toDataSourceSplit(list, fio, dummyConf)); + splits.add(toDataSourceSplit(list, fio, conf)); } logger.info("Number of splits [%d]", splits.size()); @@ -214,14 +212,14 @@ static Stream getLocations( try { return Arrays.stream(split.getLocations()); } - catch (final IOException e) { + catch (final Exception e) { logger.error(e, "Exception getting locations"); return Stream.empty(); } } ); } - catch (final IOException e) { + catch (final Exception e) { logger.error(e, "Exception getting splits"); return Stream.empty(); } From 561b1b9cfdb28ef562958a2841a040cd638d2cfa Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Mon, 9 Apr 2018 17:35:55 -0700 Subject: [PATCH 47/82] Lookups: Inherit "injective" from registered lookups, improve docs. (#5316) (#5600) Code changes: - In the lookup-based extractionFns, inherit injective property from the lookup itself if not specified. Doc changes: - Add a "Query execution" section to the lookups doc explaining how injective lookups and their optimizations work. - Remove scary warnings against using registeredLookup extractionFns. They are necessary and important since they work with filters and function cascades -- two things that the dimension specs do not do. They deserve to be first class citizens. - Move the "registeredLookup" fn above the "lookup" fn. It's probably more commonly used, so the docs read better this way. --- .../extensions-core/lookups-cached-global.md | 2 +- docs/content/querying/dimensionspecs.md | 121 ++++++++---------- docs/content/querying/lookups.md | 52 ++++++-- .../query/lookup/LookupExtractionFn.java | 19 +-- .../query/lookup/LookupExtractionFnTest.java | 29 ++--- .../lookup/RegisteredLookupExtractionFn.java | 53 +++----- .../RegisteredLookupExtractionFnTest.java | 26 +++- 7 files changed, 161 insertions(+), 141 deletions(-) diff --git a/docs/content/development/extensions-core/lookups-cached-global.md b/docs/content/development/extensions-core/lookups-cached-global.md index 7e4b11af8437..a5e7d5eb04ff 100644 --- a/docs/content/development/extensions-core/lookups-cached-global.md +++ b/docs/content/development/extensions-core/lookups-cached-global.md @@ -71,7 +71,7 @@ The parameters are as follows |--------|-----------|--------|-------| |`extractionNamespace`|Specifies how to populate the local cache. See below|Yes|-| |`firstCacheTimeout`|How long to wait (in ms) for the first run of the cache to populate. 0 indicates to not wait|No|`0` (do not wait)| -|`injective`|If the underlying map is injective (keys and values are unique) then optimizations can occur internally by setting this to `true`|No|`false`| +|`injective`|If the underlying map is [injective](../../querying/lookups.html#query-execution) (keys and values are unique) then optimizations can occur internally by setting this to `true`|No|`false`| If `firstCacheTimeout` is set to a non-zero value, it should be less than `druid.manager.lookups.hostUpdateTimeout`. If `firstCacheTimeout` is NOT set, then management is essentially asynchronous and does not know if a lookup succeeded or failed in starting. In such a case logs from the lookup nodes should be monitored for repeated failures. diff --git a/docs/content/querying/dimensionspecs.md b/docs/content/querying/dimensionspecs.md index 236e13b5a7a0..19d080493fe1 100644 --- a/docs/content/querying/dimensionspecs.md +++ b/docs/content/querying/dimensionspecs.md @@ -95,8 +95,6 @@ The default values are `replaceMissingValueWith = null` and `retainMissingValue It is illegal to set `retainMissingValue = true` and also specify a `replaceMissingValueWith`. -A property of `injective` specifies if optimizations can be used which assume there is no combining of multiple names into one. For example: If ABC123 is the only key that maps to SomeCompany, that can be optimized since it is a unique lookup. But if both ABC123 and DEF456 BOTH map to SomeCompany, then that is NOT a unique lookup. Setting this value to true and setting `retainMissingValue` to FALSE (the default) may cause undesired behavior. - A property `optimize` can be supplied to allow optimization of lookup based extraction filter (by default `optimize = true`). The second kind where it is not possible to pass at query time due to their size, will be based on an external lookup table or resource that is already registered via configuration file or/and coordinator. @@ -316,67 +314,38 @@ Example for the `__time` dimension: JavaScript-based functionality is disabled by default. Please refer to the Druid JavaScript programming guide for guidelines about using Druid's JavaScript functionality, including instructions on how to enable it. -### Lookup extraction function +### Registered lookup extraction function -Lookups are a concept in Druid where dimension values are (optionally) replaced with new values. -For more documentation on using lookups, please see [here](../querying/lookups.html). -Explicit lookups allow you to specify a set of keys and values to use when performing the extraction. +Lookups are a concept in Druid where dimension values are (optionally) replaced with new values. +For more documentation on using lookups, please see [Lookups](../querying/lookups.html). +The "registeredLookup" extraction function lets you refer to a lookup that has been registered in the cluster-wide +configuration. -```json -{ - "type":"lookup", - "lookup":{ - "type":"map", - "map":{"foo":"bar", "baz":"bat"} - }, - "retainMissingValue":true, - "injective":true -} -``` - -```json -{ - "type":"lookup", - "lookup":{ - "type":"map", - "map":{"foo":"bar", "baz":"bat"} - }, - "retainMissingValue":false, - "injective":false, - "replaceMissingValueWith":"MISSING" -} -``` - -```json -{ - "type":"lookup", - "lookup":{"type":"namespace","namespace":"some_lookup"}, - "replaceMissingValueWith":"Unknown", - "injective":false -} -``` +An example: ```json { - "type":"lookup", - "lookup":{"type":"namespace","namespace":"some_lookup"}, - "retainMissingValue":true, - "injective":false + "type":"registeredLookup", + "lookup":"some_lookup_name", + "retainMissingValue":true } ``` -A lookup can be of type `namespace` or `map`. A `map` lookup is passed as part of the query. -A `namespace` lookup is populated on all the nodes which handle queries as per [lookups](../querying/lookups.html) +A property of `retainMissingValue` and `replaceMissingValueWith` can be specified at query time to hint how to handle +missing values. Setting `replaceMissingValueWith` to `""` has the same effect as setting it to `null` or omitting the +property. Setting `retainMissingValue` to true will use the dimension's original value if it is not found in the lookup. +The default values are `replaceMissingValueWith = null` and `retainMissingValue = false` which causes missing values to +be treated as missing. -A property of `retainMissingValue` and `replaceMissingValueWith` can be specified at query time to hint how to handle missing values. Setting `replaceMissingValueWith` to `""` has the same effect as setting it to `null` or omitting the property. Setting `retainMissingValue` to true will use the dimension's original value if it is not found in the lookup. The default values are `replaceMissingValueWith = null` and `retainMissingValue = false` which causes missing values to be treated as missing. - It is illegal to set `retainMissingValue = true` and also specify a `replaceMissingValueWith`. -A property of `injective` specifies if optimizations can be used which assume there is no combining of multiple names into one. For example: If ABC123 is the only key that maps to SomeCompany, that can be optimized since it is a unique lookup. But if both ABC123 and DEF456 BOTH map to SomeCompany, then that is NOT a unique lookup. Setting this value to true and setting `retainMissingValue` to FALSE (the default) may cause undesired behavior. +A property of `injective` can override the lookup's own sense of whether or not it is +[injective](lookups.html#query-execution). If left unspecified, Druid will use the registered cluster-wide lookup +configuration. -A property `optimize` can be supplied to allow optimization of lookup based extraction filter (by default `optimize = true`). +A property `optimize` can be supplied to allow optimization of lookup based extraction filter (by default `optimize = true`). The optimization layer will run on the broker and it will rewrite the extraction filter as clause of selector filters. -For instance the following filter +For instance the following filter ```json { @@ -385,21 +354,16 @@ For instance the following filter "dimension": "product", "value": "bar_1", "extractionFn": { - "type": "lookup", + "type": "registeredLookup", "optimize": true, - "lookup": { - "type": "map", - "map": { - "product_1": "bar_1", - "product_3": "bar_1" - } - } + "lookup": "some_lookup_name" } } } ``` -will be rewritten as +will be rewritten as the following simpler query, assuming a lookup that maps "product_1" and "product_3" to the value +"bar_1": ```json { @@ -425,30 +389,49 @@ will be rewritten as } ``` -A null dimension value can be mapped to a specific value by specifying the empty string as the key. +A null dimension value can be mapped to a specific value by specifying the empty string as the key in your lookup file. This allows distinguishing between a null dimension and a lookup resulting in a null. For example, specifying `{"":"bar","bat":"baz"}` with dimension values `[null, "foo", "bat"]` and replacing missing values with `"oof"` will yield results of `["bar", "oof", "baz"]`. Omitting the empty string key will cause the missing value to take over. For example, specifying `{"bat":"baz"}` with dimension values `[null, "foo", "bat"]` and replacing missing values with `"oof"` will yield results of `["oof", "oof", "baz"]`. -### Registered Lookup Extraction Function +### Inline lookup extraction function -While it is recommended that the [lookup dimension spec](#lookup-dimensionspecs) be used whenever possible, any lookup that is registered for use as a lookup dimension spec can be used as a dimension extraction. +Lookups are a concept in Druid where dimension values are (optionally) replaced with new values. +For more documentation on using lookups, please see [Lookups](../querying/lookups.html). +The "lookup" extraction function lets you specify an inline lookup map without registering one in the cluster-wide +configuration. -The specification for dimension extraction using dimension specification named lookups is formatted as per the following example: +Examples: ```json { - "type":"registeredLookup", - "lookup":"some_lookup_name", + "type":"lookup", + "lookup":{ + "type":"map", + "map":{"foo":"bar", "baz":"bat"} + }, "retainMissingValue":true, - "injective":false + "injective":true +} +``` + +```json +{ + "type":"lookup", + "lookup":{ + "type":"map", + "map":{"foo":"bar", "baz":"bat"} + }, + "retainMissingValue":false, + "injective":false, + "replaceMissingValueWith":"MISSING" } ``` -All the flags for [lookup extraction function](#lookup-extraction-function) apply here as well. +The inline lookup should be of type `map`. -In general, the dimension specification should be used. This dimension **extraction** implementation is made available for testing, validation, and transitioning from dimension extraction to the dimension specification style lookups. -There is also a chance that a feature uses dimension extraction in such a way that it is not applied to dimension specification lookups. Such a scenario should be brought to the attention of the development mailing list. +The properties `retainMissingValue`, `replaceMissingValueWith`, `injective`, and `optimize` behave similarly to the +[registered lookup extraction function](#registered-lookup-extraction-function). ### Cascade Extraction Function diff --git a/docs/content/querying/lookups.md b/docs/content/querying/lookups.md index 616b602eb3ba..771c9adfb743 100644 --- a/docs/content/querying/lookups.md +++ b/docs/content/querying/lookups.md @@ -8,15 +8,17 @@ layout: doc_page Lookups are an experimental feature. -Lookups are a concept in Druid where dimension values are (optionally) replaced with new values. -See [dimension specs](../querying/dimensionspecs.html) for more information. For the purpose of these documents, -a "key" refers to a dimension value to match, and a "value" refers to its replacement. -So if you wanted to rename `appid-12345` to `Super Mega Awesome App` then the key would be `appid-12345` and the value -would be `Super Mega Awesome App`. - -It is worth noting that lookups support use cases where keys map to unique values (injective) such as a country code and -a country name, and also supports use cases where multiple IDs map to the same value, e.g. multiple app-ids belonging to -a single account manager. +Lookups are a concept in Druid where dimension values are (optionally) replaced with new values, allowing join-like +functionality. Applying lookups in Druid is similar to joining a dimension table in a data warehouse. See +[dimension specs](../querying/dimensionspecs.html) for more information. For the purpose of these documents, a "key" +refers to a dimension value to match, and a "value" refers to its replacement. So if you wanted to map +`appid-12345` to `Super Mega Awesome App` then the key would be `appid-12345` and the value would be +`Super Mega Awesome App`. + +It is worth noting that lookups support not just use cases where keys map one-to-one to unique values, such as country +code and country name, but also support use cases where multiple IDs map to the same value, e.g. multiple app-ids +mapping to a single account manager. When lookups are one-to-one, Druid is able to apply additional optimizations at +query time; see [Query execution](#query-execution) below for more details. Lookups do not have history. They always use the current data. This means that if the chief account manager for a particular app-id changes, and you issue a query with a lookup to store the app-id to account manager relationship, @@ -33,6 +35,38 @@ Other lookup types are available as extensions, including: - Globally cached lookups from local files, remote URIs, or JDBC through [lookups-cached-global](../development/extensions-core/lookups-cached-global.html). - Globally cached lookups from a Kafka topic through [kafka-extraction-namespace](../development/extensions-core/kafka-extraction-namespace.html). +Query Execution +--------------- +When executing an aggregation query involving lookups, Druid can decide to apply lookups either while scanning and +aggregating rows, or to apply them after aggregation is complete. It is more efficient to apply lookups after +aggregation is complete, so Druid will do this if it can. Druid decides this by checking if the lookup is marked +as "injective" or not. In general, you should set this property for any lookup that is naturally one-to-one, to allow +Druid to run your queries as fast as possible. + +Injective lookups should include _all_ possible keys that may show up in your dataset, and should also map all keys to +_unique values_. This matters because non-injective lookups may map different keys to the same value, which must be +accounted for during aggregation, lest query results contain two result values that should have been aggregated into +one. + +This lookup is injective (assuming it contains all possible keys from your data): + +``` +1 -> Foo +2 -> Bar +3 -> Billy +``` + +But this one is not, since both "2" and "3" map to the same key: + +``` +1 -> Foo +2 -> Bar +3 -> Bar +``` + +To tell Druid that your lookup is injective, you must specify `"injective" : true` in the lookup configuration. Druid +will not detect this automatically. + Dynamic Configuration ---------------------
    diff --git a/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java b/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java index b0c65a7e098f..1c2605fe9853 100644 --- a/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java +++ b/processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java @@ -37,18 +37,14 @@ public class LookupExtractionFn extends FunctionalExtraction { private final LookupExtractor lookup; private final boolean optimize; - // Thes are retained for auto generated hashCode and Equals - private final boolean retainMissingValue; - private final String replaceMissingValueWith; - private final boolean injective; @JsonCreator public LookupExtractionFn( @JsonProperty("lookup") final LookupExtractor lookup, @JsonProperty("retainMissingValue") final boolean retainMissingValue, @Nullable @JsonProperty("replaceMissingValueWith") final String replaceMissingValueWith, - @JsonProperty("injective") final boolean injective, - @JsonProperty("optimize") Boolean optimize + @JsonProperty("injective") final Boolean injective, + @JsonProperty("optimize") final Boolean optimize ) { super( @@ -63,13 +59,10 @@ public String apply(String input) }, retainMissingValue, replaceMissingValueWith, - injective + injective != null ? injective : lookup.isOneToOne() ); this.lookup = lookup; this.optimize = optimize == null ? true : optimize; - this.retainMissingValue = retainMissingValue; - this.injective = injective; - this.replaceMissingValueWith = replaceMissingValueWith; } @@ -175,9 +168,9 @@ public String toString() return "LookupExtractionFn{" + "lookup=" + lookup + ", optimize=" + optimize + - ", retainMissingValue=" + retainMissingValue + - ", replaceMissingValueWith='" + replaceMissingValueWith + '\'' + - ", injective=" + injective + + ", retainMissingValue=" + isRetainMissingValue() + + ", replaceMissingValueWith='" + getReplaceMissingValueWith() + '\'' + + ", injective=" + isInjective() + '}'; } } diff --git a/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnTest.java b/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnTest.java index d60f728764e9..aea8bf626d6d 100644 --- a/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/lookup/LookupExtractionFnTest.java @@ -20,7 +20,6 @@ package io.druid.query.lookup; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Function; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -36,11 +35,11 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import javax.annotation.Nullable; import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; @RunWith(Parameterized.class) public class LookupExtractionFnTest @@ -53,30 +52,23 @@ public static Iterable constructorFeeder() ImmutableList.of( ImmutableSet.of(true, false), ImmutableSet.of("", "MISSING VALUE"), - ImmutableSet.of(true, false) + ImmutableSet.of(Optional.of(true), Optional.of(false), Optional.empty()) ) - ), new Function, Object[]>() - { - @Nullable - @Override - public Object[] apply(List input) - { - return input.toArray(); - } - } + ), + List::toArray ); } private static final ObjectMapper OBJECT_MAPPER = new DefaultObjectMapper(); private final boolean retainMissing; private final String replaceMissing; - private final boolean injective; + private final Boolean injective; - public LookupExtractionFnTest(boolean retainMissing, String replaceMissing, boolean injective) + public LookupExtractionFnTest(boolean retainMissing, String replaceMissing, Optional injective) { this.replaceMissing = Strings.emptyToNull(replaceMissing); this.retainMissing = retainMissing; - this.injective = injective; + this.injective = injective.orElse(null); } @Test @@ -136,7 +128,12 @@ public void testSimpleSerDe() throws IOException Assert.assertEquals(retainMissing, lookupExtractionFn2.isRetainMissingValue()); Assert.assertEquals(replaceMissing, lookupExtractionFn2.getReplaceMissingValueWith()); - Assert.assertEquals(injective, lookupExtractionFn2.isInjective()); + + if (injective == null) { + Assert.assertEquals(lookupExtractionFn2.getLookup().isOneToOne(), lookupExtractionFn2.isInjective()); + } else { + Assert.assertEquals(injective, lookupExtractionFn2.isInjective()); + } Assert.assertArrayEquals(lookupExtractionFn.getCacheKey(), lookupExtractionFn2.getCacheKey()); diff --git a/server/src/main/java/io/druid/query/lookup/RegisteredLookupExtractionFn.java b/server/src/main/java/io/druid/query/lookup/RegisteredLookupExtractionFn.java index 525f72dcb78a..d0fd34cf2365 100644 --- a/server/src/main/java/io/druid/query/lookup/RegisteredLookupExtractionFn.java +++ b/server/src/main/java/io/druid/query/lookup/RegisteredLookupExtractionFn.java @@ -28,6 +28,7 @@ import javax.annotation.Nullable; import java.nio.ByteBuffer; +import java.util.Objects; public class RegisteredLookupExtractionFn implements ExtractionFn { @@ -38,7 +39,7 @@ public class RegisteredLookupExtractionFn implements ExtractionFn private final String lookup; private final boolean retainMissingValue; private final String replaceMissingValueWith; - private final boolean injective; + private final Boolean injective; private final boolean optimize; @JsonCreator @@ -47,7 +48,7 @@ public RegisteredLookupExtractionFn( @JsonProperty("lookup") String lookup, @JsonProperty("retainMissingValue") final boolean retainMissingValue, @Nullable @JsonProperty("replaceMissingValueWith") final String replaceMissingValueWith, - @JsonProperty("injective") final boolean injective, + @JsonProperty("injective") final Boolean injective, @JsonProperty("optimize") Boolean optimize ) { @@ -79,7 +80,7 @@ public String getReplaceMissingValueWith() } @JsonProperty("injective") - public boolean isInjective() + public Boolean isInjective() { return injective; } @@ -142,13 +143,17 @@ private LookupExtractionFn ensureDelegate() // http://www.javamex.com/tutorials/double_checked_locking.shtml synchronized (delegateLock) { if (null == delegate) { + final LookupExtractor factory = Preconditions.checkNotNull( + manager.get(getLookup()), + "Lookup [%s] not found", + getLookup() + ).getLookupExtractorFactory().get(); + delegate = new LookupExtractionFn( - Preconditions.checkNotNull(manager.get(getLookup()), "Lookup [%s] not found", getLookup()) - .getLookupExtractorFactory() - .get(), + factory, isRetainMissingValue(), getReplaceMissingValueWith(), - isInjective(), + injective == null ? factory.isOneToOne() : injective, isOptimize() ); } @@ -158,7 +163,7 @@ private LookupExtractionFn ensureDelegate() } @Override - public boolean equals(Object o) + public boolean equals(final Object o) { if (this == o) { return true; @@ -166,35 +171,19 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - - RegisteredLookupExtractionFn that = (RegisteredLookupExtractionFn) o; - - if (isRetainMissingValue() != that.isRetainMissingValue()) { - return false; - } - if (isInjective() != that.isInjective()) { - return false; - } - if (isOptimize() != that.isOptimize()) { - return false; - } - if (!getLookup().equals(that.getLookup())) { - return false; - } - return getReplaceMissingValueWith() != null - ? getReplaceMissingValueWith().equals(that.getReplaceMissingValueWith()) - : that.getReplaceMissingValueWith() == null; + final RegisteredLookupExtractionFn that = (RegisteredLookupExtractionFn) o; + return retainMissingValue == that.retainMissingValue && + optimize == that.optimize && + Objects.equals(lookup, that.lookup) && + Objects.equals(replaceMissingValueWith, that.replaceMissingValueWith) && + Objects.equals(injective, that.injective); } @Override public int hashCode() { - int result = getLookup().hashCode(); - result = 31 * result + (isRetainMissingValue() ? 1 : 0); - result = 31 * result + (getReplaceMissingValueWith() != null ? getReplaceMissingValueWith().hashCode() : 0); - result = 31 * result + (isInjective() ? 1 : 0); - result = 31 * result + (isOptimize() ? 1 : 0); - return result; + + return Objects.hash(lookup, retainMissingValue, replaceMissingValueWith, injective, optimize); } @Override diff --git a/server/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java b/server/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java index f1332e21ec5f..0e990d89e37e 100644 --- a/server/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java +++ b/server/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.jackson.JacksonUtils; +import io.druid.query.extraction.ExtractionFn; import io.druid.query.extraction.MapLookupExtractor; import org.easymock.EasyMock; import org.junit.Assert; @@ -57,16 +58,39 @@ public void testSimpleDelegation() LOOKUP_NAME, true, null, - true, + false, false ); EasyMock.verify(manager); + + Assert.assertEquals(false, fn.isInjective()); + Assert.assertEquals(ExtractionFn.ExtractionType.MANY_TO_ONE, fn.getExtractionType()); + for (String orig : Arrays.asList("", "foo", "bat")) { Assert.assertEquals(LOOKUP_EXTRACTOR.apply(orig), fn.apply(orig)); } Assert.assertEquals("not in the map", fn.apply("not in the map")); } + @Test + public void testInheritInjective() + { + final LookupReferencesManager manager = EasyMock.createStrictMock(LookupReferencesManager.class); + managerReturnsMap(manager); + EasyMock.replay(manager); + final RegisteredLookupExtractionFn fn = new RegisteredLookupExtractionFn( + manager, + LOOKUP_NAME, + true, + null, + null, + false + ); + EasyMock.verify(manager); + + Assert.assertNull(fn.isInjective()); + Assert.assertEquals(ExtractionFn.ExtractionType.ONE_TO_ONE, fn.getExtractionType()); + } @Test public void testMissingDelegation() From d39a73e9370dfaec17d0b08d32a29931099eaab4 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Mon, 9 Apr 2018 17:36:36 -0700 Subject: [PATCH 48/82] More memory limiting for HttpPostEmitter (#5300) (#5602) * More memory limiting for HttpPostEmitter * Less aggressive large events test * Fix tests * Restrict batch queue size first, keep minimum of 2 queue items --- docs/content/configuration/index.md | 4 +- .../emitter/core/BaseHttpEmittingConfig.java | 32 ++++- .../util/emitter/core/HttpPostEmitter.java | 2 + .../java/util/emitter/core/EmitterTest.java | 4 + .../emitter/core/HttpEmitterConfigTest.java | 43 ++++++- .../core/HttpPostEmitterStressTest.java | 116 ++++++++++++++++++ .../ParametrizedUriEmitterConfigTest.java | 7 +- 7 files changed, 199 insertions(+), 9 deletions(-) diff --git a/docs/content/configuration/index.md b/docs/content/configuration/index.md index 6719c574edf0..02a0d20e9de2 100644 --- a/docs/content/configuration/index.md +++ b/docs/content/configuration/index.md @@ -222,8 +222,8 @@ The Druid servers [emit various metrics](../operations/metrics.html) and alerts |`druid.emitter.http.basicAuthentication`|Login and password for authentification in "login:password" form, e. g. `druid.emitter.http.basicAuthentication=admin:adminpassword`|not specified = no authentification| |`druid.emitter.http.flushTimeOut|The timeout after which an event should be sent to the endpoint, even if internal buffers are not filled, in milliseconds.|not specified = no timeout| |`druid.emitter.http.batchingStrategy`|The strategy of how the batch is formatted. "ARRAY" means `[event1,event2]`, "NEWLINES" means `event1\nevent2`, ONLY_EVENTS means `event1event2`.|ARRAY| -|`druid.emitter.http.maxBatchSize`|The maximum batch size, in bytes.|5191680 (i. e. 5 MB)| -|`druid.emitter.http.batchQueueSizeLimit`|The maximum number of batches in emitter queue, if there are problems with emitting.|50| +|`druid.emitter.http.maxBatchSize`|The maximum batch size, in bytes.|the minimum of (10% of JVM heap size divided by 2) or (5191680 (i. e. 5 MB))| +|`druid.emitter.http.batchQueueSizeLimit`|The maximum number of batches in emitter queue, if there are problems with emitting.|the maximum of (2) or (10% of the JVM heap size divided by 5MB)| |`druid.emitter.http.minHttpTimeoutMillis`|If the speed of filling batches imposes timeout smaller than that, not even trying to send batch to endpoint, because it will likely fail, not being able to send the data that fast. Configure this depending based on emitter/successfulSending/minTimeMs metric. Reasonable values are 10ms..100ms.|0| |`druid.emitter.http.recipientBaseUrl`|The base URL to emit messages to. Druid will POST JSON to be consumed at the HTTP endpoint specified by this property.|none, required config| diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/BaseHttpEmittingConfig.java b/java-util/src/main/java/io/druid/java/util/emitter/core/BaseHttpEmittingConfig.java index 7078a3bf679e..9838bfb1903a 100644 --- a/java-util/src/main/java/io/druid/java/util/emitter/core/BaseHttpEmittingConfig.java +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/BaseHttpEmittingConfig.java @@ -20,6 +20,7 @@ package io.druid.java.util.emitter.core; import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.java.util.common.Pair; import javax.validation.constraints.Min; @@ -27,7 +28,16 @@ public class BaseHttpEmittingConfig { public static final long DEFAULT_FLUSH_MILLIS = 60 * 1000; public static final int DEFAULT_FLUSH_COUNTS = 500; - public static final int DEFAULT_MAX_BATCH_SIZE = 5 * 1024 * 1024; + + /** ensure the event buffers don't use more than 10% of memory by default */ + public static final int DEFAULT_MAX_BATCH_SIZE; + public static final int DEFAULT_BATCH_QUEUE_SIZE_LIMIT; + static { + Pair batchConfigPair = getDefaultBatchSizeAndLimit(Runtime.getRuntime().maxMemory()); + DEFAULT_MAX_BATCH_SIZE = batchConfigPair.lhs; + DEFAULT_BATCH_QUEUE_SIZE_LIMIT = batchConfigPair.rhs; + } + /** * Do not time out in case flushTimeOut is not set */ @@ -35,13 +45,31 @@ public class BaseHttpEmittingConfig public static final String DEFAULT_BASIC_AUTHENTICATION = null; public static final BatchingStrategy DEFAULT_BATCHING_STRATEGY = BatchingStrategy.ARRAY; public static final ContentEncoding DEFAULT_CONTENT_ENCODING = null; - public static final int DEFAULT_BATCH_QUEUE_SIZE_LIMIT = 50; public static final float DEFAULT_HTTP_TIMEOUT_ALLOWANCE_FACTOR = 2.0f; /** * The default value effective doesn't set the min timeout */ public static final int DEFAULT_MIN_HTTP_TIMEOUT_MILLIS = 0; + public static Pair getDefaultBatchSizeAndLimit(long maxMemory) + { + long memoryLimit = maxMemory / 10; + long batchSize = 5 * 1024 * 1024; + long queueLimit = 50; + + if (batchSize * queueLimit > memoryLimit) { + queueLimit = memoryLimit / batchSize; + } + + // make room for at least two queue items + if (queueLimit < 2) { + queueLimit = 2; + batchSize = memoryLimit / queueLimit; + } + + return new Pair<>((int) batchSize, (int) queueLimit); + } + @Min(1) @JsonProperty long flushMillis = DEFAULT_FLUSH_MILLIS; diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java b/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java index f954d458bf75..3db529d764cc 100644 --- a/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java @@ -292,6 +292,7 @@ private void writeLargeEvent(byte[] eventBytes) ); } else { largeEventsToEmit.add(eventBytes); + approximateBuffersToEmitCount.incrementAndGet(); approximateLargeEventsToEmitCount.incrementAndGet(); approximateEventsToEmitCount.incrementAndGet(); } @@ -609,6 +610,7 @@ private void emitLargeEvents() largeEventsToEmit.add(LARGE_EVENTS_STOP); for (byte[] largeEvent; (largeEvent = largeEventsToEmit.poll()) != LARGE_EVENTS_STOP; ) { emitLargeEvent(largeEvent); + approximateBuffersToEmitCount.decrementAndGet(); approximateLargeEventsToEmitCount.decrementAndGet(); approximateEventsToEmitCount.decrementAndGet(); } diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/EmitterTest.java b/java-util/src/test/java/io/druid/java/util/emitter/core/EmitterTest.java index b4eca7ab8279..ac537fb4b191 100644 --- a/java-util/src/test/java/io/druid/java/util/emitter/core/EmitterTest.java +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/EmitterTest.java @@ -65,6 +65,10 @@ public class EmitterTest .accumulate(new EagerResponseBodyPart(Unpooled.wrappedBuffer("Yay".getBytes(StandardCharsets.UTF_8)), true)) .build(); + public static final Response BAD_RESPONSE = responseBuilder(HttpVersion.HTTP_1_1, HttpResponseStatus.FORBIDDEN) + .accumulate(new EagerResponseBodyPart(Unpooled.wrappedBuffer("Not yay".getBytes(StandardCharsets.UTF_8)), true)) + .build(); + private static Response.ResponseBuilder responseBuilder(HttpVersion version, HttpResponseStatus status) { return new Response.ResponseBuilder() diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/HttpEmitterConfigTest.java b/java-util/src/test/java/io/druid/java/util/emitter/core/HttpEmitterConfigTest.java index a80740f9eb4e..b94f92accda4 100644 --- a/java-util/src/test/java/io/druid/java/util/emitter/core/HttpEmitterConfigTest.java +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/HttpEmitterConfigTest.java @@ -20,6 +20,7 @@ package io.druid.java.util.emitter.core; import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.java.util.common.Pair; import org.junit.Assert; import org.junit.Test; @@ -44,9 +45,12 @@ public void testDefaults() Assert.assertEquals("http://example.com/", config.getRecipientBaseUrl()); Assert.assertEquals(null, config.getBasicAuthentication()); Assert.assertEquals(BatchingStrategy.ARRAY, config.getBatchingStrategy()); - Assert.assertEquals(5 * 1024 * 1024, config.getMaxBatchSize()); + Pair batchConfigPair = BaseHttpEmittingConfig.getDefaultBatchSizeAndLimit( + Runtime.getRuntime().maxMemory() + ); + Assert.assertEquals(batchConfigPair.lhs.intValue(), config.getMaxBatchSize()); + Assert.assertEquals(batchConfigPair.rhs.intValue(), config.getBatchQueueSizeLimit()); Assert.assertEquals(Long.MAX_VALUE, config.getFlushTimeOut()); - Assert.assertEquals(50, config.getBatchQueueSizeLimit()); Assert.assertEquals(2.0f, config.getHttpTimeoutAllowanceFactor(), 0.0f); Assert.assertEquals(0, config.getMinHttpTimeoutMillis()); } @@ -65,9 +69,12 @@ public void testDefaultsLegacy() Assert.assertEquals("http://example.com/", config.getRecipientBaseUrl()); Assert.assertEquals(null, config.getBasicAuthentication()); Assert.assertEquals(BatchingStrategy.ARRAY, config.getBatchingStrategy()); - Assert.assertEquals(5 * 1024 * 1024, config.getMaxBatchSize()); + Pair batchConfigPair = BaseHttpEmittingConfig.getDefaultBatchSizeAndLimit( + Runtime.getRuntime().maxMemory() + ); + Assert.assertEquals(batchConfigPair.lhs.intValue(), config.getMaxBatchSize()); + Assert.assertEquals(batchConfigPair.rhs.intValue(), config.getBatchQueueSizeLimit()); Assert.assertEquals(Long.MAX_VALUE, config.getFlushTimeOut()); - Assert.assertEquals(50, config.getBatchQueueSizeLimit()); Assert.assertEquals(2.0f, config.getHttpTimeoutAllowanceFactor(), 0.0f); Assert.assertEquals(0, config.getMinHttpTimeoutMillis()); } @@ -134,4 +141,32 @@ public void testSettingEverythingLegacy() Assert.assertEquals(3.0f, config.getHttpTimeoutAllowanceFactor(), 0.0f); Assert.assertEquals(100, config.getMinHttpTimeoutMillis()); } + + @Test + public void testMemoryLimits() + { + Pair batchConfigPair = BaseHttpEmittingConfig.getDefaultBatchSizeAndLimit( + 64 * 1024 * 1024 + ); + Assert.assertEquals(3355443, batchConfigPair.lhs.intValue()); + Assert.assertEquals(2, batchConfigPair.rhs.intValue()); + + Pair batchConfigPair2 = BaseHttpEmittingConfig.getDefaultBatchSizeAndLimit( + 128 * 1024 * 1024 + ); + Assert.assertEquals(5242880, batchConfigPair2.lhs.intValue()); + Assert.assertEquals(2, batchConfigPair2.rhs.intValue()); + + Pair batchConfigPair3 = BaseHttpEmittingConfig.getDefaultBatchSizeAndLimit( + 256 * 1024 * 1024 + ); + Assert.assertEquals(5242880, batchConfigPair3.lhs.intValue()); + Assert.assertEquals(5, batchConfigPair3.rhs.intValue()); + + Pair batchConfigPair4 = BaseHttpEmittingConfig.getDefaultBatchSizeAndLimit( + Long.MAX_VALUE + ); + Assert.assertEquals(5242880, batchConfigPair4.lhs.intValue()); + Assert.assertEquals(50, batchConfigPair4.rhs.intValue()); + } } diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/HttpPostEmitterStressTest.java b/java-util/src/test/java/io/druid/java/util/emitter/core/HttpPostEmitterStressTest.java index 06ad38c9751e..a467d366a000 100644 --- a/java-util/src/test/java/io/druid/java/util/emitter/core/HttpPostEmitterStressTest.java +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/HttpPostEmitterStressTest.java @@ -23,16 +23,19 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.primitives.Ints; import com.google.common.util.concurrent.Futures; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.ints.IntList; import org.asynchttpclient.ListenableFuture; import org.asynchttpclient.Request; import org.asynchttpclient.Response; +import org.junit.Assert; import org.junit.Test; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.BitSet; import java.util.List; import java.util.concurrent.CountDownLatch; @@ -140,4 +143,117 @@ public void run() } } } + + @Test + public void testLargeEventsQueueLimit() throws InterruptedException, IOException + { + ObjectMapper mapper = new ObjectMapper(); + + HttpEmitterConfig config = new HttpEmitterConfig.Builder("http://foo.bar") + .setFlushMillis(100) + .setFlushCount(4) + .setBatchingStrategy(BatchingStrategy.ONLY_EVENTS) + .setMaxBatchSize(1024 * 1024) + .setBatchQueueSizeLimit(10) + .build(); + final HttpPostEmitter emitter = new HttpPostEmitter(config, httpClient, new ObjectMapper()); + + emitter.start(); + + httpClient.setGoHandler(new GoHandler() { + @Override + protected ListenableFuture go(Request request) throws X + { + return GoHandlers.immediateFuture(EmitterTest.BAD_RESPONSE); + } + }); + + char[] chars = new char[600000]; + Arrays.fill(chars, '*'); + String bigString = new String(chars); + + Event bigEvent = ServiceMetricEvent.builder() + .setFeed("bigEvents") + .setDimension("test", bigString) + .build("metric", 10) + .build("qwerty", "asdfgh"); + + for (int i = 0; i < 1000; i++) { + emitter.emit(bigEvent); + Assert.assertTrue(emitter.getLargeEventsToEmit() <= 11); + } + + emitter.flush(); + } + + @Test + public void testLargeAndSmallEventsQueueLimit() throws InterruptedException, IOException + { + HttpEmitterConfig config = new HttpEmitterConfig.Builder("http://foo.bar") + .setFlushMillis(100) + .setFlushCount(4) + .setBatchingStrategy(BatchingStrategy.ONLY_EVENTS) + .setMaxBatchSize(1024 * 1024) + .setBatchQueueSizeLimit(10) + .build(); + final HttpPostEmitter emitter = new HttpPostEmitter(config, httpClient, new ObjectMapper()); + + emitter.start(); + + httpClient.setGoHandler(new GoHandler() { + @Override + protected ListenableFuture go(Request request) throws X + { + return GoHandlers.immediateFuture(EmitterTest.BAD_RESPONSE); + } + }); + + char[] chars = new char[600000]; + Arrays.fill(chars, '*'); + String bigString = new String(chars); + + Event smallEvent = ServiceMetricEvent.builder() + .setFeed("smallEvents") + .setDimension("test", "hi") + .build("metric", 10) + .build("qwerty", "asdfgh"); + + Event bigEvent = ServiceMetricEvent.builder() + .setFeed("bigEvents") + .setDimension("test", bigString) + .build("metric", 10) + .build("qwerty", "asdfgh"); + + final CountDownLatch threadsCompleted = new CountDownLatch(2); + new Thread() { + @Override + public void run() + { + for (int i = 0; i < 1000; i++) { + + emitter.emit(smallEvent); + + Assert.assertTrue(emitter.getFailedBuffers() <= 10); + Assert.assertTrue(emitter.getBuffersToEmit() <= 12); + } + threadsCompleted.countDown(); + } + }.start(); + new Thread() { + @Override + public void run() + { + for (int i = 0; i < 1000; i++) { + + emitter.emit(bigEvent); + + Assert.assertTrue(emitter.getFailedBuffers() <= 10); + Assert.assertTrue(emitter.getBuffersToEmit() <= 12); + } + threadsCompleted.countDown(); + } + }.start(); + threadsCompleted.await(); + emitter.flush(); + } } diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/ParametrizedUriEmitterConfigTest.java b/java-util/src/test/java/io/druid/java/util/emitter/core/ParametrizedUriEmitterConfigTest.java index 63f7c63a19d0..c860483a1578 100644 --- a/java-util/src/test/java/io/druid/java/util/emitter/core/ParametrizedUriEmitterConfigTest.java +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/ParametrizedUriEmitterConfigTest.java @@ -20,6 +20,7 @@ package io.druid.java.util.emitter.core; import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.java.util.common.Pair; import org.junit.Assert; import org.junit.Test; @@ -41,7 +42,11 @@ public void testDefaults() Assert.assertEquals("http://example.com/topic", config.getRecipientBaseUrl()); Assert.assertEquals(null, config.getBasicAuthentication()); Assert.assertEquals(BatchingStrategy.ARRAY, config.getBatchingStrategy()); - Assert.assertEquals(5 * 1024 * 1024, config.getMaxBatchSize()); + Pair batchConfigPair = BaseHttpEmittingConfig.getDefaultBatchSizeAndLimit( + Runtime.getRuntime().maxMemory() + ); + Assert.assertEquals(batchConfigPair.lhs.intValue(), config.getMaxBatchSize()); + Assert.assertEquals(batchConfigPair.rhs.intValue(), config.getBatchQueueSizeLimit()); Assert.assertEquals(Long.MAX_VALUE, config.getFlushTimeOut()); } From 8f43374f8b3f3d46b7438cd1281659ba15692d9f Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Mon, 9 Apr 2018 18:01:05 -0700 Subject: [PATCH 49/82] Fix round robining in router. (#5500) (#5612) * Fix round robining in router. Say that ten times fast. For query endpoints, AsyncQueryForwardingServlet called hostFinder.getDefaultServer() to set a default server, followed by hostFinder.getServer(inputQuery) to override it with query-specific routing. Since hostFinder is round-robin, this skips a server. When there are only two servers, one server is _always_ skipped and the router sends all queries to the same broker. * Adjust spacing. --- .../query/groupby/GroupByQueryRunnerTest.java | 2 +- .../java/io/druid/segment/TestHelper.java | 2 +- .../server/AsyncQueryForwardingServlet.java | 64 +++++++--- .../druid/server/router/QueryHostFinder.java | 6 +- .../AsyncQueryForwardingServletTest.java | 120 ++++++++++++++++-- 5 files changed, 154 insertions(+), 40 deletions(-) diff --git a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index 906f1095d69e..d02ea057cad7 100644 --- a/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -156,7 +156,7 @@ @RunWith(Parameterized.class) public class GroupByQueryRunnerTest { - public static final ObjectMapper DEFAULT_MAPPER = TestHelper.getSmileMapper(); + public static final ObjectMapper DEFAULT_MAPPER = TestHelper.makeSmileMapper(); public static final DruidProcessingConfig DEFAULT_PROCESSING_CONFIG = new DruidProcessingConfig() { @Override diff --git a/processing/src/test/java/io/druid/segment/TestHelper.java b/processing/src/test/java/io/druid/segment/TestHelper.java index 6d27a4b2d9a5..83eb8f88763e 100644 --- a/processing/src/test/java/io/druid/segment/TestHelper.java +++ b/processing/src/test/java/io/druid/segment/TestHelper.java @@ -83,7 +83,7 @@ public static ObjectMapper makeJsonMapper() return mapper; } - public static ObjectMapper getSmileMapper() + public static ObjectMapper makeSmileMapper() { final ObjectMapper mapper = new DefaultObjectMapper(); mapper.setInjectableValues( diff --git a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java index daa26455a48a..927d9a3a2146 100644 --- a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java +++ b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java @@ -26,14 +26,15 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.google.inject.Provider; -import io.druid.java.util.emitter.EmittingLogger; -import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.selector.Server; import io.druid.guice.annotations.Json; import io.druid.guice.annotations.Smile; import io.druid.guice.http.DruidHttpClientConfig; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.IAE; +import io.druid.java.util.common.jackson.JacksonUtils; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.query.DruidMetrics; import io.druid.query.GenericQueryMetricsFactory; import io.druid.query.Query; @@ -174,31 +175,34 @@ protected void service(HttpServletRequest request, HttpServletResponse response) final ObjectMapper objectMapper = isSmile ? smileMapper : jsonMapper; request.setAttribute(OBJECTMAPPER_ATTRIBUTE, objectMapper); - final Server defaultServer = hostFinder.getDefaultServer(); - request.setAttribute(HOST_ATTRIBUTE, defaultServer.getHost()); - request.setAttribute(SCHEME_ATTRIBUTE, defaultServer.getScheme()); + final String requestURI = request.getRequestURI(); + final String method = request.getMethod(); + final Server targetServer; // The Router does not have the ability to look inside SQL queries and route them intelligently, so just treat // them as a generic request. - final boolean isQueryEndpoint = request.getRequestURI().startsWith("/druid/v2") - && !request.getRequestURI().startsWith("/druid/v2/sql"); + final boolean isQueryEndpoint = requestURI.startsWith("/druid/v2") + && !requestURI.startsWith("/druid/v2/sql"); - final boolean isAvatica = request.getRequestURI().startsWith("/druid/v2/sql/avatica"); + final boolean isAvatica = requestURI.startsWith("/druid/v2/sql/avatica"); if (isAvatica) { - Map requestMap = objectMapper.readValue(request.getInputStream(), Map.class); + Map requestMap = objectMapper.readValue( + request.getInputStream(), + JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT + ); String connectionId = getAvaticaConnectionId(requestMap); - Server targetServer = hostFinder.findServerAvatica(connectionId); + targetServer = hostFinder.findServerAvatica(connectionId); byte[] requestBytes = objectMapper.writeValueAsBytes(requestMap); - request.setAttribute(HOST_ATTRIBUTE, targetServer.getHost()); - request.setAttribute(SCHEME_ATTRIBUTE, targetServer.getScheme()); request.setAttribute(AVATICA_QUERY_ATTRIBUTE, requestBytes); - } else if (isQueryEndpoint && HttpMethod.DELETE.is(request.getMethod())) { + } else if (isQueryEndpoint && HttpMethod.DELETE.is(method)) { // query cancellation request - for (final Server server: hostFinder.getAllServers()) { + targetServer = hostFinder.pickDefaultServer(); + + for (final Server server : hostFinder.getAllServers()) { // send query cancellation to all brokers this query may have gone to - // to keep the code simple, the proxy servlet will also send a request to one of the default brokers - if (!server.getHost().equals(defaultServer.getHost())) { + // to keep the code simple, the proxy servlet will also send a request to the default targetServer. + if (!server.getHost().equals(targetServer.getHost())) { // issue async requests Response.CompleteListener completeListener = result -> { if (result.isFailed()) { @@ -220,17 +224,17 @@ protected void service(HttpServletRequest request, HttpServletResponse response) } interruptedQueryCount.incrementAndGet(); } - } else if (isQueryEndpoint && HttpMethod.POST.is(request.getMethod())) { + } else if (isQueryEndpoint && HttpMethod.POST.is(method)) { // query request try { Query inputQuery = objectMapper.readValue(request.getInputStream(), Query.class); if (inputQuery != null) { - final Server server = hostFinder.getServer(inputQuery); - request.setAttribute(HOST_ATTRIBUTE, server.getHost()); - request.setAttribute(SCHEME_ATTRIBUTE, server.getScheme()); + targetServer = hostFinder.pickServer(inputQuery); if (inputQuery.getId() == null) { inputQuery = inputQuery.withId(UUID.randomUUID().toString()); } + } else { + targetServer = hostFinder.pickDefaultServer(); } request.setAttribute(QUERY_ATTRIBUTE, inputQuery); } @@ -258,8 +262,22 @@ protected void service(HttpServletRequest request, HttpServletResponse response) handleException(response, objectMapper, e); return; } + } else { + targetServer = hostFinder.pickDefaultServer(); } + request.setAttribute(HOST_ATTRIBUTE, targetServer.getHost()); + request.setAttribute(SCHEME_ATTRIBUTE, targetServer.getScheme()); + + doService(request, response); + } + + protected void doService( + HttpServletRequest request, + HttpServletResponse response + ) throws ServletException, IOException + { + // Just call the superclass service method. Overriden in tests. super.service(request, response); } @@ -318,7 +336,11 @@ protected Response.Listener newProxyResponseListener( @Override protected String rewriteTarget(HttpServletRequest request) { - return rewriteURI(request, (String) request.getAttribute(SCHEME_ATTRIBUTE), (String) request.getAttribute(HOST_ATTRIBUTE)).toString(); + return rewriteURI( + request, + (String) request.getAttribute(SCHEME_ATTRIBUTE), + (String) request.getAttribute(HOST_ATTRIBUTE) + ).toString(); } protected URI rewriteURI(HttpServletRequest request, String scheme, String host) diff --git a/server/src/main/java/io/druid/server/router/QueryHostFinder.java b/server/src/main/java/io/druid/server/router/QueryHostFinder.java index 527fd6340d68..fa077ef2d545 100644 --- a/server/src/main/java/io/druid/server/router/QueryHostFinder.java +++ b/server/src/main/java/io/druid/server/router/QueryHostFinder.java @@ -20,10 +20,10 @@ package io.druid.server.router; import com.google.inject.Inject; -import io.druid.java.util.emitter.EmittingLogger; import io.druid.client.selector.Server; import io.druid.java.util.common.ISE; import io.druid.java.util.common.Pair; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.query.Query; import java.util.Collection; @@ -90,7 +90,7 @@ public Server findServerAvatica(String connectionId) return chosenServer; } - public Server getServer(Query query) + public Server pickServer(Query query) { Server server = findServer(query); @@ -107,7 +107,7 @@ public Server getServer(Query query) return server; } - public Server getDefaultServer() + public Server pickDefaultServer() { Server server = findDefaultServer(); diff --git a/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java b/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java index 4118e7ae4f98..c587ad8759ef 100644 --- a/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java +++ b/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java @@ -39,21 +39,24 @@ import io.druid.guice.annotations.Smile; import io.druid.guice.http.DruidHttpClientConfig; import io.druid.initialization.Initialization; +import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.lifecycle.Lifecycle; import io.druid.query.DefaultGenericQueryMetricsFactory; +import io.druid.query.Druids; import io.druid.query.MapQueryToolChestWarehouse; import io.druid.query.Query; -import io.druid.query.QueryToolChest; +import io.druid.query.timeseries.TimeseriesQuery; +import io.druid.segment.TestHelper; import io.druid.server.initialization.BaseJettyTest; import io.druid.server.initialization.jetty.JettyServerInitUtils; import io.druid.server.initialization.jetty.JettyServerInitializer; -import io.druid.server.log.RequestLogger; import io.druid.server.metrics.NoopServiceEmitter; import io.druid.server.router.QueryHostFinder; import io.druid.server.router.RendezvousHashAvaticaConnectionBalancer; import io.druid.server.security.AllowAllAuthorizer; import io.druid.server.security.Authorizer; import io.druid.server.security.AuthorizerMapper; +import org.easymock.EasyMock; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Server; @@ -66,16 +69,20 @@ import org.junit.Before; import org.junit.Test; +import javax.servlet.ReadListener; import javax.servlet.ServletException; +import javax.servlet.ServletInputStream; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URI; import java.net.URL; import java.util.Collection; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicLong; public class AsyncQueryForwardingServletTest extends BaseJettyTest { @@ -116,7 +123,8 @@ public void configure(Binder binder) ); binder.bind(JettyServerInitializer.class).to(ProxyJettyServerInit.class).in(LazySingleton.class); binder.bind(AuthorizerMapper.class).toInstance( - new AuthorizerMapper(null) { + new AuthorizerMapper(null) + { @Override public Authorizer getAuthorizer(String name) @@ -173,6 +181,97 @@ public void testDeleteBroadcast() throws Exception latch.await(); } + @Test + public void testQueryProxy() throws Exception + { + final ObjectMapper jsonMapper = TestHelper.makeJsonMapper(); + final TimeseriesQuery query = Druids.newTimeseriesQueryBuilder() + .dataSource("foo") + .intervals("2000/P1D") + .granularity(Granularities.ALL) + .context(ImmutableMap.of("queryId", "dummy")) + .build(); + + final QueryHostFinder hostFinder = EasyMock.createMock(QueryHostFinder.class); + EasyMock.expect(hostFinder.pickServer(query)).andReturn(new TestServer("http", "1.2.3.4", 9999)).once(); + EasyMock.replay(hostFinder); + + final HttpServletRequest requestMock = EasyMock.createMock(HttpServletRequest.class); + final ByteArrayInputStream inputStream = new ByteArrayInputStream(jsonMapper.writeValueAsBytes(query)); + final ServletInputStream servletInputStream = new ServletInputStream() + { + private boolean finished; + + @Override + public boolean isFinished() + { + return finished; + } + + @Override + public boolean isReady() + { + return true; + } + + @Override + public void setReadListener(final ReadListener readListener) + { + // do nothing + } + + @Override + public int read() + { + final int b = inputStream.read(); + if (b < 0) { + finished = true; + } + return b; + } + }; + EasyMock.expect(requestMock.getContentType()).andReturn("application/json").times(2); + requestMock.setAttribute("io.druid.proxy.objectMapper", jsonMapper); + EasyMock.expectLastCall(); + EasyMock.expect(requestMock.getRequestURI()).andReturn("/druid/v2/"); + EasyMock.expect(requestMock.getMethod()).andReturn("POST"); + EasyMock.expect(requestMock.getInputStream()).andReturn(servletInputStream); + requestMock.setAttribute("io.druid.proxy.query", query); + requestMock.setAttribute("io.druid.proxy.to.host", "1.2.3.4:9999"); + requestMock.setAttribute("io.druid.proxy.to.host.scheme", "http"); + EasyMock.expectLastCall(); + EasyMock.replay(requestMock); + + final AtomicLong didService = new AtomicLong(); + final AsyncQueryForwardingServlet servlet = new AsyncQueryForwardingServlet( + new MapQueryToolChestWarehouse(ImmutableMap.of()), + jsonMapper, + TestHelper.makeSmileMapper(), + hostFinder, + null, + null, + new NoopServiceEmitter(), + requestLogLine -> { /* noop */ }, + new DefaultGenericQueryMetricsFactory(jsonMapper) + ) + { + @Override + protected void doService( + final HttpServletRequest request, + final HttpServletResponse response + ) + { + didService.incrementAndGet(); + } + }; + + servlet.service(requestMock, null); + + // This test is mostly about verifying that the servlet calls the right methods the right number of times. + EasyMock.verify(hostFinder, requestMock); + Assert.assertEquals(1, didService.get()); + } + private static Server makeTestDeleteServer(int port, final CountDownLatch latch) { Server server = new Server(port); @@ -211,13 +310,13 @@ public void initialize(Server server, Injector injector) final QueryHostFinder hostFinder = new QueryHostFinder(null, new RendezvousHashAvaticaConnectionBalancer()) { @Override - public io.druid.client.selector.Server getServer(Query query) + public io.druid.client.selector.Server pickServer(Query query) { return new TestServer("http", "localhost", node.getPlaintextPort()); } @Override - public io.druid.client.selector.Server getDefaultServer() + public io.druid.client.selector.Server pickDefaultServer() { return new TestServer("http", "localhost", node.getPlaintextPort()); } @@ -236,21 +335,14 @@ public Collection getAllServers() ObjectMapper jsonMapper = injector.getInstance(ObjectMapper.class); ServletHolder holder = new ServletHolder( new AsyncQueryForwardingServlet( - new MapQueryToolChestWarehouse(ImmutableMap., QueryToolChest>of()), + new MapQueryToolChestWarehouse(ImmutableMap.of()), jsonMapper, injector.getInstance(Key.get(ObjectMapper.class, Smile.class)), hostFinder, injector.getProvider(HttpClient.class), injector.getInstance(DruidHttpClientConfig.class), new NoopServiceEmitter(), - new RequestLogger() - { - @Override - public void log(RequestLogLine requestLogLine) throws IOException - { - // noop - } - }, + requestLogLine -> { /* noop */ }, new DefaultGenericQueryMetricsFactory(jsonMapper) ) { From a35fe63141313816dadf42ac995f95c63217e2ff Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Mon, 9 Apr 2018 18:10:56 -0700 Subject: [PATCH 50/82] SegmentMetadataQuery: Fix default interval handling. (#5489) (#5603) * SegmentMetadataQuery: Fix default interval handling. PR #4131 introduced a new copy builder for segmentMetadata that did not retain the value of usingDefaultInterval. This led to it being dropped and the default-interval handling not working as expected. Instead of using the default 1 week history when intervals are not provided, the segmentMetadata query would query _all_ segments, incurring an unexpected performance hit. This patch fixes the bug and adds a test for the copy builder. * Intervals --- .../src/main/java/io/druid/query/Druids.java | 11 +++++- ...egmentMetadataQueryQueryToolChestTest.java | 36 +++++++++++++++++++ .../metadata/SegmentMetadataQueryTest.java | 3 ++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/io/druid/query/Druids.java b/processing/src/main/java/io/druid/query/Druids.java index 1576c01a1c79..e235a99d0220 100644 --- a/processing/src/main/java/io/druid/query/Druids.java +++ b/processing/src/main/java/io/druid/query/Druids.java @@ -591,6 +591,7 @@ public static class SegmentMetadataQueryBuilder private EnumSet analysisTypes; private Boolean merge; private Boolean lenientAggregatorMerge; + private Boolean usingDefaultInterval; private Map context; public SegmentMetadataQueryBuilder() @@ -601,6 +602,7 @@ public SegmentMetadataQueryBuilder() analysisTypes = null; merge = null; lenientAggregatorMerge = null; + usingDefaultInterval = null; context = null; } @@ -613,7 +615,7 @@ public SegmentMetadataQuery build() merge, context, analysisTypes, - false, + usingDefaultInterval, lenientAggregatorMerge ); } @@ -627,6 +629,7 @@ public static SegmentMetadataQueryBuilder copy(SegmentMetadataQuery query) .analysisTypes(query.getAnalysisTypes()) .merge(query.isMerge()) .lenientAggregatorMerge(query.isLenientAggregatorMerge()) + .usingDefaultInterval(query.isUsingDefaultInterval()) .context(query.getContext()); } @@ -696,6 +699,12 @@ public SegmentMetadataQueryBuilder lenientAggregatorMerge(boolean lenientAggrega return this; } + public SegmentMetadataQueryBuilder usingDefaultInterval(boolean usingDefaultInterval) + { + this.usingDefaultInterval = usingDefaultInterval; + return this; + } + public SegmentMetadataQueryBuilder context(Map c) { context = c; diff --git a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java index 6a0e3c42a42b..359e39f6af2b 100644 --- a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java +++ b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryQueryToolChestTest.java @@ -27,6 +27,7 @@ import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Intervals; import io.druid.query.CacheStrategy; +import io.druid.query.Druids; import io.druid.query.TableDataSource; import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.DoubleMaxAggregatorFactory; @@ -38,10 +39,14 @@ import io.druid.query.metadata.metadata.SegmentMetadataQuery; import io.druid.query.spec.LegacySegmentSpec; import io.druid.segment.column.ValueType; +import io.druid.timeline.LogicalSegment; +import org.joda.time.Period; import org.junit.Assert; import org.junit.Test; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class SegmentMetadataQueryQueryToolChestTest { @@ -271,6 +276,37 @@ public void testMergeAggregatorsConflict() ); } + @Test + public void testFilterSegments() + { + final SegmentMetadataQueryConfig config = new SegmentMetadataQueryConfig(); + final SegmentMetadataQueryQueryToolChest toolChest = new SegmentMetadataQueryQueryToolChest(config); + + final List filteredSegments = toolChest.filterSegments( + Druids.newSegmentMetadataQueryBuilder().dataSource("foo").merge(true).build(), + ImmutableList + .of( + "2000-01-01/P1D", + "2000-01-04/P1D", + "2000-01-09/P1D", + "2000-01-09/P1D" + ) + .stream() + .map(interval -> (LogicalSegment) () -> Intervals.of(interval)) + .collect(Collectors.toList()) + ); + + Assert.assertEquals(Period.weeks(1), config.getDefaultHistory()); + Assert.assertEquals( + ImmutableList.of( + Intervals.of("2000-01-04/P1D"), + Intervals.of("2000-01-09/P1D"), + Intervals.of("2000-01-09/P1D") + ), + filteredSegments.stream().map(LogicalSegment::getInterval).collect(Collectors.toList()) + ); + } + @SuppressWarnings("ArgumentParameterSwap") @Test public void testMergeRollup() diff --git a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java index deaebcc734ec..fa524f166355 100644 --- a/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java +++ b/processing/src/test/java/io/druid/query/metadata/SegmentMetadataQueryTest.java @@ -899,6 +899,9 @@ public void testSerdeWithDefaultInterval() throws Exception // test serialize and deserialize Assert.assertEquals(query, MAPPER.readValue(MAPPER.writeValueAsString(query), Query.class)); + + // test copy + Assert.assertEquals(query, Druids.SegmentMetadataQueryBuilder.copy((SegmentMetadataQuery) query).build()); } @Test From 1a0c6d541fcb3aff59969f7bbe4e08afb834b26c Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Mon, 9 Apr 2018 18:11:10 -0700 Subject: [PATCH 51/82] Authorize supervisor history instead of current active supervisors for supervisor history API (#5501) (#5604) --- .../supervisor/SupervisorResource.java | 91 +++--- .../supervisor/SupervisorResourceTest.java | 267 ++++++++++++++---- .../supervisor/VersionedSupervisorSpec.java | 27 ++ .../server/security/AuthorizationUtils.java | 60 ++++ 4 files changed, 351 insertions(+), 94 deletions(-) diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorResource.java b/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorResource.java index b7cefa3fcd85..348c40a58ab4 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorResource.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorResource.java @@ -22,10 +22,9 @@ import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.inject.Inject; import com.sun.jersey.spi.container.ResourceFilters; @@ -61,6 +60,20 @@ @Path("/druid/indexer/v1/supervisor") public class SupervisorResource { + private static final Function> SPEC_DATASOURCE_READ_RA_GENERATOR = + supervisorSpec -> { + if (supervisorSpec.getSpec() == null) { + return null; + } + if (supervisorSpec.getSpec().getDataSources() == null) { + return null; + } + return Iterables.transform( + supervisorSpec.getSpec().getDataSources(), + AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR + ); + }; + private final TaskMaster taskMaster; private final AuthConfig authConfig; private final AuthorizerMapper authorizerMapper; @@ -216,26 +229,14 @@ public Response specGetAllHistory(@Context final HttpServletRequest req) @Override public Response apply(final SupervisorManager manager) { - final Map> supervisorHistory = manager.getSupervisorHistory(); - - final Set authorizedSupervisorIds = filterAuthorizedSupervisorIds( - req, - manager, - supervisorHistory.keySet() - ); - - final Map> authorizedSupervisorHistory = Maps.filterKeys( - supervisorHistory, - new Predicate() - { - @Override - public boolean apply(String id) - { - return authorizedSupervisorIds.contains(id); - } - } - ); - return Response.ok(authorizedSupervisorHistory).build(); + return Response.ok( + AuthorizationUtils.filterAuthorizedResources( + req, + manager.getSupervisorHistory(), + SPEC_DATASOURCE_READ_RA_GENERATOR, + authorizerMapper + ) + ).build(); } } ); @@ -244,8 +245,9 @@ public boolean apply(String id) @GET @Path("/{id}/history") @Produces(MediaType.APPLICATION_JSON) - @ResourceFilters(SupervisorResourceFilter.class) - public Response specGetHistory(@PathParam("id") final String id) + public Response specGetHistory( + @Context final HttpServletRequest req, + @PathParam("id") final String id) { return asLeaderWithSupervisorManager( new Function() @@ -253,23 +255,32 @@ public Response specGetHistory(@PathParam("id") final String id) @Override public Response apply(SupervisorManager manager) { - Map> history = manager.getSupervisorHistory(); - if (history.containsKey(id)) { - return Response.ok(history.get(id)).build(); - } else { - return Response.status(Response.Status.NOT_FOUND) - .entity( - ImmutableMap.of( - "error", - StringUtils.format( - "No history for [%s] (history available for %s)", - id, - history.keySet() - ) - ) - ) - .build(); + Map> supervisorHistory = manager.getSupervisorHistory(); + Iterable historyForId = supervisorHistory.get(id); + if (historyForId != null) { + final List authorizedHistoryForId = + Lists.newArrayList( + AuthorizationUtils.filterAuthorizedResources( + req, + historyForId, + SPEC_DATASOURCE_READ_RA_GENERATOR, + authorizerMapper + ) + ); + if (authorizedHistoryForId.size() > 0) { + return Response.ok(authorizedHistoryForId).build(); + } } + + return Response.status(Response.Status.NOT_FOUND) + .entity( + ImmutableMap.of( + "error", + StringUtils.format("No history for [%s].", id) + ) + ) + .build(); + } } ); diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorResourceTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorResourceTest.java index 961f1e91069e..43283997031d 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorResourceTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorResourceTest.java @@ -47,6 +47,7 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.Response; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Set; @@ -101,7 +102,7 @@ public Access authorize( @Test public void testSpecPost() throws Exception { - SupervisorSpec spec = new TestSupervisorSpec("my-id", null) { + SupervisorSpec spec = new TestSupervisorSpec("my-id", null, null) { @Override public List getDataSources() @@ -140,7 +141,7 @@ public List getDataSources() public void testSpecGetAll() throws Exception { Set supervisorIds = ImmutableSet.of("id1", "id2"); - SupervisorSpec spec1 = new TestSupervisorSpec("id1", null) { + SupervisorSpec spec1 = new TestSupervisorSpec("id1", null, null) { @Override public List getDataSources() @@ -148,7 +149,7 @@ public List getDataSources() return Lists.newArrayList("datasource1"); } }; - SupervisorSpec spec2 = new TestSupervisorSpec("id2", null) { + SupervisorSpec spec2 = new TestSupervisorSpec("id2", null, null) { @Override public List getDataSources() @@ -188,7 +189,7 @@ public List getDataSources() @Test public void testSpecGet() throws Exception { - SupervisorSpec spec = new TestSupervisorSpec("my-id", null); + SupervisorSpec spec = new TestSupervisorSpec("my-id", null, null); EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.of(supervisorManager)).times(2); EasyMock.expect(supervisorManager.getSupervisorSpec("my-id")).andReturn(Optional.of(spec)); @@ -286,28 +287,34 @@ public void testShutdown() throws Exception @Test public void testSpecGetAllHistory() throws Exception { + List versions1 = ImmutableList.of( + new VersionedSupervisorSpec( + new TestSupervisorSpec("id1", null, Arrays.asList("datasource1")), + "v1" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id1", null, Arrays.asList("datasource1")), + "v2" + ) + ); + List versions2 = ImmutableList.of( + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), + "v1" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), + "v2" + ) + ); Map> history = Maps.newHashMap(); - history.put("id1", null); - history.put("id2", null); + history.put("id1", versions1); + history.put("id2", versions2); EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.of(supervisorManager)).times(2); EasyMock.expect(supervisorManager.getSupervisorHistory()).andReturn(history); - SupervisorSpec spec1 = new TestSupervisorSpec("id1", null) { - - @Override - public List getDataSources() - { - return Lists.newArrayList("datasource1"); - } - }; - SupervisorSpec spec2 = new TestSupervisorSpec("id2", null) { - - @Override - public List getDataSources() - { - return Lists.newArrayList("datasource2"); - } - }; + SupervisorSpec spec1 = new TestSupervisorSpec("id1", null, Arrays.asList("datasource1")); + SupervisorSpec spec2 = new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")); EasyMock.expect(supervisorManager.getSupervisorSpec("id1")).andReturn(Optional.of(spec1)).atLeastOnce(); EasyMock.expect(supervisorManager.getSupervisorSpec("id2")).andReturn(Optional.of(spec2)).atLeastOnce(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).atLeastOnce(); @@ -337,28 +344,35 @@ public List getDataSources() @Test public void testSpecGetAllHistoryWithAuthFailureFiltering() throws Exception { + List versions1 = ImmutableList.of( + new VersionedSupervisorSpec( + new TestSupervisorSpec("id1", null, Arrays.asList("datasource1")), + "v1" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id1", null, Arrays.asList("datasource1")), + "v2" + ) + ); + List versions2 = ImmutableList.of( + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), + "v1" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), + "v2" + ) + ); + Map> history = Maps.newHashMap(); - history.put("id1", null); - history.put("id2", null); + history.put("id1", versions1); + history.put("id2", versions2); EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.of(supervisorManager)).times(2); EasyMock.expect(supervisorManager.getSupervisorHistory()).andReturn(history); - SupervisorSpec spec1 = new TestSupervisorSpec("id1", null) { - - @Override - public List getDataSources() - { - return Lists.newArrayList("datasource1"); - } - }; - SupervisorSpec spec2 = new TestSupervisorSpec("id2", null) { - - @Override - public List getDataSources() - { - return Lists.newArrayList("datasource2"); - } - }; + SupervisorSpec spec1 = new TestSupervisorSpec("id1", null, Arrays.asList("datasource1")); + SupervisorSpec spec2 = new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")); EasyMock.expect(supervisorManager.getSupervisorSpec("id1")).andReturn(Optional.of(spec1)).atLeastOnce(); EasyMock.expect(supervisorManager.getSupervisorSpec("id2")).andReturn(Optional.of(spec2)).atLeastOnce(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).atLeastOnce(); @@ -372,7 +386,7 @@ public List getDataSources() Response response = supervisorResource.specGetAllHistory(request); Map> filteredHistory = Maps.newHashMap(); - filteredHistory.put("id1", null); + filteredHistory.put("id1", versions1); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(filteredHistory, response.getEntity()); @@ -391,24 +405,51 @@ public List getDataSources() @Test public void testSpecGetHistory() throws Exception { - List versions = ImmutableList.of( - new VersionedSupervisorSpec(null, "v1"), - new VersionedSupervisorSpec(null, "v2") + List versions1 = ImmutableList.of( + new VersionedSupervisorSpec( + new TestSupervisorSpec("id1", null, Arrays.asList("datasource1")), + "v1" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id1", null, Arrays.asList("datasource1")), + "v2" + ) + ); + List versions2 = ImmutableList.of( + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), + "v1" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), + "v2" + ) ); Map> history = Maps.newHashMap(); - history.put("id1", versions); - history.put("id2", null); + history.put("id1", versions1); + history.put("id2", versions2); - EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.of(supervisorManager)).times(2); - EasyMock.expect(supervisorManager.getSupervisorHistory()).andReturn(history).times(2); + EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.of(supervisorManager)).times(3); + EasyMock.expect(supervisorManager.getSupervisorHistory()).andReturn(history).times(3); + EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).atLeastOnce(); + EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( + new AuthenticationResult("druid", "druid", null) + ).atLeastOnce(); + request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); + EasyMock.expectLastCall().anyTimes(); replayAll(); - Response response = supervisorResource.specGetHistory("id1"); + Response response = supervisorResource.specGetHistory(request, "id1"); + + Assert.assertEquals(200, response.getStatus()); + Assert.assertEquals(versions1, response.getEntity()); + + response = supervisorResource.specGetHistory(request, "id2"); Assert.assertEquals(200, response.getStatus()); - Assert.assertEquals(versions, response.getEntity()); + Assert.assertEquals(versions2, response.getEntity()); - response = supervisorResource.specGetHistory("id3"); + response = supervisorResource.specGetHistory(request, "id3"); Assert.assertEquals(404, response.getStatus()); @@ -417,7 +458,92 @@ public void testSpecGetHistory() throws Exception EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.absent()); replayAll(); - response = supervisorResource.specGetHistory("id1"); + response = supervisorResource.specGetHistory(request, "id1"); + verifyAll(); + + Assert.assertEquals(503, response.getStatus()); + } + + @Test + public void testSpecGetHistoryWithAuthFailure() throws Exception + { + List versions1 = ImmutableList.of( + new VersionedSupervisorSpec( + new TestSupervisorSpec("id1", null, Arrays.asList("datasource1")), + "v1" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id1", null, Arrays.asList("datasource1")), + "v2" + ) + ); + List versions2 = ImmutableList.of( + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), + "v1" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), + "v2" + ) + ); + List versions3 = ImmutableList.of( + new VersionedSupervisorSpec( + new TestSupervisorSpec("id3", null, Arrays.asList("datasource3")), + "v1" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id3", null, Arrays.asList("datasource2")), + "v2" + ) + ); + Map> history = Maps.newHashMap(); + history.put("id1", versions1); + history.put("id2", versions2); + history.put("id3", versions3); + + EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.of(supervisorManager)).times(4); + EasyMock.expect(supervisorManager.getSupervisorHistory()).andReturn(history).times(4); + EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).atLeastOnce(); + EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( + new AuthenticationResult("notdruid", "druid", null) + ).atLeastOnce(); + request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); + EasyMock.expectLastCall().anyTimes(); + replayAll(); + + Response response = supervisorResource.specGetHistory(request, "id1"); + + Assert.assertEquals(200, response.getStatus()); + Assert.assertEquals(versions1, response.getEntity()); + + response = supervisorResource.specGetHistory(request, "id2"); + + // user is not authorized to access datasource2 + Assert.assertEquals(404, response.getStatus()); + + response = supervisorResource.specGetHistory(request, "id3"); + Assert.assertEquals(200, response.getStatus()); + Assert.assertEquals( + ImmutableList.of( + new VersionedSupervisorSpec( + new TestSupervisorSpec("id3", null, Arrays.asList("datasource3")), + "v1" + ) + ), + response.getEntity() + ); + + response = supervisorResource.specGetHistory(request, "id4"); + Assert.assertEquals(404, response.getStatus()); + + + resetAll(); + + EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.absent()); + replayAll(); + + response = supervisorResource.specGetHistory(request, "id1"); verifyAll(); Assert.assertEquals(503, response.getStatus()); @@ -460,11 +586,13 @@ private static class TestSupervisorSpec implements SupervisorSpec { private final String id; private final Supervisor supervisor; + private final List datasources; - public TestSupervisorSpec(String id, Supervisor supervisor) + public TestSupervisorSpec(String id, Supervisor supervisor, List datasources) { this.id = id; this.supervisor = supervisor; + this.datasources = datasources; } @Override @@ -482,7 +610,38 @@ public Supervisor createSupervisor() @Override public List getDataSources() { - return null; + return datasources; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + TestSupervisorSpec that = (TestSupervisorSpec) o; + + if (getId() != null ? !getId().equals(that.getId()) : that.getId() != null) { + return false; + } + if (supervisor != null ? !supervisor.equals(that.supervisor) : that.supervisor != null) { + return false; + } + return datasources != null ? datasources.equals(that.datasources) : that.datasources == null; + + } + + @Override + public int hashCode() + { + int result = getId() != null ? getId().hashCode() : 0; + result = 31 * result + (supervisor != null ? supervisor.hashCode() : 0); + result = 31 * result + (datasources != null ? datasources.hashCode() : 0); + return result; } } } diff --git a/server/src/main/java/io/druid/indexing/overlord/supervisor/VersionedSupervisorSpec.java b/server/src/main/java/io/druid/indexing/overlord/supervisor/VersionedSupervisorSpec.java index b96edf37406d..93772486c3a6 100644 --- a/server/src/main/java/io/druid/indexing/overlord/supervisor/VersionedSupervisorSpec.java +++ b/server/src/main/java/io/druid/indexing/overlord/supervisor/VersionedSupervisorSpec.java @@ -45,4 +45,31 @@ public String getVersion() { return version; } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + VersionedSupervisorSpec that = (VersionedSupervisorSpec) o; + + if (getSpec() != null ? !getSpec().equals(that.getSpec()) : that.getSpec() != null) { + return false; + } + return getVersion() != null ? getVersion().equals(that.getVersion()) : that.getVersion() == null; + + } + + @Override + public int hashCode() + { + int result = getSpec() != null ? getSpec().hashCode() : 0; + result = 31 * result + (getVersion() != null ? getVersion().hashCode() : 0); + return result; + } } diff --git a/server/src/main/java/io/druid/server/security/AuthorizationUtils.java b/server/src/main/java/io/druid/server/security/AuthorizationUtils.java index 68e35b5484c2..b14d6a788511 100644 --- a/server/src/main/java/io/druid/server/security/AuthorizationUtils.java +++ b/server/src/main/java/io/druid/server/security/AuthorizationUtils.java @@ -27,6 +27,7 @@ import io.druid.java.util.common.ISE; import javax.servlet.http.HttpServletRequest; +import java.util.List; import java.util.Map; import java.util.Set; @@ -281,6 +282,65 @@ public static Iterable filterAuthorizedResources( return filteredResources; } + /** + * Given a map of resource lists, filter each resources list by applying the resource action generator to each + * item in each resource list. + * + * The resourceActionGenerator returns an Iterable for each resource. + * + * If a resource list is null or has no authorized items after filtering, it will not be included in the returned + * map. + * + * This function will set the DRUID_AUTHORIZATION_CHECKED attribute in the request. + * + * If this attribute is already set when this function is called, an exception is thrown. + * + * @param request HTTP request to be authorized + * @param unfilteredResources Map of resource lists to be filtered + * @param resourceActionGenerator Function that creates an iterable of resource-actions from a resource + * @param authorizerMapper authorizer mapper + * + * @return Map containing lists of resources that were authorized + */ + public static Map> filterAuthorizedResources( + final HttpServletRequest request, + final Map> unfilteredResources, + final Function> resourceActionGenerator, + final AuthorizerMapper authorizerMapper + ) + { + if (request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED) != null) { + throw new ISE("Request already had authorization check."); + } + + final AuthenticationResult authenticationResult = AuthorizationUtils.authenticationResultFromRequest(request); + + Map> filteredResources = Maps.newHashMap(); + for (Map.Entry> entry : unfilteredResources.entrySet()) { + if (entry.getValue() == null) { + continue; + } + + final List filteredList = Lists.newArrayList( + AuthorizationUtils.filterAuthorizedResources( + authenticationResult, + entry.getValue(), + resourceActionGenerator, + authorizerMapper + ) + ); + + if (filteredList.size() > 0) { + filteredResources.put( + entry.getKey(), + filteredList + ); + } + } + + request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); + return filteredResources; + } /** * Function for the common pattern of generating a resource-action for reading from a datasource, using the From 62042c85e5d5a5710f329b7924c8c238ef889950 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Mon, 9 Apr 2018 18:11:20 -0700 Subject: [PATCH 52/82] Fix SQLMetadataSegmentManager to allow succesive start and stop (#5554) (#5605) * Fix SQLMetadataSegmentManager to allow succesive start and stop * address comment * add synchronization --- .../io/druid/concurrent/LifecycleLock.java | 20 +++++----- .../druid/concurrent/LifecycleLockTest.java | 3 +- .../metadata/SQLMetadataSegmentManager.java | 37 +++++++++---------- .../cache/LookupCoordinatorManager.java | 31 ++++++++++------ .../metadata/SQLMetadataRuleManagerTest.java | 10 +++++ ...ava => SQLMetadataSegmentManagerTest.java} | 12 +++++- .../cache/LookupCoordinatorManagerTest.java | 21 +++++------ 7 files changed, 81 insertions(+), 53 deletions(-) rename server/src/test/java/io/druid/metadata/{MetadataSegmentManagerTest.java => SQLMetadataSegmentManagerTest.java} (96%) diff --git a/common/src/main/java/io/druid/concurrent/LifecycleLock.java b/common/src/main/java/io/druid/concurrent/LifecycleLock.java index a94e3cd31a3e..7cc8e64a113f 100644 --- a/common/src/main/java/io/druid/concurrent/LifecycleLock.java +++ b/common/src/main/java/io/druid/concurrent/LifecycleLock.java @@ -175,9 +175,9 @@ void exitStop() } } - void reset() + void exitStopAndReset() { - if (!compareAndSetState(STOPPED, NOT_STARTED)) { + if (!compareAndSetState(STOPPING, NOT_STARTED)) { throw new IllegalMonitorStateException("Not called exitStop() before reset()"); } } @@ -187,7 +187,7 @@ void reset() /** * Start latch, only one canStart() call in any thread on this LifecycleLock object could return true, if {@link - * #reset()} is not called in between. + * #exitStopAndReset()} is not called in between. */ public boolean canStart() { @@ -257,8 +257,8 @@ public boolean canStop() } /** - * If this LifecycleLock is used in a restartable object, which uses {@link #reset()}, exitStop() must be called - * before exit from stop() on this object, usually in a finally block. + * Finalizes stopping the the LifecycleLock. This method must be called before exit from stop() on this object, + * usually in a finally block. If you're using a restartable object, use {@link #exitStopAndReset()} instead. * * @throws IllegalMonitorStateException if {@link #canStop()} is not yet called on this LifecycleLock */ @@ -268,12 +268,14 @@ public void exitStop() } /** - * Resets the LifecycleLock after {@link #exitStop()}, so that {@link #canStart()} could be called again. + * Finalizes stopping the LifecycleLock and resets it, so that {@link #canStart()} could be called again. If this + * LifecycleLock is used in a restartable object, this method must be called before exit from stop() on this object, + * usually in a finally block. * - * @throws IllegalMonitorStateException if {@link #exitStop()} is not yet called on this LifecycleLock + * @throws IllegalMonitorStateException if {@link #canStop()} is not yet called on this LifecycleLock */ - public void reset() + public void exitStopAndReset() { - sync.reset(); + sync.exitStopAndReset(); } } diff --git a/common/src/test/java/io/druid/concurrent/LifecycleLockTest.java b/common/src/test/java/io/druid/concurrent/LifecycleLockTest.java index e1d9f8adbbf7..afb55bf2c2c2 100644 --- a/common/src/test/java/io/druid/concurrent/LifecycleLockTest.java +++ b/common/src/test/java/io/druid/concurrent/LifecycleLockTest.java @@ -138,8 +138,7 @@ public void testRestart() lifecycleLock.started(); lifecycleLock.exitStart(); Assert.assertTrue(lifecycleLock.canStop()); - lifecycleLock.exitStop(); - lifecycleLock.reset(); + lifecycleLock.exitStopAndReset(); Assert.assertTrue(lifecycleLock.canStart()); } diff --git a/server/src/main/java/io/druid/metadata/SQLMetadataSegmentManager.java b/server/src/main/java/io/druid/metadata/SQLMetadataSegmentManager.java index 59e923855be3..6f3f4581991d 100644 --- a/server/src/main/java/io/druid/metadata/SQLMetadataSegmentManager.java +++ b/server/src/main/java/io/druid/metadata/SQLMetadataSegmentManager.java @@ -36,7 +36,6 @@ import io.druid.java.util.emitter.EmittingLogger; import io.druid.client.DruidDataSource; import io.druid.client.ImmutableDruidDataSource; -import io.druid.concurrent.LifecycleLock; import io.druid.guice.ManageLifecycle; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; @@ -86,7 +85,10 @@ public class SQLMetadataSegmentManager implements MetadataSegmentManager private static final Interner DATA_SEGMENT_INTERNER = Interners.newWeakInterner(); private static final EmittingLogger log = new EmittingLogger(SQLMetadataSegmentManager.class); - private final LifecycleLock lifecycleLock = new LifecycleLock(); + // Use to synchronize start() and stop(). These methods should be synchronized to prevent from being called at the + // same time if two different threads are calling them. This might be possible if a druid coordinator gets and drops + // leadership repeatedly in quick succession. + private final Object lock = new Object(); private final ObjectMapper jsonMapper; private final Supplier config; @@ -96,6 +98,7 @@ public class SQLMetadataSegmentManager implements MetadataSegmentManager private volatile ListeningScheduledExecutorService exec = null; private volatile ListenableFuture future = null; + private volatile boolean started; @Inject public SQLMetadataSegmentManager( @@ -118,11 +121,11 @@ public SQLMetadataSegmentManager( @LifecycleStart public void start() { - if (!lifecycleLock.canStart()) { - return; - } + synchronized (lock) { + if (started) { + return; + } - try { exec = MoreExecutors.listeningDecorator(Execs.scheduledSingleThreaded("DatabaseSegmentManager-Exec--%d")); final Duration delay = config.get().getPollDuration().toStandardDuration(); @@ -145,10 +148,7 @@ public void run() delay.getMillis(), TimeUnit.MILLISECONDS ); - lifecycleLock.started(); - } - finally { - lifecycleLock.exitStart(); + started = true; } } @@ -156,10 +156,11 @@ public void run() @LifecycleStop public void stop() { - if (!lifecycleLock.canStop()) { - return; - } - try { + synchronized (lock) { + if (!started) { + return; + } + final ConcurrentHashMap emptyMap = new ConcurrentHashMap<>(); ConcurrentHashMap current; do { @@ -170,9 +171,7 @@ public void stop() future = null; exec.shutdownNow(); exec = null; - } - finally { - lifecycleLock.exitStop(); + started = false; } } @@ -368,7 +367,7 @@ public boolean removeSegment(String ds, final String segmentID) @Override public boolean isStarted() { - return lifecycleLock.isStarted(); + return started; } @Override @@ -422,7 +421,7 @@ public List fold( public void poll() { try { - if (!lifecycleLock.isStarted()) { + if (!started) { return; } diff --git a/server/src/main/java/io/druid/server/lookup/cache/LookupCoordinatorManager.java b/server/src/main/java/io/druid/server/lookup/cache/LookupCoordinatorManager.java index eb8995b840ed..8e7424c8f252 100644 --- a/server/src/main/java/io/druid/server/lookup/cache/LookupCoordinatorManager.java +++ b/server/src/main/java/io/druid/server/lookup/cache/LookupCoordinatorManager.java @@ -36,15 +36,8 @@ import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.inject.Inject; -import io.druid.java.util.emitter.EmittingLogger; -import io.druid.java.util.http.client.HttpClient; -import io.druid.java.util.http.client.Request; -import io.druid.java.util.http.client.response.ClientResponse; -import io.druid.java.util.http.client.response.HttpResponseHandler; -import io.druid.java.util.http.client.response.SequenceInputStreamResponseHandler; import io.druid.audit.AuditInfo; import io.druid.common.config.JacksonConfigManager; -import io.druid.java.util.common.concurrent.Execs; import io.druid.concurrent.LifecycleLock; import io.druid.discovery.DruidNodeDiscoveryProvider; import io.druid.guice.annotations.EscalatedGlobal; @@ -54,6 +47,13 @@ import io.druid.java.util.common.ISE; import io.druid.java.util.common.StreamUtils; import io.druid.java.util.common.StringUtils; +import io.druid.java.util.common.concurrent.Execs; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.ClientResponse; +import io.druid.java.util.http.client.response.HttpResponseHandler; +import io.druid.java.util.http.client.response.SequenceInputStreamResponseHandler; import io.druid.query.lookup.LookupsState; import io.druid.server.http.HostAndPortWithScheme; import io.druid.server.listener.resource.ListenerResource; @@ -124,8 +124,7 @@ public class LookupCoordinatorManager // Updated by config watching service private AtomicReference>> lookupMapConfigRef; - @VisibleForTesting - final LifecycleLock lifecycleLock = new LifecycleLock(); + private final LifecycleLock lifecycleLock = new LifecycleLock(); private ListeningScheduledExecutorService executorService; private ListenableScheduledFuture backgroundManagerFuture; @@ -333,6 +332,17 @@ public LookupExtractorFactoryMapContainer getLookup(final String tier, final Str return tierLookups.get(lookupName); } + public boolean isStarted() + { + return lifecycleLock.isStarted(); + } + + @VisibleForTesting + boolean awaitStarted(long waitTimeMs) + { + return lifecycleLock.awaitStarted(waitTimeMs, TimeUnit.MILLISECONDS); + } + // start() and stop() are synchronized so that they never run in parallel in case of ZK acting funny or druid bug and // coordinator becomes leader and drops leadership in quick succession. public void start() @@ -439,8 +449,7 @@ public void stop() } finally { //so that subsequent start() would happen, even if stop() failed with exception - lifecycleLock.exitStop(); - lifecycleLock.reset(); + lifecycleLock.exitStopAndReset(); } } } diff --git a/server/src/test/java/io/druid/metadata/SQLMetadataRuleManagerTest.java b/server/src/test/java/io/druid/metadata/SQLMetadataRuleManagerTest.java index 5df9c6b535ec..36cf995df3a0 100644 --- a/server/src/test/java/io/druid/metadata/SQLMetadataRuleManagerTest.java +++ b/server/src/test/java/io/druid/metadata/SQLMetadataRuleManagerTest.java @@ -82,6 +82,16 @@ public void setUp() ); } + @Test + public void testMultipleStopAndStart() + { + // Simulate successive losing and getting the coordinator leadership + ruleManager.start(); + ruleManager.stop(); + ruleManager.start(); + ruleManager.stop(); + } + @Test public void testRuleInsert() { diff --git a/server/src/test/java/io/druid/metadata/MetadataSegmentManagerTest.java b/server/src/test/java/io/druid/metadata/SQLMetadataSegmentManagerTest.java similarity index 96% rename from server/src/test/java/io/druid/metadata/MetadataSegmentManagerTest.java rename to server/src/test/java/io/druid/metadata/SQLMetadataSegmentManagerTest.java index 648033ee1c03..927894551654 100644 --- a/server/src/test/java/io/druid/metadata/MetadataSegmentManagerTest.java +++ b/server/src/test/java/io/druid/metadata/SQLMetadataSegmentManagerTest.java @@ -41,7 +41,7 @@ import java.io.IOException; -public class MetadataSegmentManagerTest +public class SQLMetadataSegmentManagerTest { @Rule public final TestDerbyConnector.DerbyConnectorRule derbyConnectorRule = new TestDerbyConnector.DerbyConnectorRule(); @@ -230,4 +230,14 @@ public void testRemoveDataSegment() throws IOException Assert.assertNull(manager.getInventoryValue(newDataSource)); Assert.assertTrue(manager.removeSegment(newDataSource, newSegment.getIdentifier())); } + + @Test + public void testStopAndStart() + { + // Simulate successive losing and getting the coordinator leadership + manager.start(); + manager.stop(); + manager.start(); + manager.stop(); + } } diff --git a/server/src/test/java/io/druid/server/lookup/cache/LookupCoordinatorManagerTest.java b/server/src/test/java/io/druid/server/lookup/cache/LookupCoordinatorManagerTest.java index b0b9aaf71efd..3e39801352e7 100644 --- a/server/src/test/java/io/druid/server/lookup/cache/LookupCoordinatorManagerTest.java +++ b/server/src/test/java/io/druid/server/lookup/cache/LookupCoordinatorManagerTest.java @@ -61,7 +61,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -1255,15 +1254,15 @@ public void testStartStop() throws Exception lookupCoordinatorManagerConfig ); - Assert.assertFalse(manager.lifecycleLock.awaitStarted(1, TimeUnit.MILLISECONDS)); + Assert.assertFalse(manager.isStarted()); manager.start(); - Assert.assertTrue(manager.lifecycleLock.awaitStarted(1, TimeUnit.MILLISECONDS)); + Assert.assertTrue(manager.awaitStarted(1)); Assert.assertTrue(manager.backgroundManagerIsRunning()); Assert.assertFalse(manager.waitForBackgroundTermination(10)); manager.stop(); - Assert.assertFalse(manager.lifecycleLock.awaitStarted(1, TimeUnit.MILLISECONDS)); + Assert.assertFalse(manager.awaitStarted(1)); Assert.assertTrue(manager.waitForBackgroundTermination(10)); Assert.assertFalse(manager.backgroundManagerIsRunning()); @@ -1292,35 +1291,35 @@ public void testMultipleStartStop() throws Exception lookupCoordinatorManagerConfig ); - Assert.assertFalse(manager.lifecycleLock.awaitStarted(1, TimeUnit.MILLISECONDS)); + Assert.assertFalse(manager.awaitStarted(1)); manager.start(); - Assert.assertTrue(manager.lifecycleLock.awaitStarted(1, TimeUnit.MILLISECONDS)); + Assert.assertTrue(manager.awaitStarted(1)); Assert.assertTrue(manager.backgroundManagerIsRunning()); Assert.assertFalse(manager.waitForBackgroundTermination(10)); manager.stop(); - Assert.assertFalse(manager.lifecycleLock.awaitStarted(1, TimeUnit.MILLISECONDS)); + Assert.assertFalse(manager.awaitStarted(1)); Assert.assertTrue(manager.waitForBackgroundTermination(10)); Assert.assertFalse(manager.backgroundManagerIsRunning()); manager.start(); - Assert.assertTrue(manager.lifecycleLock.awaitStarted(1, TimeUnit.MILLISECONDS)); + Assert.assertTrue(manager.awaitStarted(1)); Assert.assertTrue(manager.backgroundManagerIsRunning()); Assert.assertFalse(manager.waitForBackgroundTermination(10)); manager.stop(); - Assert.assertFalse(manager.lifecycleLock.awaitStarted(1, TimeUnit.MILLISECONDS)); + Assert.assertFalse(manager.awaitStarted(1)); Assert.assertTrue(manager.waitForBackgroundTermination(10)); Assert.assertFalse(manager.backgroundManagerIsRunning()); manager.start(); - Assert.assertTrue(manager.lifecycleLock.awaitStarted(1, TimeUnit.MILLISECONDS)); + Assert.assertTrue(manager.awaitStarted(1)); Assert.assertTrue(manager.backgroundManagerIsRunning()); Assert.assertFalse(manager.waitForBackgroundTermination(10)); manager.stop(); - Assert.assertFalse(manager.lifecycleLock.awaitStarted(1, TimeUnit.MILLISECONDS)); + Assert.assertFalse(manager.awaitStarted(1)); Assert.assertTrue(manager.waitForBackgroundTermination(10)); Assert.assertFalse(manager.backgroundManagerIsRunning()); From ae298956b60ca240a8c79db5cea83d5193c9c401 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Mon, 9 Apr 2018 21:06:17 -0700 Subject: [PATCH 53/82] Fix indexTask to respect forceExtendableShardSpecs (#5509) (#5607) * Fix indexTask to respect forceExtendableShardSpecs * add comments --- .../druid/indexing/common/task/IndexTask.java | 24 ++++++++++++------- .../indexing/common/task/IndexTaskTest.java | 2 +- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java index 55296f05a8de..f0987b21ea8c 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java @@ -280,16 +280,15 @@ private static String findVersion(Map versions, Interval inter private static boolean isGuaranteedRollup(IndexIOConfig ioConfig, IndexTuningConfig tuningConfig) { Preconditions.checkState( - !(tuningConfig.isForceGuaranteedRollup() && - (tuningConfig.isForceExtendableShardSpecs() || ioConfig.isAppendToExisting())), - "Perfect rollup cannot be guaranteed with extendable shardSpecs" + !tuningConfig.isForceGuaranteedRollup() || !ioConfig.isAppendToExisting(), + "Perfect rollup cannot be guaranteed when appending to existing dataSources" ); return tuningConfig.isForceGuaranteedRollup(); } private static boolean isExtendableShardSpecs(IndexIOConfig ioConfig, IndexTuningConfig tuningConfig) { - return !isGuaranteedRollup(ioConfig, tuningConfig); + return tuningConfig.isForceExtendableShardSpecs() || ioConfig.isAppendToExisting(); } /** @@ -936,7 +935,19 @@ public static class IndexTuningConfig implements TuningConfig, AppenderatorConfi private final IndexSpec indexSpec; private final File basePersistDirectory; private final int maxPendingPersists; + + /** + * This flag is to force to always use an extendableShardSpec (like {@link NumberedShardSpec} even if + * {@link #forceGuaranteedRollup} is set. + */ private final boolean forceExtendableShardSpecs; + + /** + * This flag is to force _perfect rollup mode_. {@link IndexTask} will scan the whole input data twice to 1) figure + * out proper shard specs for each segment and 2) generate segments. Note that perfect rollup mode basically assumes + * that no more data will be appended in the future. As a result, in perfect rollup mode, {@link NoneShardSpec} and + * {@link HashBasedNumberedShardSpec} are used for a single shard and two or shards, respectively. + */ private final boolean forceGuaranteedRollup; private final boolean reportParseExceptions; private final long pushTimeout; @@ -1019,11 +1030,6 @@ private IndexTuningConfig( this.pushTimeout = pushTimeout == null ? DEFAULT_PUSH_TIMEOUT : pushTimeout; this.basePersistDirectory = basePersistDirectory; - Preconditions.checkArgument( - !(this.forceExtendableShardSpecs && this.forceGuaranteedRollup), - "Perfect rollup cannot be guaranteed with extendable shardSpecs" - ); - this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory; } diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java index d0cf4e39d2cf..f813220e25d2 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java @@ -191,7 +191,7 @@ public void testForceExtendableShardSpecs() throws Exception tmpDir, null, null, - createTuningConfig(2, null, true, false), + createTuningConfig(2, null, true, true), false ), null From f4e9e6b64ffb31243d70e8336e794c61ac91a6f9 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Mon, 9 Apr 2018 21:34:55 -0700 Subject: [PATCH 54/82] ArrayAggregation: Use long to avoid overflow (#5544) (#5610) * ArrayAggregation: Use long to avoid overflow * Add Tests --- .../epinephelinae/BufferArrayGrouper.java | 4 ++-- .../epinephelinae/GroupByQueryEngineV2.java | 2 +- .../epinephelinae/BufferArrayGrouperTest.java | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/BufferArrayGrouper.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/BufferArrayGrouper.java index 7a39b5e34cf1..e816fbd3d9c0 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/BufferArrayGrouper.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/BufferArrayGrouper.java @@ -62,7 +62,7 @@ public class BufferArrayGrouper implements IntGrouper private ByteBuffer usedFlagBuffer; private ByteBuffer valBuffer; - static int requiredBufferCapacity( + static long requiredBufferCapacity( int cardinality, AggregatorFactory[] aggregatorFactories ) @@ -73,7 +73,7 @@ static int requiredBufferCapacity( .sum(); return getUsedFlagBufferCapacity(cardinalityWithMissingValue) + // total used flags size - cardinalityWithMissingValue * recordSize; // total values size + (long) cardinalityWithMissingValue * recordSize; // total values size } /** diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java index 3e3330cfefdb..dffc4f626038 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/GroupByQueryEngineV2.java @@ -219,7 +219,7 @@ private static boolean isArrayAggregateApplicable( final AggregatorFactory[] aggregatorFactories = query .getAggregatorSpecs() .toArray(new AggregatorFactory[query.getAggregatorSpecs().size()]); - final int requiredBufferCapacity = BufferArrayGrouper.requiredBufferCapacity( + final long requiredBufferCapacity = BufferArrayGrouper.requiredBufferCapacity( cardinality, aggregatorFactories ); diff --git a/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferArrayGrouperTest.java b/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferArrayGrouperTest.java index 6ca584f2b8d3..acdc2c406047 100644 --- a/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferArrayGrouperTest.java +++ b/processing/src/test/java/io/druid/query/groupby/epinephelinae/BufferArrayGrouperTest.java @@ -86,4 +86,21 @@ private BufferArrayGrouper newGrouper( grouper.init(); return grouper; } + + @Test + public void testRequiredBufferCapacity() + { + int[] cardinalityArray = new int[] {1, 10, Integer.MAX_VALUE - 1}; + AggregatorFactory[] aggregatorFactories = new AggregatorFactory[] { + new LongSumAggregatorFactory("sum", "sum") + }; + + long[] requiredSizes = new long[] {17, 90, 16911433721L}; + + for (int i = 0; i < cardinalityArray.length; i++) { + Assert.assertEquals(requiredSizes[i], BufferArrayGrouper.requiredBufferCapacity( + cardinalityArray[i], + aggregatorFactories)); + } + } } From 05fc1a3365a9584138e6288ae1965a27ac0af1cc Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 10 Apr 2018 10:04:00 -0700 Subject: [PATCH 55/82] DoublesSketchModule: Fix serde for DoublesSketchMergeAggregatorFactory. (#5587) (#5618) Fixes #5580. --- .../quantiles/DoublesSketchModule.java | 2 + .../DoublesSketchAggregatorTest.java | 109 ++++++++++++------ 2 files changed, 78 insertions(+), 33 deletions(-) diff --git a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchModule.java b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchModule.java index 0e3081965b03..f26bf348a33a 100644 --- a/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchModule.java +++ b/extensions-core/datasketches/src/main/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchModule.java @@ -35,6 +35,7 @@ public class DoublesSketchModule implements DruidModule { public static final String DOUBLES_SKETCH = "quantilesDoublesSketch"; + public static final String DOUBLES_SKETCH_MERGE = "quantilesDoublesSketchMerge"; public static final String DOUBLES_SKETCH_HISTOGRAM_POST_AGG = "quantilesDoublesSketchToHistogram"; public static final String DOUBLES_SKETCH_QUANTILE_POST_AGG = "quantilesDoublesSketchToQuantile"; @@ -55,6 +56,7 @@ public List getJacksonModules() return Arrays. asList( new SimpleModule("DoublesQuantilesSketchModule").registerSubtypes( new NamedType(DoublesSketchAggregatorFactory.class, DOUBLES_SKETCH), + new NamedType(DoublesSketchMergeAggregatorFactory.class, DOUBLES_SKETCH_MERGE), new NamedType(DoublesSketchToHistogramPostAggregator.class, DOUBLES_SKETCH_HISTOGRAM_POST_AGG), new NamedType(DoublesSketchToQuantilePostAggregator.class, DOUBLES_SKETCH_QUANTILE_POST_AGG), new NamedType(DoublesSketchToQuantilesPostAggregator.class, DOUBLES_SKETCH_QUANTILES_POST_AGG), diff --git a/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java index 899c8392e4fd..ea07be5567b0 100644 --- a/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/io/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java @@ -28,6 +28,7 @@ import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; import io.druid.query.aggregation.AggregationTestHelper; +import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.groupby.GroupByQueryConfig; import io.druid.query.groupby.GroupByQueryRunnerTest; import org.junit.Assert; @@ -60,7 +61,8 @@ public DoublesSketchAggregatorTest(final GroupByQueryConfig config) module.getJacksonModules(), config, tempFolder); timeSeriesHelper = AggregationTestHelper.createTimeseriesQueryAggregationTestHelper( module.getJacksonModules(), - tempFolder); + tempFolder + ); } @Parameterized.Parameters(name = "{0}") @@ -68,7 +70,7 @@ public static Collection constructorFeeder() { final List constructors = Lists.newArrayList(); for (GroupByQueryConfig config : GroupByQueryRunnerTest.testConfigs()) { - constructors.add(new Object[] {config}); + constructors.add(new Object[]{config}); } return constructors; } @@ -78,11 +80,29 @@ public static Collection constructorFeeder() public void serializeDeserializeFactoryWithFieldName() throws Exception { ObjectMapper objectMapper = new DefaultObjectMapper(); + new DoublesSketchModule().getJacksonModules().forEach(objectMapper::registerModule); DoublesSketchAggregatorFactory factory = new DoublesSketchAggregatorFactory("name", "filedName", 128); - DoublesSketchAggregatorFactory other = objectMapper.readValue( + AggregatorFactory other = objectMapper.readValue( objectMapper.writeValueAsString(factory), - DoublesSketchAggregatorFactory.class); + AggregatorFactory.class + ); + + Assert.assertEquals(factory, other); + } + + // this is to test Json properties and equals for the combining factory + @Test + public void serializeDeserializeCombiningFactoryWithFieldName() throws Exception + { + ObjectMapper objectMapper = new DefaultObjectMapper(); + new DoublesSketchModule().getJacksonModules().forEach(objectMapper::registerModule); + DoublesSketchAggregatorFactory factory = new DoublesSketchMergeAggregatorFactory("name", 128); + + AggregatorFactory other = objectMapper.readValue( + objectMapper.writeValueAsString(factory), + AggregatorFactory.class + ); Assert.assertEquals(factory, other); } @@ -92,7 +112,8 @@ public void ingestingSketches() throws Exception { Sequence seq = helper.createIndexAndRunQueryOnSegment( new File(this.getClass().getClassLoader().getResource("quantiles/doubles_sketch_data.tsv").getFile()), - String.join("\n", + String.join( + "\n", "{", " \"type\": \"string\",", " \"parseSpec\": {", @@ -105,16 +126,20 @@ public void ingestingSketches() throws Exception " },", " \"columns\": [\"timestamp\", \"product\", \"sketch\"]", " }", - "}"), - String.join("\n", + "}" + ), + String.join( + "\n", "[", " {\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"sketch\", \"k\": 128},", " {\"type\": \"quantilesDoublesSketch\", \"name\": \"non_existent_sketch\", \"fieldName\": \"non_existent_sketch\", \"k\": 128}", - "]"), + "]" + ), 0, // minTimestamp Granularities.NONE, 10, // maxRowCount - String.join("\n", + String.join( + "\n", "{", " \"queryType\": \"groupBy\",", " \"dataSource\": \"test_datasource\",", @@ -129,8 +154,10 @@ public void ingestingSketches() throws Exception " {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}", " ],", " \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]", - "}")); - List results = Sequences.toList(seq, new ArrayList()); + "}" + ) + ); + List results = Sequences.toList(seq, new ArrayList<>()); Assert.assertEquals(1, results.size()); Row row = results.get(0); @@ -156,9 +183,9 @@ public void ingestingSketches() throws Exception Object histogramObject = row.getRaw("histogram"); Assert.assertTrue(histogramObject instanceof double[]); double[] histogram = (double[]) histogramObject; - for (final double bin: histogram) { - Assert.assertEquals(100, bin, 100 * 0.2); // 400 items uniformly - // distributed into 4 bins + for (final double bin : histogram) { + // 400 items uniformly distributed into 4 bins + Assert.assertEquals(100, bin, 100 * 0.2); } } @@ -167,7 +194,8 @@ public void buildingSketchesAtIngestionTime() throws Exception { Sequence seq = helper.createIndexAndRunQueryOnSegment( new File(this.getClass().getClassLoader().getResource("quantiles/doubles_build_data.tsv").getFile()), - String.join("\n", + String.join( + "\n", "{", " \"type\": \"string\",", " \"parseSpec\": {", @@ -180,12 +208,14 @@ public void buildingSketchesAtIngestionTime() throws Exception " },", " \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\"]", " }", - "}"), + "}" + ), "[{\"type\": \"quantilesDoublesSketch\", \"name\": \"sketch\", \"fieldName\": \"value\", \"k\": 128}]", 0, // minTimestamp Granularities.NONE, 10, // maxRowCount - String.join("\n", + String.join( + "\n", "{", " \"queryType\": \"groupBy\",", " \"dataSource\": \"test_datasource\",", @@ -200,8 +230,10 @@ public void buildingSketchesAtIngestionTime() throws Exception " {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}", " ],", " \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]", - "}")); - List results = Sequences.toList(seq, new ArrayList()); + "}" + ) + ); + List results = Sequences.toList(seq, new ArrayList<>()); Assert.assertEquals(1, results.size()); Row row = results.get(0); @@ -225,7 +257,7 @@ public void buildingSketchesAtIngestionTime() throws Exception Assert.assertEquals(4, histogram.length); for (final double bin: histogram) { Assert.assertEquals(100, bin, 100 * 0.2); // 400 items uniformly - // distributed into 4 bins + // distributed into 4 bins } } @@ -234,7 +266,8 @@ public void buildingSketchesAtQueryTime() throws Exception { Sequence seq = helper.createIndexAndRunQueryOnSegment( new File(this.getClass().getClassLoader().getResource("quantiles/doubles_build_data.tsv").getFile()), - String.join("\n", + String.join( + "\n", "{", " \"type\": \"string\",", " \"parseSpec\": {", @@ -247,12 +280,14 @@ public void buildingSketchesAtQueryTime() throws Exception " },", " \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\"]", " }", - "}"), + "}" + ), "[{\"type\": \"doubleSum\", \"name\": \"value\", \"fieldName\": \"value\"}]", 0, // minTimestamp Granularities.NONE, 10, // maxRowCount - String.join("\n", + String.join( + "\n", "{", " \"queryType\": \"groupBy\",", " \"dataSource\": \"test_datasource\",", @@ -267,8 +302,10 @@ public void buildingSketchesAtQueryTime() throws Exception " {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}", " ],", " \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]", - "}")); - List results = Sequences.toList(seq, new ArrayList()); + "}" + ) + ); + List results = Sequences.toList(seq, new ArrayList<>()); Assert.assertEquals(1, results.size()); Row row = results.get(0); @@ -296,7 +333,7 @@ public void buildingSketchesAtQueryTime() throws Exception double[] histogram = (double[]) histogramObject; for (final double bin: histogram) { Assert.assertEquals(100, bin, 100 * 0.2); // 400 items uniformly - // distributed into 4 bins + // distributed into 4 bins } } @@ -319,7 +356,8 @@ public void QueryingDataWithFieldNameValueAsFloatInsteadOfSketch() throws Except " },", " \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\"]", " }", - "}"), + "}" + ), "[{\"type\": \"doubleSum\", \"name\": \"value\", \"fieldName\": \"value\"}]", 0, // minTimestamp Granularities.NONE, @@ -340,8 +378,10 @@ public void QueryingDataWithFieldNameValueAsFloatInsteadOfSketch() throws Except " {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}", " ],", " \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]", - "}")); - List results = Sequences.toList(seq, new ArrayList()); + "}" + ) + ); + List results = Sequences.toList(seq, new ArrayList<>()); Assert.assertEquals(1, results.size()); Row row = results.get(0); @@ -369,7 +409,7 @@ public void QueryingDataWithFieldNameValueAsFloatInsteadOfSketch() throws Except double[] histogram = (double[]) histogramObject; for (final double bin: histogram) { Assert.assertEquals(100, bin, 100 * 0.2); // 400 items uniformly - // distributed into 4 bins + // distributed into 4 bins } } @@ -392,7 +432,8 @@ public void TimeSeriesQueryInputAsFloat() throws Exception " },", " \"columns\": [\"timestamp\", \"sequenceNumber\", \"product\", \"value\"]", " }", - "}"), + "}" + ), "[{\"type\": \"doubleSum\", \"name\": \"value\", \"fieldName\": \"value\"}]", 0, // minTimestamp Granularities.NONE, @@ -412,8 +453,10 @@ public void TimeSeriesQueryInputAsFloat() throws Exception " {\"type\": \"quantilesDoublesSketchToHistogram\", \"name\": \"histogram1\", \"splitPoints\": [0.25, 0.5, 0.75], \"field\": {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}}", " ],", " \"intervals\": [\"2016-01-01T00:00:00.000Z/2016-01-31T00:00:00.000Z\"]", - "}")); - List results = Sequences.toList(seq, new ArrayList()); + "}" + ) + ); + List results = Sequences.toList(seq, new ArrayList<>()); Assert.assertEquals(1, results.size()); } } From 3a51416620cb049f7bd18c6b204ddecc177ca423 Mon Sep 17 00:00:00 2001 From: Nishant Bangarwa Date: Tue, 10 Apr 2018 22:34:17 +0530 Subject: [PATCH 56/82] Fix Kerberos Authentication failing requests without cookies and excludedPaths config. (#5596) (#5621) * Fix Kerberos Authentication failing requests without cookies. KerberosAuthenticator was failing `First` request from the clients. After authentication we were setting the cookie properly but not setting the the authenticated flag in the request. This PR fixed that. Additional Fixes - * Removing of Unused SpnegoFilterConfig - replaced by KerberosAuthenticator * Unused internalClientKeytab and principal from KerberosAuthenticator * Fix docs accordingly and add docs for configuring an escalated client. * Fix excluded path config behavior * spelling correction * Revert "spelling correction" This reverts commit fb754b43d86d7183e661d6b2027be52efff8a83b. * Revert "Fix excluded path config behavior" This reverts commit 390104776991a368f838d9326bc48cb0ddd1f870. --- .../extensions-core/druid-kerberos.md | 13 +- .../kerberos/KerberosAuthenticator.java | 11 +- .../security/kerberos/SpnegoFilterConfig.java | 134 ------------------ .../kerberos/SpnegoFilterConfigTest.java | 75 ---------- 4 files changed, 16 insertions(+), 217 deletions(-) delete mode 100644 extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/SpnegoFilterConfig.java delete mode 100644 extensions-core/druid-kerberos/src/test/java/io/druid/security/kerberos/SpnegoFilterConfigTest.java diff --git a/docs/content/development/extensions-core/druid-kerberos.md b/docs/content/development/extensions-core/druid-kerberos.md index 1ded0e0f5a54..dc0d38a7b5af 100644 --- a/docs/content/development/extensions-core/druid-kerberos.md +++ b/docs/content/development/extensions-core/druid-kerberos.md @@ -31,8 +31,6 @@ The configuration examples in the rest of this document will use "kerberos" as t ### Properties |Property|Possible Values|Description|Default|required| |--------|---------------|-----------|-------|--------| -|`druid.auth.authenticator.kerberos.internalClientPrincipal`|`druid@EXAMPLE.COM`| Principal user name, used for internal node communication|empty|Yes| -|`druid.auth.authenticator.kerberos.internalClientKeytab`|`/etc/security/keytabs/druid.keytab`|Path to keytab file used for internal node communication|empty|Yes| |`druid.auth.authenticator.kerberos.serverPrincipal`|`HTTP/_HOST@EXAMPLE.COM`| SPNego service principal used by druid nodes|empty|Yes| |`druid.auth.authenticator.kerberos.serverKeytab`|`/etc/security/keytabs/spnego.service.keytab`|SPNego service keytab used by druid nodes|empty|Yes| |`druid.auth.authenticator.kerberos.authToLocal`|`RULE:[1:$1@$0](druid@EXAMPLE.COM)s/.*/druid DEFAULT`|It allows you to set a general rule for mapping principal names to local user names. It will be used if there is not an explicit mapping for the principal name that is being translated.|DEFAULT|No| @@ -54,6 +52,17 @@ In Active Directory environment, SPNEGO token in the Authorization header includ which includes all security groups for the user. In some cases when the user belongs to many security groups the header to grow beyond what druid can handle by default. In such cases, max request header size that druid can handle can be increased by setting `druid.server.http.maxRequestHeaderSize` (default 8Kb) and `druid.router.http.maxRequestBufferSize` (default 8Kb). +## Configuring Kerberos Escalated Client + +Druid internal nodes communicate with each other using an escalated http Client. A Kerberos enabled escalated HTTP Client can be configured by following properties - + + +|Property|Example Values|Description|Default|required| +|--------|---------------|-----------|-------|--------| +|`druid.escalator.type`|`kerberos`| Type of Escalator client used for internal node communication.|n/a|Yes| +|`druid.escalator.internalClientPrincipal`|`druid@EXAMPLE.COM`| Principal user name, used for internal node communication|n/a|Yes| +|`druid.escalator.internalClientKeytab`|`/etc/security/keytabs/druid.keytab`|Path to keytab file used for internal node communication|n/a|Yes| +|`druid.escalator.authorizerName`|`MyBasicAuthorizer`|Authorizer that requests should be directed to.|n/a|Yes| ## Accessing Druid HTTP end points when kerberos security is enabled 1. To access druid HTTP endpoints via curl user will need to first login using `kinit` command as follows - diff --git a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java index 785a5a415cb5..d0b7bcaabfbd 100644 --- a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java +++ b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java @@ -95,8 +95,6 @@ public class KerberosAuthenticator implements Authenticator private final DruidNode node; private final String serverPrincipal; private final String serverKeytab; - private final String internalClientPrincipal; - private final String internalClientKeytab; private final String authToLocal; private final List excludedPaths; private final String cookieSignatureSecret; @@ -107,8 +105,6 @@ public class KerberosAuthenticator implements Authenticator public KerberosAuthenticator( @JsonProperty("serverPrincipal") String serverPrincipal, @JsonProperty("serverKeytab") String serverKeytab, - @JsonProperty("internalClientPrincipal") String internalClientPrincipal, - @JsonProperty("internalClientKeytab") String internalClientKeytab, @JsonProperty("authToLocal") String authToLocal, @JsonProperty("excludedPaths") List excludedPaths, @JsonProperty("cookieSignatureSecret") String cookieSignatureSecret, @@ -119,8 +115,6 @@ public KerberosAuthenticator( this.node = node; this.serverPrincipal = serverPrincipal; this.serverKeytab = serverKeytab; - this.internalClientPrincipal = internalClientPrincipal; - this.internalClientKeytab = internalClientKeytab; this.authToLocal = authToLocal == null ? "DEFAULT" : authToLocal; this.excludedPaths = excludedPaths == null ? DEFAULT_EXCLUDED_PATHS : excludedPaths; this.cookieSignatureSecret = cookieSignatureSecret; @@ -334,6 +328,11 @@ public Principal getUserPrincipal() getCookiePath(), token.getExpires(), isHttps ); } + // Since this request is validated also set DRUID_AUTHENTICATION_RESULT + request.setAttribute( + AuthConfig.DRUID_AUTHENTICATION_RESULT, + new AuthenticationResult(token.getName(), authorizerName, null) + ); doFilter(filterChain, httpRequest, httpResponse); } } else { diff --git a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/SpnegoFilterConfig.java b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/SpnegoFilterConfig.java deleted file mode 100644 index 4dd4f9b01f97..000000000000 --- a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/SpnegoFilterConfig.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.security.kerberos; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import java.util.Collections; -import java.util.List; - -public class SpnegoFilterConfig -{ - - public static final List DEFAULT_EXCLUDED_PATHS = Collections.emptyList(); - - @JsonProperty - private final String principal; - - @JsonProperty - private final String keytab; - - @JsonProperty - private final String authToLocal; - - @JsonProperty - private final List excludedPaths; - - @JsonProperty - private final String cookieSignatureSecret; - - @JsonCreator - public SpnegoFilterConfig( - @JsonProperty("principal") String principal, - @JsonProperty("keytab") String keytab, - @JsonProperty("authToLocal") String authToLocal, - @JsonProperty("excludedPaths") List excludedPaths, - @JsonProperty("cookieSignatureSecret") String cookieSignatureSecret - ) - { - this.principal = principal; - this.keytab = keytab; - this.authToLocal = authToLocal == null ? "DEFAULT" : authToLocal; - this.excludedPaths = excludedPaths == null ? DEFAULT_EXCLUDED_PATHS : excludedPaths; - this.cookieSignatureSecret = cookieSignatureSecret; - } - - @JsonProperty - public String getPrincipal() - { - return principal; - } - - @JsonProperty - public String getKeytab() - { - return keytab; - } - - @JsonProperty - public String getAuthToLocal() - { - return authToLocal; - } - - @JsonProperty - public List getExcludedPaths() - { - return excludedPaths; - } - - @JsonProperty - public String getCookieSignatureSecret() - { - return cookieSignatureSecret; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - SpnegoFilterConfig that = (SpnegoFilterConfig) o; - - if (principal != null ? !principal.equals(that.principal) : that.principal != null) { - return false; - } - if (keytab != null ? !keytab.equals(that.keytab) : that.keytab != null) { - return false; - } - if (authToLocal != null ? !authToLocal.equals(that.authToLocal) : that.authToLocal != null) { - return false; - } - if (excludedPaths != null ? !excludedPaths.equals(that.excludedPaths) : that.excludedPaths != null) { - return false; - } - return cookieSignatureSecret != null - ? cookieSignatureSecret.equals(that.cookieSignatureSecret) - : that.cookieSignatureSecret == null; - - } - - @Override - public int hashCode() - { - int result = principal != null ? principal.hashCode() : 0; - result = 31 * result + (keytab != null ? keytab.hashCode() : 0); - result = 31 * result + (authToLocal != null ? authToLocal.hashCode() : 0); - result = 31 * result + (excludedPaths != null ? excludedPaths.hashCode() : 0); - result = 31 * result + (cookieSignatureSecret != null ? cookieSignatureSecret.hashCode() : 0); - return result; - } -} diff --git a/extensions-core/druid-kerberos/src/test/java/io/druid/security/kerberos/SpnegoFilterConfigTest.java b/extensions-core/druid-kerberos/src/test/java/io/druid/security/kerberos/SpnegoFilterConfigTest.java deleted file mode 100644 index e1ee98606324..000000000000 --- a/extensions-core/druid-kerberos/src/test/java/io/druid/security/kerberos/SpnegoFilterConfigTest.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.security.kerberos; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.inject.Binder; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.Provides; -import io.druid.guice.ConfigModule; -import io.druid.guice.DruidGuiceExtensions; -import io.druid.guice.JsonConfigProvider; -import io.druid.guice.LazySingleton; -import io.druid.guice.PropertiesModule; -import io.druid.jackson.DefaultObjectMapper; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Arrays; -import java.util.Properties; - -public class SpnegoFilterConfigTest -{ - @Test - public void testserde() - { - Injector injector = Guice.createInjector( - new Module() - { - @Override - public void configure(Binder binder) - { - binder.install(new PropertiesModule(Arrays.asList("test.runtime.properties"))); - binder.install(new ConfigModule()); - binder.install(new DruidGuiceExtensions()); - JsonConfigProvider.bind(binder, "druid.hadoop.security.spnego", SpnegoFilterConfig.class); - } - - @Provides - @LazySingleton - public ObjectMapper jsonMapper() - { - return new DefaultObjectMapper(); - } - } - ); - - Properties props = injector.getInstance(Properties.class); - SpnegoFilterConfig config = injector.getInstance(SpnegoFilterConfig.class); - - Assert.assertEquals(props.getProperty("druid.hadoop.security.spnego.principal"), config.getPrincipal()); - Assert.assertEquals(props.getProperty("druid.hadoop.security.spnego.keytab"), config.getKeytab()); - Assert.assertEquals(props.getProperty("druid.hadoop.security.spnego.authToLocal"), config.getAuthToLocal()); - - - } -} From 0b4632cac315a5f9cf67f877dab7287accec8ba6 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 10 Apr 2018 10:55:13 -0700 Subject: [PATCH 57/82] [Backport] Add missing type for MapVirtualColumn (#5616) --- .../io/druid/segment/DruidVirtualColumnsModule.java | 8 +++++++- .../java/io/druid/segment/MapVirtualColumnTest.java | 13 +++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/DruidVirtualColumnsModule.java b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/DruidVirtualColumnsModule.java index 184225d02453..9db5be737a22 100644 --- a/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/DruidVirtualColumnsModule.java +++ b/extensions-contrib/virtual-columns/src/main/java/io/druid/segment/DruidVirtualColumnsModule.java @@ -20,6 +20,7 @@ package io.druid.segment; import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.jsontype.NamedType; import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; @@ -34,7 +35,12 @@ public class DruidVirtualColumnsModule implements DruidModule @Override public List getJacksonModules() { - return ImmutableList.of(new SimpleModule().registerSubtypes(MapVirtualColumn.class)); + return ImmutableList.of( + new SimpleModule(getClass().getSimpleName()) + .registerSubtypes( + new NamedType(MapVirtualColumn.class, "map") + ) + ); } @Override diff --git a/extensions-contrib/virtual-columns/src/test/java/io/druid/segment/MapVirtualColumnTest.java b/extensions-contrib/virtual-columns/src/test/java/io/druid/segment/MapVirtualColumnTest.java index 283aa8684a9f..cc9417f7a0b3 100644 --- a/extensions-contrib/virtual-columns/src/test/java/io/druid/segment/MapVirtualColumnTest.java +++ b/extensions-contrib/virtual-columns/src/test/java/io/druid/segment/MapVirtualColumnTest.java @@ -19,6 +19,7 @@ package io.druid.segment; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; @@ -138,6 +139,18 @@ private Druids.SelectQueryBuilder testBuilder() .pagingSpec(new PagingSpec(null, 3)); } + @Test + public void testSerde() throws IOException + { + final ObjectMapper mapper = new DefaultObjectMapper(); + new DruidVirtualColumnsModule().getJacksonModules().forEach(mapper::registerModule); + + final MapVirtualColumn column = new MapVirtualColumn("keys", "values", "params"); + final String json = mapper.writeValueAsString(column); + final VirtualColumn fromJson = mapper.readValue(json, VirtualColumn.class); + Assert.assertEquals(column, fromJson); + } + @Test public void testBasic() throws Exception { From aa37a783e26f7bd759c6959b4de826cf39b7e46d Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 10 Apr 2018 10:55:25 -0700 Subject: [PATCH 58/82] Fix supervisor tombstone auth handling (#5504) (#5617) --- .../supervisor/SupervisorManager.java | 5 +- .../supervisor/SupervisorResource.java | 3 +- .../supervisor/SupervisorManagerTest.java | 3 +- .../supervisor/SupervisorResourceTest.java | 181 ++++++++++++++++++ .../supervisor/NoopSupervisorSpec.java | 52 ++++- 5 files changed, 238 insertions(+), 6 deletions(-) diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorManager.java b/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorManager.java index 5c1eecc38ff3..00f2c92c4b75 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorManager.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorManager.java @@ -71,6 +71,7 @@ public boolean createOrUpdateAndStartSupervisor(SupervisorSpec spec) Preconditions.checkState(started, "SupervisorManager not started"); Preconditions.checkNotNull(spec, "spec"); Preconditions.checkNotNull(spec.getId(), "spec.getId()"); + Preconditions.checkNotNull(spec.getDataSources(), "spec.getDatasources()"); synchronized (lock) { Preconditions.checkState(started, "SupervisorManager not started"); @@ -197,7 +198,7 @@ private boolean possiblyStopAndRemoveSupervisorInternal(String id, boolean write } if (writeTombstone) { - metadataSupervisorManager.insert(id, new NoopSupervisorSpec()); // where NoopSupervisorSpec is a tombstone + metadataSupervisorManager.insert(id, new NoopSupervisorSpec(null, pair.rhs.getDataSources())); // where NoopSupervisorSpec is a tombstone } pair.lhs.stop(true); supervisors.remove(id); @@ -232,7 +233,7 @@ private boolean createAndStartSupervisorInternal(SupervisorSpec spec, boolean pe catch (Exception e) { // Supervisor creation or start failed write tombstone only when trying to start a new supervisor if (persistSpec) { - metadataSupervisorManager.insert(id, new NoopSupervisorSpec()); + metadataSupervisorManager.insert(id, new NoopSupervisorSpec(null, spec.getDataSources())); } Throwables.propagate(e); } diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorResource.java b/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorResource.java index 348c40a58ab4..09196df401c3 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorResource.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/supervisor/SupervisorResource.java @@ -48,6 +48,7 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; @@ -66,7 +67,7 @@ public class SupervisorResource return null; } if (supervisorSpec.getSpec().getDataSources() == null) { - return null; + return new ArrayList<>(); } return Iterables.transform( supervisorSpec.getSpec().getDataSources(), diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorManagerTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorManagerTest.java index c22dde9eb1f4..8415b919b721 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorManagerTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorManagerTest.java @@ -35,6 +35,7 @@ import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -295,7 +296,7 @@ public Supervisor createSupervisor() @Override public List getDataSources() { - return null; + return new ArrayList<>(); } } diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorResourceTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorResourceTest.java index 43283997031d..3c0a2bd1585c 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorResourceTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorResourceTest.java @@ -19,6 +19,7 @@ package io.druid.indexing.overlord.supervisor; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -295,6 +296,10 @@ public void testSpecGetAllHistory() throws Exception new VersionedSupervisorSpec( new TestSupervisorSpec("id1", null, Arrays.asList("datasource1")), "v2" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, Arrays.asList("datasource1")), + "tombstone" ) ); List versions2 = ImmutableList.of( @@ -305,11 +310,42 @@ public void testSpecGetAllHistory() throws Exception new VersionedSupervisorSpec( new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), "v2" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, Arrays.asList("datasource2")), + "tombstone" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), + "v3" + ) + ); + List versions3 = ImmutableList.of( + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource3")), + "v1" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, null), + "tombstone" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource3")), + "v2" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, null), + "tombstone" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource3")), + "v3" ) ); Map> history = Maps.newHashMap(); history.put("id1", versions1); history.put("id2", versions2); + history.put("id3", versions3); EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.of(supervisorManager)).times(2); EasyMock.expect(supervisorManager.getSupervisorHistory()).andReturn(history); @@ -352,6 +388,10 @@ public void testSpecGetAllHistoryWithAuthFailureFiltering() throws Exception new VersionedSupervisorSpec( new TestSupervisorSpec("id1", null, Arrays.asList("datasource1")), "v2" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, Arrays.asList("datasource1")), + "tombstone" ) ); List versions2 = ImmutableList.of( @@ -362,12 +402,62 @@ public void testSpecGetAllHistoryWithAuthFailureFiltering() throws Exception new VersionedSupervisorSpec( new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), "v2" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, Arrays.asList("datasource2")), + "tombstone" + ) + ); + List versions3 = ImmutableList.of( + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), + "v1" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), + "v2" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, Arrays.asList("datasource2")), + "tombstone" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id3", null, Arrays.asList("datasource3")), + "v1" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, Arrays.asList("datasource3")), + "tombstone" + ) + ); + List versions4 = ImmutableList.of( + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), + "v1" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, null), + "tombstone" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), + "v2" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, null), + "tombstone" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), + "v3" ) ); Map> history = Maps.newHashMap(); history.put("id1", versions1); history.put("id2", versions2); + history.put("id3", versions3); + history.put("id4", versions4); EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.of(supervisorManager)).times(2); EasyMock.expect(supervisorManager.getSupervisorHistory()).andReturn(history); @@ -387,6 +477,32 @@ public void testSpecGetAllHistoryWithAuthFailureFiltering() throws Exception Map> filteredHistory = Maps.newHashMap(); filteredHistory.put("id1", versions1); + filteredHistory.put( + "id3", + ImmutableList.of( + new VersionedSupervisorSpec( + new TestSupervisorSpec("id3", null, Arrays.asList("datasource3")), + "v1" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, Arrays.asList("datasource3")), + "tombstone" + ) + ) + ); + filteredHistory.put( + "id4", + ImmutableList.of( + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, null), + "tombstone" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, null), + "tombstone" + ) + ) + ); Assert.assertEquals(200, response.getStatus()); Assert.assertEquals(filteredHistory, response.getEntity()); @@ -410,6 +526,10 @@ public void testSpecGetHistory() throws Exception new TestSupervisorSpec("id1", null, Arrays.asList("datasource1")), "v1" ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, Arrays.asList("datasource1")), + "tombstone" + ), new VersionedSupervisorSpec( new TestSupervisorSpec("id1", null, Arrays.asList("datasource1")), "v2" @@ -420,6 +540,10 @@ public void testSpecGetHistory() throws Exception new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), "v1" ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, Arrays.asList("datasource2")), + "tombstone" + ), new VersionedSupervisorSpec( new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), "v2" @@ -472,6 +596,10 @@ public void testSpecGetHistoryWithAuthFailure() throws Exception new TestSupervisorSpec("id1", null, Arrays.asList("datasource1")), "v1" ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, Arrays.asList("datasource3")), + "tombstone" + ), new VersionedSupervisorSpec( new TestSupervisorSpec("id1", null, Arrays.asList("datasource1")), "v2" @@ -482,6 +610,10 @@ public void testSpecGetHistoryWithAuthFailure() throws Exception new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), "v1" ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, Arrays.asList("datasource2")), + "tombstone" + ), new VersionedSupervisorSpec( new TestSupervisorSpec("id2", null, Arrays.asList("datasource2")), "v2" @@ -492,9 +624,25 @@ public void testSpecGetHistoryWithAuthFailure() throws Exception new TestSupervisorSpec("id3", null, Arrays.asList("datasource3")), "v1" ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, null), + "tombstone" + ), new VersionedSupervisorSpec( new TestSupervisorSpec("id3", null, Arrays.asList("datasource2")), "v2" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, null), + "tombstone" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id3", null, Arrays.asList("datasource3")), + "v2" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, Arrays.asList("datasource3")), + "tombstone" ) ); Map> history = Maps.newHashMap(); @@ -529,6 +677,22 @@ public void testSpecGetHistoryWithAuthFailure() throws Exception new VersionedSupervisorSpec( new TestSupervisorSpec("id3", null, Arrays.asList("datasource3")), "v1" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, null), + "tombstone" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, null), + "tombstone" + ), + new VersionedSupervisorSpec( + new TestSupervisorSpec("id3", null, Arrays.asList("datasource3")), + "v2" + ), + new VersionedSupervisorSpec( + new NoopSupervisorSpec(null, Arrays.asList("datasource3")), + "tombstone" ) ), response.getEntity() @@ -582,6 +746,23 @@ public void testReset() throws Exception verifyAll(); } + @Test + public void testNoopSupervisorSpecSerde() throws Exception + { + ObjectMapper mapper = new ObjectMapper(); + String oldSpec = "{\"type\":\"NoopSupervisorSpec\",\"id\":null,\"dataSources\":null}"; + NoopSupervisorSpec expectedSpec = new NoopSupervisorSpec(null, null); + NoopSupervisorSpec deserializedSpec = mapper.readValue(oldSpec, NoopSupervisorSpec.class); + Assert.assertEquals(expectedSpec, deserializedSpec); + + NoopSupervisorSpec spec1 = new NoopSupervisorSpec("abcd", Lists.newArrayList("defg")); + NoopSupervisorSpec spec2 = mapper.readValue( + mapper.writeValueAsBytes(spec1), + NoopSupervisorSpec.class + ); + Assert.assertEquals(spec1, spec2); + } + private static class TestSupervisorSpec implements SupervisorSpec { private final String id; diff --git a/server/src/main/java/io/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java b/server/src/main/java/io/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java index 401023fe032a..380861d9fb8f 100644 --- a/server/src/main/java/io/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java +++ b/server/src/main/java/io/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java @@ -19,20 +19,46 @@ package io.druid.indexing.overlord.supervisor; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import io.druid.indexing.overlord.DataSourceMetadata; import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.List; +import java.util.Objects; /** * Used as a tombstone marker in the supervisors metadata table to indicate that the supervisor has been removed. */ public class NoopSupervisorSpec implements SupervisorSpec { + // NoopSupervisorSpec is used as a tombstone, added when a previously running supervisor is stopped. + // Inherit the datasources of the previous running spec, so that we can determine whether a user is authorized to see + // this tombstone (users can only see tombstones for datasources that they have access to). + @Nullable + @JsonProperty("dataSources") + private List datasources; + + @Nullable + @JsonProperty("id") + private String id; + + @JsonCreator + public NoopSupervisorSpec( + @Nullable @JsonProperty("id") String id, + @Nullable @JsonProperty("dataSources") List datasources + ) + { + this.id = id; + this.datasources = datasources == null ? new ArrayList<>() : datasources; + } + @Override + @JsonProperty public String getId() { - return null; + return id; } @Override @@ -68,8 +94,30 @@ public void checkpoint( } @Override + @Nullable + @JsonProperty("dataSources") public List getDataSources() { - return null; + return datasources; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + NoopSupervisorSpec spec = (NoopSupervisorSpec) o; + return Objects.equals(datasources, spec.datasources) && + Objects.equals(getId(), spec.getId()); + } + + @Override + public int hashCode() + { + return Objects.hash(datasources, getId()); } } From ce89de570248ad573dc23c3ffae2bd6273341c74 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Thu, 12 Apr 2018 12:55:33 -0400 Subject: [PATCH 59/82] ParallelCombiner: Fix buffer leak on exception in "combine". (#5630) (#5633) Once a buffer is acquired, we need to make sure to release it if an exception is thrown before the closeable iterator is created. --- .../epinephelinae/ParallelCombiner.java | 83 +++++++++++-------- 1 file changed, 47 insertions(+), 36 deletions(-) diff --git a/processing/src/main/java/io/druid/query/groupby/epinephelinae/ParallelCombiner.java b/processing/src/main/java/io/druid/query/groupby/epinephelinae/ParallelCombiner.java index d043a9070d76..c3340d26df12 100644 --- a/processing/src/main/java/io/druid/query/groupby/epinephelinae/ParallelCombiner.java +++ b/processing/src/main/java/io/druid/query/groupby/epinephelinae/ParallelCombiner.java @@ -136,44 +136,55 @@ public CloseableIterator> combine( ) { // CombineBuffer is initialized when this method is called and closed after the result iterator is done + final Closer closer = Closer.create(); final ResourceHolder combineBufferHolder = combineBufferSupplier.get(); - final ByteBuffer combineBuffer = combineBufferHolder.get(); - final int minimumRequiredBufferCapacity = StreamingMergeSortedGrouper.requiredBufferCapacity( - combineKeySerdeFactory.factorizeWithDictionary(mergedDictionary), - combiningFactories - ); - // We want to maximize the parallelism while the size of buffer slice is greater than the minimum buffer size - // required by StreamingMergeSortedGrouper. Here, we find the leafCombineDegree of the cominbing tree and the - // required number of buffers maximizing the parallelism. - final Pair degreeAndNumBuffers = findLeafCombineDegreeAndNumBuffers( - combineBuffer, - minimumRequiredBufferCapacity, - concurrencyHint, - sortedIterators.size() - ); + closer.register(combineBufferHolder); - final int leafCombineDegree = degreeAndNumBuffers.lhs; - final int numBuffers = degreeAndNumBuffers.rhs; - final int sliceSize = combineBuffer.capacity() / numBuffers; + try { + final ByteBuffer combineBuffer = combineBufferHolder.get(); + final int minimumRequiredBufferCapacity = StreamingMergeSortedGrouper.requiredBufferCapacity( + combineKeySerdeFactory.factorizeWithDictionary(mergedDictionary), + combiningFactories + ); + // We want to maximize the parallelism while the size of buffer slice is greater than the minimum buffer size + // required by StreamingMergeSortedGrouper. Here, we find the leafCombineDegree of the cominbing tree and the + // required number of buffers maximizing the parallelism. + final Pair degreeAndNumBuffers = findLeafCombineDegreeAndNumBuffers( + combineBuffer, + minimumRequiredBufferCapacity, + concurrencyHint, + sortedIterators.size() + ); - final Supplier bufferSupplier = createCombineBufferSupplier(combineBuffer, numBuffers, sliceSize); + final int leafCombineDegree = degreeAndNumBuffers.lhs; + final int numBuffers = degreeAndNumBuffers.rhs; + final int sliceSize = combineBuffer.capacity() / numBuffers; - final Pair>>, List> combineIteratorAndFutures = buildCombineTree( - sortedIterators, - bufferSupplier, - combiningFactories, - leafCombineDegree, - mergedDictionary - ); + final Supplier bufferSupplier = createCombineBufferSupplier(combineBuffer, numBuffers, sliceSize); - final CloseableIterator> combineIterator = Iterables.getOnlyElement(combineIteratorAndFutures.lhs); - final List combineFutures = combineIteratorAndFutures.rhs; + final Pair>>, List> combineIteratorAndFutures = buildCombineTree( + sortedIterators, + bufferSupplier, + combiningFactories, + leafCombineDegree, + mergedDictionary + ); - final Closer closer = Closer.create(); - closer.register(combineBufferHolder); - closer.register(() -> checkCombineFutures(combineFutures)); + final CloseableIterator> combineIterator = Iterables.getOnlyElement(combineIteratorAndFutures.lhs); + final List combineFutures = combineIteratorAndFutures.rhs; + closer.register(() -> checkCombineFutures(combineFutures)); - return CloseableIterators.wrap(combineIterator, closer); + return CloseableIterators.wrap(combineIterator, closer); + } + catch (Throwable t) { + try { + closer.close(); + } + catch (Throwable t2) { + t.addSuppressed(t2); + } + throw t; + } } private static void checkCombineFutures(List combineFutures) @@ -289,11 +300,11 @@ private int computeRequiredBufferNum(int numChildNodes, int combineDegree) * Recursively build a combining tree in a bottom-up manner. Each node of the tree is a task that combines input * iterators asynchronously. * - * @param childIterators all iterators of the child level - * @param bufferSupplier combining buffer supplier - * @param combiningFactories array of combining aggregator factories - * @param combineDegree combining degree for the current level - * @param dictionary merged dictionary + * @param childIterators all iterators of the child level + * @param bufferSupplier combining buffer supplier + * @param combiningFactories array of combining aggregator factories + * @param combineDegree combining degree for the current level + * @param dictionary merged dictionary * * @return a pair of a list of iterators of the current level in the combining tree and a list of futures of all * executed combining tasks From 5936b4004ccf597f4e354ddb1caab8e7b61e9ac9 Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Thu, 12 Apr 2018 16:39:37 -0700 Subject: [PATCH 60/82] Fix coordinator loadStatus performance (#5632) (#5636) * Optimize coordinator loadStatus * Add comment * Fix teamcity * Checkstyle * More checkstyle * Checkstyle --- .../druid/benchmark/LoadStatusBenchmark.java | 121 ++++++++++++++++++ .../server/coordinator/DruidCoordinator.java | 6 +- 2 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 benchmarks/src/main/java/io/druid/benchmark/LoadStatusBenchmark.java diff --git a/benchmarks/src/main/java/io/druid/benchmark/LoadStatusBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/LoadStatusBenchmark.java new file mode 100644 index 000000000000..e1b877d481d8 --- /dev/null +++ b/benchmarks/src/main/java/io/druid/benchmark/LoadStatusBenchmark.java @@ -0,0 +1,121 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.benchmark; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import io.druid.java.util.common.Intervals; +import io.druid.java.util.common.StringUtils; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +@State(Scope.Benchmark) +@Fork(value = 1) +@Warmup(iterations = 15) +@Measurement(iterations = 30) +public class LoadStatusBenchmark +{ + // Number of total data segments + @Param({"10000"}) + int totalSegmentsCount; + + @Param({"true", "false"}) + private boolean serverHasAllSegments; + + private Set datasourceSegments; + private Collection serverSegments; + + @Setup(Level.Invocation) + public void setup() + { + Map immutableDatasourceSegmentsMap; + ConcurrentHashMap serverSegmentsMap; + + HashMap datasourceSegmentsMap = Maps.newHashMap(); + serverSegmentsMap = new ConcurrentHashMap<>(); + + for (int i = 0; i < totalSegmentsCount; i++) { + DataSegment segment = new DataSegment( + "benchmarkDatasource", + Intervals.of(StringUtils.format("%s-01-01/%s-12-31", i + 1970, i + 1970)), + "1", + null, + null, + null, + NoneShardSpec.instance(), + 1, + 1 + ); + + datasourceSegmentsMap.put(segment.getIdentifier(), segment); + + if (serverHasAllSegments || i % 2 == 0) { + serverSegmentsMap.put(segment.getIdentifier(), segment); + } + } + + immutableDatasourceSegmentsMap = ImmutableMap.copyOf(datasourceSegmentsMap); + + datasourceSegments = Sets.newHashSet(immutableDatasourceSegmentsMap.values()); + serverSegments = Collections.unmodifiableCollection(serverSegmentsMap.values()); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void oldVersion(Blackhole blackhole) + { + datasourceSegments.removeAll(serverSegments); + blackhole.consume(datasourceSegments); + } + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MICROSECONDS) + public void newVersion(Blackhole blackhole) + { + for (DataSegment segment : serverSegments) { + datasourceSegments.remove(segment); + } + blackhole.consume(datasourceSegments); + } +} diff --git a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java index 1be52f43cac9..29f6ee12190b 100644 --- a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java +++ b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java @@ -299,7 +299,11 @@ public Map getLoadStatus() for (DruidServer druidServer : serverInventoryView.getInventory()) { final DruidDataSource loadedView = druidServer.getDataSource(dataSource.getName()); if (loadedView != null) { - segments.removeAll(loadedView.getSegments()); + // This does not use segments.removeAll(loadedView.getSegments()) for performance reasons. + // Please see https://github.com/druid-io/druid/pull/5632 and LoadStatusBenchmark for more info. + for (DataSegment serverSegment : loadedView.getSegments()) { + segments.remove(serverSegment); + } } } final int unloadedSegmentSize = segments.size(); From 66ea105f61e92088ce09805c7cd1c85227e40a4f Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Fri, 13 Apr 2018 18:14:05 -0400 Subject: [PATCH 61/82] Replace EmittedBatchCounter and UpdateCounter with ConcurrentAwaitableCounter (#5592) (#5642) * Replace EmittedBatchCounter and UpdateCounter with (both not safe for concurrent increments/updates) with ConcurrentAwaitableCounter (safe for concurrent increments) * Fixes * Fix EmitterTest * Added Javadoc and make awaitCount() to throw exceptions on wrong count instead of masking errors --- .../namespace/cache/CacheScheduler.java | 17 +- .../lookup/namespace/cache/UpdateCounter.java | 88 ---------- .../ConcurrentAwaitableCounter.java | 166 ++++++++++++++++++ .../druid/java/util/emitter/core/Batch.java | 6 +- .../emitter/core/EmittedBatchCounter.java | 73 -------- .../util/emitter/core/HttpPostEmitter.java | 16 +- .../ConcurrentAwaitableCounterTest.java | 89 ++++++++++ .../java/util/emitter/core/EmitterTest.java | 22 +-- 8 files changed, 288 insertions(+), 189 deletions(-) delete mode 100644 extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/UpdateCounter.java create mode 100644 java-util/src/main/java/io/druid/concurrent/ConcurrentAwaitableCounter.java delete mode 100644 java-util/src/main/java/io/druid/java/util/emitter/core/EmittedBatchCounter.java create mode 100644 java-util/src/test/java/io/druid/concurrent/ConcurrentAwaitableCounterTest.java diff --git a/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/CacheScheduler.java b/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/CacheScheduler.java index 7669168d1f4a..1f1d3e3f8135 100644 --- a/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/CacheScheduler.java +++ b/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/CacheScheduler.java @@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Throwables; import com.google.inject.Inject; +import io.druid.concurrent.ConcurrentAwaitableCounter; import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.guice.LazySingleton; @@ -51,11 +52,11 @@ * // cacheState could be either NoCache or VersionedCache. * if (cacheState instanceof NoCache) { * // the cache is not yet created, or already closed - * } else if (cacheState instanceof VersionedCache) { + * } else { * Map cache = ((VersionedCache) cacheState).getCache(); // use the cache * // Although VersionedCache implements AutoCloseable, versionedCache shouldn't be manually closed * // when obtained from entry.getCacheState(). If the namespace updates should be ceased completely, - * // entry.close() (see below) should be called, it will close the last VersionedCache itself. + * // entry.close() (see below) should be called, it will close the last VersionedCache as well. * // On scheduled updates, outdated VersionedCaches are also closed automatically. * } * ... @@ -105,14 +106,16 @@ Future getUpdaterFuture() return impl.updaterFuture; } + @VisibleForTesting public void awaitTotalUpdates(int totalUpdates) throws InterruptedException { - impl.updateCounter.awaitTotalUpdates(totalUpdates); + impl.updateCounter.awaitCount(totalUpdates); } + @VisibleForTesting void awaitNextUpdates(int nextUpdates) throws InterruptedException { - impl.updateCounter.awaitNextUpdates(nextUpdates); + impl.updateCounter.awaitNextIncrements(nextUpdates); } /** @@ -145,7 +148,7 @@ public class EntryImpl implements AutoCloseable private final Future updaterFuture; private final Cleaner entryCleaner; private final CacheGenerator cacheGenerator; - private final UpdateCounter updateCounter = new UpdateCounter(); + private final ConcurrentAwaitableCounter updateCounter = new ConcurrentAwaitableCounter(); private final CountDownLatch startLatch = new CountDownLatch(1); private EntryImpl(final T namespace, final Entry entry, final CacheGenerator cacheGenerator) @@ -276,7 +279,7 @@ private CacheState swapCacheState(VersionedCache newVersionedCache) return lastCacheState; } } while (!cacheStateHolder.compareAndSet(lastCacheState, newVersionedCache)); - updateCounter.update(); + updateCounter.increment(); return lastCacheState; } @@ -485,7 +488,7 @@ public Entry scheduleAndWait(ExtractionNamespace namespace, long waitForFirstRun log.debug("Scheduled new %s", entry); boolean success = false; try { - success = entry.impl.updateCounter.awaitFirstUpdate(waitForFirstRunMs, TimeUnit.MILLISECONDS); + success = entry.impl.updateCounter.awaitFirstIncrement(waitForFirstRunMs, TimeUnit.MILLISECONDS); if (success) { return entry; } else { diff --git a/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/UpdateCounter.java b/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/UpdateCounter.java deleted file mode 100644 index ed229b45f891..000000000000 --- a/extensions-core/lookups-cached-global/src/main/java/io/druid/server/lookup/namespace/cache/UpdateCounter.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.server.lookup.namespace.cache; - -import java.util.concurrent.Phaser; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -final class UpdateCounter -{ - /** - * Max {@link Phaser}'s phase, specified in it's javadoc. Then it wraps to zero. - */ - private static final int MAX_PHASE = Integer.MAX_VALUE; - - private final Phaser phaser = new Phaser(1); - - void update() - { - phaser.arrive(); - } - - void awaitTotalUpdates(int totalUpdates) throws InterruptedException - { - totalUpdates &= MAX_PHASE; - int currentUpdates = phaser.getPhase(); - checkNotTerminated(currentUpdates); - while (comparePhases(totalUpdates, currentUpdates) > 0) { - currentUpdates = phaser.awaitAdvanceInterruptibly(currentUpdates); - checkNotTerminated(currentUpdates); - } - } - - private static int comparePhases(int phase1, int phase2) - { - int diff = (phase1 - phase2) & MAX_PHASE; - if (diff == 0) { - return 0; - } - return diff < MAX_PHASE / 2 ? 1 : -1; - } - - private void checkNotTerminated(int phase) - { - if (phase < 0) { - throw new IllegalStateException("Phaser[" + phaser + "] unexpectedly terminated."); - } - } - - void awaitNextUpdates(int nextUpdates) throws InterruptedException - { - if (nextUpdates <= 0) { - throw new IllegalArgumentException("nextUpdates is not positive: " + nextUpdates); - } - if (nextUpdates > MAX_PHASE / 4) { - throw new UnsupportedOperationException("Couldn't wait for so many updates: " + nextUpdates); - } - awaitTotalUpdates(phaser.getPhase() + nextUpdates); - } - - boolean awaitFirstUpdate(long timeout, TimeUnit unit) throws InterruptedException - { - try { - phaser.awaitAdvanceInterruptibly(0, timeout, unit); - return true; - } - catch (TimeoutException e) { - return false; - } - } -} diff --git a/java-util/src/main/java/io/druid/concurrent/ConcurrentAwaitableCounter.java b/java-util/src/main/java/io/druid/concurrent/ConcurrentAwaitableCounter.java new file mode 100644 index 000000000000..e9ffa61ff726 --- /dev/null +++ b/java-util/src/main/java/io/druid/concurrent/ConcurrentAwaitableCounter.java @@ -0,0 +1,166 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.concurrent; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.AbstractQueuedLongSynchronizer; + +/** + * This synchronization object allows to {@link #increment} a counter without blocking, potentially from multiple + * threads (although in some use cases there is just one incrementer thread), and block in other thread(s), awaiting + * when the count reaches the provided value: see {@link #awaitCount}, or the specified number of events since the + * call: see {@link #awaitNextIncrements}. + * + * This counter wraps around {@link Long#MAX_VALUE} and starts from 0 again, so "next" count should be generally + * obtained by calling {@link #nextCount nextCount(currentCount)} rather than {@code currentCount + 1}. + * + * Memory consistency effects: actions in threads prior to calling {@link #increment} while the count was less than the + * awaited value happen-before actions following count awaiting methods such as {@link #awaitCount}. + */ +public final class ConcurrentAwaitableCounter +{ + private static final long MAX_COUNT = Long.MAX_VALUE; + + /** + * This method should be called to obtain the next total increment count to be passed to {@link #awaitCount} methods, + * instead of just adding 1 to the previous count, because the count must wrap around {@link Long#MAX_VALUE} and start + * from 0 again. + */ + public static long nextCount(long prevCount) + { + return (prevCount + 1) & MAX_COUNT; + } + + private static class Sync extends AbstractQueuedLongSynchronizer + { + @Override + protected long tryAcquireShared(long countWhenWaitStarted) + { + long currentCount = getState(); + return compareCounts(currentCount, countWhenWaitStarted) > 0 ? 1 : -1; + } + + @Override + protected boolean tryReleaseShared(long increment) + { + long count; + long nextCount; + do { + count = getState(); + nextCount = (count + increment) & MAX_COUNT; + } while (!compareAndSetState(count, nextCount)); + return true; + } + + long getCount() + { + return getState(); + } + } + + private final Sync sync = new Sync(); + + /** + * Increment the count. This method could be safely called from concurrent threads. + */ + public void increment() + { + sync.releaseShared(1); + } + + /** + * Await until the {@link #increment} is called on this counter object the specified number of times from the creation + * of this counter object. + */ + public void awaitCount(long totalCount) throws InterruptedException + { + checkTotalCount(totalCount); + long currentCount = sync.getCount(); + while (compareCounts(totalCount, currentCount) > 0) { + sync.acquireSharedInterruptibly(currentCount); + currentCount = sync.getCount(); + } + } + + private static void checkTotalCount(long totalCount) + { + if (totalCount < 0) { + throw new AssertionError( + "Total count must always be >= 0, even in the face of overflow. " + + "The next count should always be obtained by calling ConcurrentAwaitableCounter.nextCount(prevCount), " + + "not just +1" + ); + } + } + + /** + * Await until the {@link #increment} is called on this counter object the specified number of times from the creation + * of this counter object, for not longer than the specified period of time. If by this time the target increment + * count is not reached, {@link TimeoutException} is thrown. + */ + public void awaitCount(long totalCount, long timeout, TimeUnit unit) throws InterruptedException, TimeoutException + { + checkTotalCount(totalCount); + long nanos = unit.toNanos(timeout); + long currentCount = sync.getCount(); + while (compareCounts(totalCount, currentCount) > 0) { + if (!sync.tryAcquireSharedNanos(currentCount, nanos)) { + throw new TimeoutException(); + } + currentCount = sync.getCount(); + } + } + + private static int compareCounts(long count1, long count2) + { + long diff = (count1 - count2) & MAX_COUNT; + if (diff == 0) { + return 0; + } + return diff < MAX_COUNT / 2 ? 1 : -1; + } + + /** + * Somewhat loosely defined wait for "next N increments", because the starting point is not defined from the Java + * Memory Model perspective. + */ + public void awaitNextIncrements(long nextIncrements) throws InterruptedException + { + if (nextIncrements <= 0) { + throw new IllegalArgumentException("nextIncrements is not positive: " + nextIncrements); + } + if (nextIncrements > MAX_COUNT / 4) { + throw new UnsupportedOperationException("Couldn't wait for so many increments: " + nextIncrements); + } + awaitCount((sync.getCount() + nextIncrements) & MAX_COUNT); + } + + /** + * The difference between this method and {@link #awaitCount(long, long, TimeUnit)} with argument 1 is that {@code + * awaitFirstIncrement()} returns boolean designating whether the count was await (while waiting for no longer than + * for the specified period of time), while {@code awaitCount()} throws {@link TimeoutException} if the count was not + * awaited. + */ + public boolean awaitFirstIncrement(long timeout, TimeUnit unit) throws InterruptedException + { + return sync.tryAcquireSharedNanos(0, unit.toNanos(timeout)); + } +} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/Batch.java b/java-util/src/main/java/io/druid/java/util/emitter/core/Batch.java index 818f54f91a94..cb0bc88042b1 100644 --- a/java-util/src/main/java/io/druid/java/util/emitter/core/Batch.java +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/Batch.java @@ -90,12 +90,12 @@ private static boolean isEmittingAllowed(long state) /** * Ordering number of this batch, as they filled & emitted in {@link HttpPostEmitter} serially, starting from 0. - * It's a boxed Integer rather than int, because we want to minimize the number of allocations done in + * It's a boxed Long rather than primitive long, because we want to minimize the number of allocations done in * {@link HttpPostEmitter#onSealExclusive} and so the probability of {@link OutOfMemoryError}. * @see HttpPostEmitter#onSealExclusive * @see HttpPostEmitter#concurrentBatch */ - final Integer batchNumber; + final Long batchNumber; /** * The number of events in this batch, needed for event count-based batch emitting. @@ -107,7 +107,7 @@ private static boolean isEmittingAllowed(long state) */ private long firstEventTimestamp = -1; - Batch(HttpPostEmitter emitter, byte[] buffer, int batchNumber) + Batch(HttpPostEmitter emitter, byte[] buffer, long batchNumber) { this.emitter = emitter; this.buffer = buffer; diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/EmittedBatchCounter.java b/java-util/src/main/java/io/druid/java/util/emitter/core/EmittedBatchCounter.java deleted file mode 100644 index 860e6957fe94..000000000000 --- a/java-util/src/main/java/io/druid/java/util/emitter/core/EmittedBatchCounter.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.java.util.emitter.core; - -import java.util.concurrent.Phaser; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - - -final class EmittedBatchCounter -{ - /** - * Max {@link Phaser}'s phase, specified in it's javadoc. Then it wraps to zero. - */ - private static final int MAX_PHASE = Integer.MAX_VALUE; - - static int nextBatchNumber(int prevBatchNumber) - { - return (prevBatchNumber + 1) & MAX_PHASE; - } - - private final Phaser phaser = new Phaser(1); - - void batchEmitted() - { - phaser.arrive(); - } - - void awaitBatchEmitted(int batchNumberToAwait, long timeout, TimeUnit unit) - throws InterruptedException, TimeoutException - { - batchNumberToAwait &= MAX_PHASE; - int currentBatch = phaser.getPhase(); - checkNotTerminated(currentBatch); - while (comparePhases(batchNumberToAwait, currentBatch) >= 0) { - currentBatch = phaser.awaitAdvanceInterruptibly(currentBatch, timeout, unit); - checkNotTerminated(currentBatch); - } - } - - private static int comparePhases(int phase1, int phase2) - { - int diff = (phase1 - phase2) & MAX_PHASE; - if (diff == 0) { - return 0; - } - return diff < MAX_PHASE / 2 ? 1 : -1; - } - - private void checkNotTerminated(int phase) - { - if (phase < 0) { - throw new IllegalStateException("Phaser[" + phaser + "] unexpectedly terminated."); - } - } -} diff --git a/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java b/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java index 3db529d764cc..c8ea0ab6cc68 100644 --- a/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java +++ b/java-util/src/main/java/io/druid/java/util/emitter/core/HttpPostEmitter.java @@ -25,6 +25,7 @@ import com.google.common.base.Predicate; import com.google.common.base.Throwables; import com.google.common.primitives.Ints; +import io.druid.concurrent.ConcurrentAwaitableCounter; import io.druid.java.util.common.ISE; import io.druid.java.util.common.RetryUtils; import io.druid.java.util.common.StringUtils; @@ -134,7 +135,7 @@ public class HttpPostEmitter implements Flushable, Closeable, Emitter */ private final AtomicInteger approximateLargeEventsToEmitCount = new AtomicInteger(); - private final EmittedBatchCounter emittedBatchCounter = new EmittedBatchCounter(); + private final ConcurrentAwaitableCounter emittedBatchCounter = new ConcurrentAwaitableCounter(); private final EmittingThread emittingThread; private final AtomicLong totalEmittedEvents = new AtomicLong(); private final AtomicInteger allocatedBuffers = new AtomicInteger(); @@ -178,7 +179,8 @@ public HttpPostEmitter(HttpEmitterConfig config, AsyncHttpClient client, ObjectM throw new ISE(e, "Bad URL: %s", config.getRecipientBaseUrl()); } emittingThread = new EmittingThread(config); - concurrentBatch.set(new Batch(this, acquireBuffer(), 0)); + long firstBatchNumber = 1; + concurrentBatch.set(new Batch(this, acquireBuffer(), firstBatchNumber)); // lastFillTimeMillis must not be 0, minHttpTimeoutMillis could be. lastFillTimeMillis = Math.max(config.minHttpTimeoutMillis, 1); } @@ -332,7 +334,7 @@ private void doOnSealExclusive(Batch batch, long elapsedTimeMillis) addBatchToEmitQueue(batch); wakeUpEmittingThread(); if (!isTerminated()) { - int nextBatchNumber = EmittedBatchCounter.nextBatchNumber(batch.batchNumber); + long nextBatchNumber = ConcurrentAwaitableCounter.nextCount(batch.batchNumber); byte[] newBuffer = acquireBuffer(); if (!concurrentBatch.compareAndSet(batch, new Batch(this, newBuffer, nextBatchNumber))) { buffersToReuse.add(newBuffer); @@ -346,7 +348,7 @@ private void doOnSealExclusive(Batch batch, long elapsedTimeMillis) private void tryRecoverCurrentBatch(Integer failedBatchNumber) { log.info("Trying to recover currentBatch"); - int nextBatchNumber = EmittedBatchCounter.nextBatchNumber(failedBatchNumber); + long nextBatchNumber = ConcurrentAwaitableCounter.nextCount(failedBatchNumber); byte[] newBuffer = acquireBuffer(); if (concurrentBatch.compareAndSet(failedBatchNumber, new Batch(this, newBuffer, nextBatchNumber))) { log.info("Successfully recovered currentBatch"); @@ -384,7 +386,7 @@ private void limitBuffersToEmitSize() private void batchFinalized() { // Notify HttpPostEmitter.flush(), that the batch is emitted, or failed, or dropped. - emittedBatchCounter.batchEmitted(); + emittedBatchCounter.increment(); } private Batch pollBatchFromEmitQueue() @@ -423,7 +425,7 @@ private void flush(Batch batch) throws IOException // This check doesn't always awaits for this exact batch to be emitted, because another batch could be dropped // from the queue ahead of this one, in limitBuffersToEmitSize(). But there is no better way currently to wait for // the exact batch, and it's not that important. - emittedBatchCounter.awaitBatchEmitted(batch.batchNumber, config.getFlushTimeOut(), TimeUnit.MILLISECONDS); + emittedBatchCounter.awaitCount(batch.batchNumber, config.getFlushTimeOut(), TimeUnit.MILLISECONDS); } catch (TimeoutException e) { String message = StringUtils.format("Timed out after [%d] millis during flushing", config.getFlushTimeOut()); @@ -924,7 +926,7 @@ public ConcurrentTimeCounter getFailedSendingTimeCounter() @VisibleForTesting void waitForEmission(int batchNumber) throws Exception { - emittedBatchCounter.awaitBatchEmitted(batchNumber, 10, TimeUnit.SECONDS); + emittedBatchCounter.awaitCount(batchNumber, 10, TimeUnit.SECONDS); } @VisibleForTesting diff --git a/java-util/src/test/java/io/druid/concurrent/ConcurrentAwaitableCounterTest.java b/java-util/src/test/java/io/druid/concurrent/ConcurrentAwaitableCounterTest.java new file mode 100644 index 000000000000..c13fa7b7d7c3 --- /dev/null +++ b/java-util/src/test/java/io/druid/concurrent/ConcurrentAwaitableCounterTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.concurrent; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class ConcurrentAwaitableCounterTest +{ + + @Test(timeout = 1000) + public void smokeTest() throws InterruptedException + { + ConcurrentAwaitableCounter counter = new ConcurrentAwaitableCounter(); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch finish = new CountDownLatch(7); + for (int i = 0; i < 2; i++) { + new Thread(() -> { + try { + start.await(); + for (int j = 0; j < 10_000; j++) { + counter.increment(); + } + finish.countDown(); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + + }).start(); + } + for (int awaitCount : new int[] {0, 1, 100, 10_000, 20_000}) { + new Thread(() -> { + try { + start.await(); + counter.awaitCount(awaitCount); + finish.countDown(); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + }).start(); + } + start.countDown(); + finish.await(); + } + + @Test + public void testAwaitFirstUpdate() throws InterruptedException + { + int[] value = new int[1]; + ConcurrentAwaitableCounter counter = new ConcurrentAwaitableCounter(); + Thread t = new Thread(() -> { + try { + Assert.assertTrue(counter.awaitFirstIncrement(10, TimeUnit.SECONDS)); + Assert.assertEquals(1, value[0]); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + + }); + t.start(); + Thread.sleep(2_000); + value[0] = 1; + counter.increment(); + t.join(); + } +} diff --git a/java-util/src/test/java/io/druid/java/util/emitter/core/EmitterTest.java b/java-util/src/test/java/io/druid/java/util/emitter/core/EmitterTest.java index ac537fb4b191..d94700be9d43 100644 --- a/java-util/src/test/java/io/druid/java/util/emitter/core/EmitterTest.java +++ b/java-util/src/test/java/io/druid/java/util/emitter/core/EmitterTest.java @@ -240,7 +240,7 @@ protected ListenableFuture go(Request request) throws JsonProcessingEx for (UnitEvent event : events) { emitter.emit(event); } - waitForEmission(emitter, 0); + waitForEmission(emitter, 1); closeNoFlush(emitter); Assert.assertTrue(httpClient.succeeded()); } @@ -282,7 +282,7 @@ protected ListenableFuture go(Request request) throws JsonProcessingEx for (UnitEvent event : events) { emitter.emit(event); } - waitForEmission(emitter, 0); + waitForEmission(emitter, 1); closeNoFlush(emitter); Assert.assertTrue(httpClient.succeeded()); } @@ -298,7 +298,7 @@ public void testSizeBasedEmission() throws Exception httpClient.setGoHandler(GoHandlers.passingHandler(okResponse()).times(1)); emitter.emit(new UnitEvent("test", 3)); - waitForEmission(emitter, 0); + waitForEmission(emitter, 1); httpClient.setGoHandler(GoHandlers.failingHandler()); emitter.emit(new UnitEvent("test", 4)); @@ -338,7 +338,7 @@ protected ListenableFuture go(Request request) timeWaited < timeBetweenEmissions * 2 ); - waitForEmission(emitter, 0); + waitForEmission(emitter, 1); final CountDownLatch thisLatch = new CountDownLatch(1); httpClient.setGoHandler( @@ -363,7 +363,7 @@ protected ListenableFuture go(Request request) timeWaited < timeBetweenEmissions * 2 ); - waitForEmission(emitter, 1); + waitForEmission(emitter, 2); closeNoFlush(emitter); Assert.assertTrue("httpClient.succeeded()", httpClient.succeeded()); } @@ -389,7 +389,7 @@ protected ListenableFuture go(Request request) ); emitter.emit(event1); emitter.flush(); - waitForEmission(emitter, 0); + waitForEmission(emitter, 1); Assert.assertTrue(httpClient.succeeded()); // Failed to emit the first event. @@ -408,7 +408,7 @@ protected ListenableFuture go(Request request) emitter.emit(event2); emitter.flush(); - waitForEmission(emitter, 1); + waitForEmission(emitter, 2); closeNoFlush(emitter); // Failed event is emitted inside emitter thread, there is no other way to wait for it other than joining the // emitterThread @@ -462,7 +462,7 @@ protected ListenableFuture go(Request request) throws JsonProcessingEx emitter.emit(event); } emitter.flush(); - waitForEmission(emitter, 0); + waitForEmission(emitter, 1); closeNoFlush(emitter); Assert.assertTrue(httpClient.succeeded()); } @@ -513,11 +513,11 @@ protected ListenableFuture go(Request request) throws JsonProcessingEx for (UnitEvent event : events) { emitter.emit(event); } - waitForEmission(emitter, 0); + waitForEmission(emitter, 1); Assert.assertEquals(2, emitter.getTotalEmittedEvents()); emitter.flush(); - waitForEmission(emitter, 1); + waitForEmission(emitter, 2); Assert.assertEquals(4, emitter.getTotalEmittedEvents()); closeNoFlush(emitter); Assert.assertTrue(httpClient.succeeded()); @@ -572,7 +572,7 @@ protected ListenableFuture go(Request request) throws IOException for (UnitEvent event : events) { emitter.emit(event); } - waitForEmission(emitter, 0); + waitForEmission(emitter, 1); closeNoFlush(emitter); Assert.assertTrue(httpClient.succeeded()); } From 012a5be5954ca9093c99c6333c392d7267b89b7e Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Mon, 16 Apr 2018 19:16:25 -0700 Subject: [PATCH 62/82] [Backport] Fix NPE in compactionTask (#5645) --- .../indexing/common/task/CompactionTask.java | 97 +++++++++++++------ .../common/task/CompactionTaskTest.java | 61 +++++++++++- .../java/io/druid/segment/QueryableIndex.java | 3 +- .../druid/segment/SimpleQueryableIndex.java | 39 +++++++- 4 files changed, 163 insertions(+), 37 deletions(-) diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/CompactionTask.java index 96f3889982b9..5e94285cd9f7 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/CompactionTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/CompactionTask.java @@ -51,6 +51,7 @@ import io.druid.java.util.common.ISE; import io.druid.java.util.common.JodaUtils; import io.druid.java.util.common.Pair; +import io.druid.java.util.common.RE; import io.druid.java.util.common.granularity.NoneGranularity; import io.druid.java.util.common.guava.Comparators; import io.druid.java.util.common.jackson.JacksonUtils; @@ -78,6 +79,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -189,26 +191,35 @@ public TaskStatus run(final TaskToolbox toolbox) throws Exception jsonMapper ); - indexTaskSpec = new IndexTask( - getId(), - getGroupId(), - getTaskResource(), - getDataSource(), - ingestionSpec, - getContext() - ); - } - - if (indexTaskSpec.getIngestionSchema() == null) { - log.info("Cannot find segments for interval"); + if (ingestionSpec != null) { + indexTaskSpec = new IndexTask( + getId(), + getGroupId(), + getTaskResource(), + getDataSource(), + ingestionSpec, + getContext() + ); + } } - final String json = jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(indexTaskSpec); - log.info("Generated compaction task details: " + json); + if (indexTaskSpec == null) { + log.warn("Failed to generate compaction spec"); + return TaskStatus.failure(getId()); + } else { + final String json = jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(indexTaskSpec); + log.info("Generated compaction task details: " + json); - return indexTaskSpec.run(toolbox); + return indexTaskSpec.run(toolbox); + } } + /** + * Generate {@link IndexIngestionSpec} from input segments. + * + * @return null if input segments don't exist. Otherwise, a generated ingestionSpec. + */ + @Nullable @VisibleForTesting static IndexIngestionSpec createIngestionSchema( TaskToolbox toolbox, @@ -281,12 +292,22 @@ private static DataSchema createDataSchema( throws IOException, SegmentLoadingException { // find metadata for interval - final List queryableIndices = loadSegments(timelineSegments, segmentFileMap, indexIO); + final List> queryableIndexAndSegments = loadSegments( + timelineSegments, + segmentFileMap, + indexIO + ); // find merged aggregators - final List aggregatorFactories = queryableIndices + for (Pair pair : queryableIndexAndSegments) { + final QueryableIndex index = pair.lhs; + if (index.getMetadata() == null) { + throw new RE("Index metadata doesn't exist for segment[%s]", pair.rhs.getIdentifier()); + } + } + final List aggregatorFactories = queryableIndexAndSegments .stream() - .map(index -> index.getMetadata().getAggregators()) + .map(pair -> pair.lhs.getMetadata().getAggregators()) // We have already done null check on index.getMetadata() .collect(Collectors.toList()); final AggregatorFactory[] mergedAggregators = AggregatorFactory.mergeAggregators(aggregatorFactories); @@ -296,7 +317,11 @@ private static DataSchema createDataSchema( // find granularity spec // set rollup only if rollup is set for all segments - final boolean rollup = queryableIndices.stream().allMatch(index -> index.getMetadata().isRollup()); + final boolean rollup = queryableIndexAndSegments.stream().allMatch(pair -> { + // We have already checked getMetadata() doesn't return null + final Boolean isRollup = pair.lhs.getMetadata().isRollup(); + return isRollup != null && isRollup; + }); final GranularitySpec granularitySpec = new ArbitraryGranularitySpec( new NoneGranularity(), rollup, @@ -305,7 +330,7 @@ private static DataSchema createDataSchema( // find unique dimensions final DimensionsSpec finalDimensionsSpec = dimensionsSpec == null ? - createDimensionsSpec(queryableIndices) : + createDimensionsSpec(queryableIndexAndSegments) : dimensionsSpec; final InputRowParser parser = new NoopInputRowParser(new TimeAndDimsParseSpec(null, finalDimensionsSpec)); @@ -319,7 +344,7 @@ private static DataSchema createDataSchema( ); } - private static DimensionsSpec createDimensionsSpec(List queryableIndices) + private static DimensionsSpec createDimensionsSpec(List> queryableIndices) { final BiMap uniqueDims = HashBiMap.create(); final Map dimensionSchemaMap = new HashMap<>(); @@ -329,9 +354,24 @@ private static DimensionsSpec createDimensionsSpec(List queryabl // Dimensions are extracted from the recent segments to olders because recent segments are likely to be queried more // frequently, and thus the performance should be optimized for recent ones rather than old ones. - // timelineSegments are sorted in order of interval + // timelineSegments are sorted in order of interval, but we do a sanity check here. + final Comparator intervalComparator = Comparators.intervalsByStartThenEnd(); + for (int i = 0; i < queryableIndices.size() - 1; i++) { + final Interval shouldBeSmaller = queryableIndices.get(i).lhs.getDataInterval(); + final Interval shouldBeLarger = queryableIndices.get(i + 1).lhs.getDataInterval(); + Preconditions.checkState( + intervalComparator.compare(shouldBeSmaller, shouldBeLarger) <= 0, + "QueryableIndexes are not sorted! Interval[%s] of segment[%s] is laster than interval[%s] of segment[%s]", + shouldBeSmaller, + queryableIndices.get(i).rhs.getIdentifier(), + shouldBeLarger, + queryableIndices.get(i + 1).rhs.getIdentifier() + ); + } + int index = 0; - for (QueryableIndex queryableIndex : Lists.reverse(queryableIndices)) { + for (Pair pair : Lists.reverse(queryableIndices)) { + final QueryableIndex queryableIndex = pair.lhs; final Map dimensionHandlerMap = queryableIndex.getDimensionHandlers(); for (String dimension : queryableIndex.getAvailableDimensions()) { @@ -376,23 +416,22 @@ private static DimensionsSpec createDimensionsSpec(List queryabl return new DimensionsSpec(dimensionSchemas, null, null); } - private static List loadSegments( + private static List> loadSegments( List> timelineSegments, Map segmentFileMap, IndexIO indexIO ) throws IOException { - final List segments = new ArrayList<>(); + final List> segments = new ArrayList<>(); for (TimelineObjectHolder timelineSegment : timelineSegments) { final PartitionHolder partitionHolder = timelineSegment.getObject(); for (PartitionChunk chunk : partitionHolder) { final DataSegment segment = chunk.getObject(); - segments.add( - indexIO.loadIndex( - Preconditions.checkNotNull(segmentFileMap.get(segment), "File for segment %s", segment.getIdentifier()) - ) + final QueryableIndex queryableIndex = indexIO.loadIndex( + Preconditions.checkNotNull(segmentFileMap.get(segment), "File for segment %s", segment.getIdentifier()) ); + segments.add(Pair.of(queryableIndex, segment)); } } diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java index 017158200806..2db8255fa9c7 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/CompactionTaskTest.java @@ -26,6 +26,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import io.druid.data.input.FirehoseFactory; import io.druid.data.input.impl.DimensionSchema; @@ -69,8 +70,8 @@ import io.druid.segment.column.Column; import io.druid.segment.column.ColumnBuilder; import io.druid.segment.column.ValueType; -import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.CompressionFactory.LongEncodingStrategy; +import io.druid.segment.data.CompressionStrategy; import io.druid.segment.data.ListIndexed; import io.druid.segment.data.RoaringBitmapSerdeFactory; import io.druid.segment.incremental.IncrementalIndex; @@ -84,6 +85,7 @@ import org.hamcrest.CoreMatchers; import org.joda.time.Interval; import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; @@ -125,10 +127,12 @@ public class CompactionTaskTest private static Map AGGREGATORS; private static List SEGMENTS; private static ObjectMapper objectMapper = setupInjectablesInObjectMapper(new DefaultObjectMapper()); - private static TaskToolbox toolbox; + private static Map segmentMap; + + private TaskToolbox toolbox; @BeforeClass - public static void setup() + public static void setupClass() { DIMENSIONS = new HashMap<>(); AGGREGATORS = new HashMap<>(); @@ -161,7 +165,7 @@ public static void setup() AGGREGATORS.put("agg_3", new FloatFirstAggregatorFactory("agg_3", "float_dim_3")); AGGREGATORS.put("agg_4", new DoubleLastAggregatorFactory("agg_4", "double_dim_4")); - final Map segmentMap = new HashMap<>(5); + segmentMap = new HashMap<>(5); for (int i = 0; i < 5; i++) { final Interval segmentInterval = Intervals.of(StringUtils.format("2017-0%d-01/2017-0%d-01", (i + 1), (i + 2))); segmentMap.put( @@ -180,7 +184,11 @@ public static void setup() ); } SEGMENTS = new ArrayList<>(segmentMap.keySet()); + } + @Before + public void setup() + { toolbox = new TestTaskToolbox( new TestTaskActionClient(new ArrayList<>(segmentMap.keySet())), new TestIndexIO(objectMapper, segmentMap), @@ -391,6 +399,24 @@ public void testCreateIngestionSchemaWithDifferentSegmentSet() throws IOExceptio ); } + @Test + public void testMissingMetadata() throws IOException, SegmentLoadingException + { + expectedException.expect(RuntimeException.class); + expectedException.expectMessage(CoreMatchers.startsWith("Index metadata doesn't exist for segment")); + + final TestIndexIO indexIO = (TestIndexIO) toolbox.getIndexIO(); + indexIO.removeMetadata(Iterables.getFirst(indexIO.getQueryableIndexMap().keySet(), null)); + final List segments = new ArrayList<>(SEGMENTS); + CompactionTask.createIngestionSchema( + toolbox, + new SegmentProvider(segments), + null, + TUNING_CONFIG, + objectMapper + ); + } + private static DimensionsSpec getExpectedDimensionsSpecForAutoGeneration() { return new DimensionsSpec( @@ -576,7 +602,7 @@ private static class TestIndexIO extends IndexIO } final Metadata metadata = new Metadata(); - metadata.setAggregators(aggregatorFactories.toArray(new AggregatorFactory[aggregatorFactories.size()])); + metadata.setAggregators(aggregatorFactories.toArray(new AggregatorFactory[0])); metadata.setRollup(false); queryableIndexMap.put( @@ -599,6 +625,31 @@ public QueryableIndex loadIndex(File file) throws IOException { return queryableIndexMap.get(file); } + + void removeMetadata(File file) + { + final SimpleQueryableIndex index = (SimpleQueryableIndex) queryableIndexMap.get(file); + if (index != null) { + queryableIndexMap.put( + file, + new SimpleQueryableIndex( + index.getDataInterval(), + index.getColumnNames(), + index.getAvailableDimensions(), + index.getBitmapFactoryForDimensions(), + index.getColumns(), + index.getFileMapper(), + null, + index.getDimensionHandlers() + ) + ); + } + } + + Map getQueryableIndexMap() + { + return queryableIndexMap; + } } private static Column createColumn(DimensionSchema dimensionSchema) diff --git a/processing/src/main/java/io/druid/segment/QueryableIndex.java b/processing/src/main/java/io/druid/segment/QueryableIndex.java index 0f6dd5182f43..d1b697b43414 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndex.java @@ -23,6 +23,7 @@ import io.druid.segment.data.Indexed; import org.joda.time.Interval; +import javax.annotation.Nullable; import java.io.Closeable; import java.io.IOException; import java.util.Map; @@ -40,7 +41,7 @@ public interface QueryableIndex extends ColumnSelector, Closeable int getNumRows(); Indexed getAvailableDimensions(); BitmapFactory getBitmapFactoryForDimensions(); - Metadata getMetadata(); + @Nullable Metadata getMetadata(); Map getDimensionHandlers(); /** diff --git a/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java b/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java index 17d81d2119ba..754b93d98387 100644 --- a/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java @@ -19,6 +19,7 @@ package io.druid.segment; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import io.druid.collections.bitmap.BitmapFactory; @@ -52,11 +53,11 @@ public SimpleQueryableIndex( BitmapFactory bitmapFactory, Map columns, SmooshedFileMapper fileMapper, - Metadata metadata + @Nullable Metadata metadata ) { Preconditions.checkNotNull(columns.get(Column.TIME_COLUMN_NAME)); - this.dataInterval = dataInterval; + this.dataInterval = Preconditions.checkNotNull(dataInterval, "dataInterval"); this.columnNames = columnNames; this.availableDimensions = dimNames; this.bitmapFactory = bitmapFactory; @@ -67,6 +68,28 @@ public SimpleQueryableIndex( initDimensionHandlers(); } + @VisibleForTesting + public SimpleQueryableIndex( + Interval interval, + Indexed columnNames, + Indexed availableDimensions, + BitmapFactory bitmapFactory, + Map columns, + SmooshedFileMapper fileMapper, + @Nullable Metadata metadata, + Map dimensionHandlers + ) + { + this.dataInterval = interval; + this.columnNames = columnNames; + this.availableDimensions = availableDimensions; + this.bitmapFactory = bitmapFactory; + this.columns = columns; + this.fileMapper = fileMapper; + this.metadata = metadata; + this.dimensionHandlers = dimensionHandlers; + } + @Override public Interval getDataInterval() { @@ -104,6 +127,18 @@ public Column getColumn(String columnName) return columns.get(columnName); } + @VisibleForTesting + public Map getColumns() + { + return columns; + } + + @VisibleForTesting + public SmooshedFileMapper getFileMapper() + { + return fileMapper; + } + @Override public void close() throws IOException { From 5740ce9e09f52a1f02330f95af46d5d4ac3d9c91 Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Tue, 17 Apr 2018 13:25:23 -0700 Subject: [PATCH 63/82] [Backport] Fix HTTP OPTIONS request auth handling (#5638) (#5654) * Fix HTTP OPTIONS request auth handling (#5638) * Fix HTTP OPTIONS request auth handling * PR comment * More PR comments * Fix * PR comment * Compile fixes * Compile --- docs/content/configuration/auth.md | 1 + .../ITBasicAuthConfigurationTest.java | 12 +++ .../security/AllowOptionsResourceFilter.java | 84 +++++++++++++++++++ .../io/druid/server/security/AuthConfig.java | 36 ++++---- .../server/security/AuthenticationUtils.java | 10 +++ .../io/druid/server/QueryResourceTest.java | 12 +-- .../server/http/DatasourcesResourceTest.java | 2 +- .../security/ResourceFilterTestHelper.java | 2 +- .../main/java/io/druid/cli/CliOverlord.java | 4 + .../CoordinatorJettyServerInitializer.java | 3 +- .../MiddleManagerJettyServerInitializer.java | 1 + .../cli/QueryJettyServerInitializer.java | 4 + .../cli/RouterJettyServerInitializer.java | 2 + 13 files changed, 148 insertions(+), 25 deletions(-) create mode 100644 server/src/main/java/io/druid/server/security/AllowOptionsResourceFilter.java diff --git a/docs/content/configuration/auth.md b/docs/content/configuration/auth.md index 358a54431b80..2fecfc1a86c3 100644 --- a/docs/content/configuration/auth.md +++ b/docs/content/configuration/auth.md @@ -9,6 +9,7 @@ layout: doc_page |`druid.auth.authenticationChain`|JSON List of Strings|List of Authenticator type names|["allowAll"]|no| |`druid.escalator.type`|String|Type of the Escalator that should be used for internal Druid communications. This Escalator must use an authentication scheme that is supported by an Authenticator in `druid.auth.authenticationChain`.|"noop"|no| |`druid.auth.authorizers`|JSON List of Strings|List of Authorizer type names |["allowAll"]|no| +|`druid.auth.allowUnauthenticatedHttpOptions`|Boolean|If true, skip authentication checks for HTTP OPTIONS requests. This is needed for certain use cases, such as supporting CORS pre-flight requests. Note that disabling authentication checks for OPTIONS requests will allow unauthenticated users to determine what Druid endpoints are valid (by checking if the OPTIONS request returns a 200 instead of 404), so enabling this option may reveal information about server configuration, including information about what extensions are loaded (if those extensions add endpoints).|false|no| ## Enabling Authentication/Authorization diff --git a/integration-tests/src/test/java/io/druid/tests/security/ITBasicAuthConfigurationTest.java b/integration-tests/src/test/java/io/druid/tests/security/ITBasicAuthConfigurationTest.java index dd9344846f11..572b565fbb9b 100644 --- a/integration-tests/src/test/java/io/druid/tests/security/ITBasicAuthConfigurationTest.java +++ b/integration-tests/src/test/java/io/druid/tests/security/ITBasicAuthConfigurationTest.java @@ -219,6 +219,18 @@ public void testAuthConfiguration() throws Exception LOG.info("Testing Avatica query on router with incorrect credentials."); testAvaticaAuthFailure(routerUrl); + + LOG.info("Checking OPTIONS requests on services..."); + testOptionsRequests(adminClient); + } + + private void testOptionsRequests(HttpClient httpClient) + { + makeRequest(httpClient, HttpMethod.OPTIONS, config.getCoordinatorUrl() + "/status", null); + makeRequest(httpClient, HttpMethod.OPTIONS, config.getIndexerUrl() + "/status", null); + makeRequest(httpClient, HttpMethod.OPTIONS, config.getBrokerUrl() + "/status", null); + makeRequest(httpClient, HttpMethod.OPTIONS, config.getHistoricalUrl() + "/status", null); + makeRequest(httpClient, HttpMethod.OPTIONS, config.getRouterUrl() + "/status", null); } private void testAvaticaQuery(String url) diff --git a/server/src/main/java/io/druid/server/security/AllowOptionsResourceFilter.java b/server/src/main/java/io/druid/server/security/AllowOptionsResourceFilter.java new file mode 100644 index 000000000000..9ec87415593a --- /dev/null +++ b/server/src/main/java/io/druid/server/security/AllowOptionsResourceFilter.java @@ -0,0 +1,84 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.server.security; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.HttpMethod; +import java.io.IOException; + +public class AllowOptionsResourceFilter implements Filter +{ + private final boolean allowUnauthenticatedHttpOptions; + + public AllowOptionsResourceFilter( + boolean allowUnauthenticatedHttpOptions + ) + { + this.allowUnauthenticatedHttpOptions = allowUnauthenticatedHttpOptions; + } + + @Override + public void init(FilterConfig filterConfig) throws ServletException + { + + } + + @Override + public void doFilter( + ServletRequest request, ServletResponse response, FilterChain chain + ) throws IOException, ServletException + { + HttpServletRequest httpReq = (HttpServletRequest) request; + + // Druid itself doesn't explictly handle OPTIONS requests, no resource handler will authorize such requests. + // so this filter catches all OPTIONS requests and authorizes them. + if (HttpMethod.OPTIONS.equals(httpReq.getMethod())) { + if (httpReq.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT) == null) { + // If the request already had credentials and authenticated successfully, keep the authenticated identity. + // Otherwise, allow the unauthenticated request. + if (allowUnauthenticatedHttpOptions) { + httpReq.setAttribute( + AuthConfig.DRUID_AUTHENTICATION_RESULT, + new AuthenticationResult(AuthConfig.ALLOW_ALL_NAME, AuthConfig.ALLOW_ALL_NAME, null) + ); + } else { + ((HttpServletResponse) response).sendError(HttpServletResponse.SC_UNAUTHORIZED); + } + } + + httpReq.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); + } + + chain.doFilter(request, response); + } + + @Override + public void destroy() + { + + } +} diff --git a/server/src/main/java/io/druid/server/security/AuthConfig.java b/server/src/main/java/io/druid/server/security/AuthConfig.java index af4768a4d5e0..9fe71afef9bc 100644 --- a/server/src/main/java/io/druid/server/security/AuthConfig.java +++ b/server/src/main/java/io/druid/server/security/AuthConfig.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; +import java.util.Objects; public class AuthConfig { @@ -40,17 +41,19 @@ public class AuthConfig public AuthConfig() { - this(null, null); + this(null, null, false); } @JsonCreator public AuthConfig( @JsonProperty("authenticatorChain") List authenticationChain, - @JsonProperty("authorizers") List authorizers + @JsonProperty("authorizers") List authorizers, + @JsonProperty("allowUnauthenticatedHttpOptions") boolean allowUnauthenticatedHttpOptions ) { this.authenticatorChain = authenticationChain; this.authorizers = authorizers; + this.allowUnauthenticatedHttpOptions = allowUnauthenticatedHttpOptions; } @JsonProperty @@ -59,6 +62,9 @@ public AuthConfig( @JsonProperty private List authorizers; + @JsonProperty + private final boolean allowUnauthenticatedHttpOptions; + public List getAuthenticatorChain() { return authenticatorChain; @@ -69,12 +75,18 @@ public List getAuthorizers() return authorizers; } + public boolean isAllowUnauthenticatedHttpOptions() + { + return allowUnauthenticatedHttpOptions; + } + @Override public String toString() { return "AuthConfig{" + - "authenticatorChain='" + authenticatorChain + '\'' + - ", authorizers='" + authorizers + '\'' + + "authenticatorChain=" + authenticatorChain + + ", authorizers=" + authorizers + + ", allowUnauthenticatedHttpOptions=" + allowUnauthenticatedHttpOptions + '}'; } @@ -87,23 +99,15 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - AuthConfig that = (AuthConfig) o; - - if (getAuthenticatorChain() != null - ? !getAuthenticatorChain().equals(that.getAuthenticatorChain()) - : that.getAuthenticatorChain() != null) { - return false; - } - return getAuthorizers() != null ? getAuthorizers().equals(that.getAuthorizers()) : that.getAuthorizers() == null; - + return isAllowUnauthenticatedHttpOptions() == that.isAllowUnauthenticatedHttpOptions() && + Objects.equals(getAuthenticatorChain(), that.getAuthenticatorChain()) && + Objects.equals(getAuthorizers(), that.getAuthorizers()); } @Override public int hashCode() { - int result = getAuthenticatorChain() != null ? getAuthenticatorChain().hashCode() : 0; - result = 31 * result + (getAuthorizers() != null ? getAuthorizers().hashCode() : 0); - return result; + return Objects.hash(getAuthenticatorChain(), getAuthorizers(), isAllowUnauthenticatedHttpOptions()); } } diff --git a/server/src/main/java/io/druid/server/security/AuthenticationUtils.java b/server/src/main/java/io/druid/server/security/AuthenticationUtils.java index cabaa828274c..bd6b2be7680f 100644 --- a/server/src/main/java/io/druid/server/security/AuthenticationUtils.java +++ b/server/src/main/java/io/druid/server/security/AuthenticationUtils.java @@ -27,6 +27,16 @@ public class AuthenticationUtils { + public static void addAllowOptionsFilter(ServletContextHandler root, boolean allowUnauthenticatedHttpOptions) + { + FilterHolder holder = new FilterHolder(new AllowOptionsResourceFilter(allowUnauthenticatedHttpOptions)); + root.addFilter( + holder, + "/*", + null + ); + } + public static void addAuthenticationFilterChain( ServletContextHandler root, List authenticators diff --git a/server/src/test/java/io/druid/server/QueryResourceTest.java b/server/src/test/java/io/druid/server/QueryResourceTest.java index b099f002d267..945aa7ec39b7 100644 --- a/server/src/test/java/io/druid/server/QueryResourceTest.java +++ b/server/src/test/java/io/druid/server/QueryResourceTest.java @@ -247,13 +247,13 @@ public Access authorize(AuthenticationResult authenticationResult, Resource reso new DefaultGenericQueryMetricsFactory(jsonMapper), new NoopServiceEmitter(), testRequestLogger, - new AuthConfig(null, null), + new AuthConfig(), authMapper ), jsonMapper, jsonMapper, queryManager, - new AuthConfig(null, null), + new AuthConfig(), authMapper, new DefaultGenericQueryMetricsFactory(jsonMapper) ); @@ -354,13 +354,13 @@ public Access authorize(AuthenticationResult authenticationResult, Resource reso new DefaultGenericQueryMetricsFactory(jsonMapper), new NoopServiceEmitter(), testRequestLogger, - new AuthConfig(null, null), + new AuthConfig(), authMapper ), jsonMapper, jsonMapper, queryManager, - new AuthConfig(null, null), + new AuthConfig(), authMapper, new DefaultGenericQueryMetricsFactory(jsonMapper) ); @@ -475,13 +475,13 @@ public Access authorize(AuthenticationResult authenticationResult, Resource reso new DefaultGenericQueryMetricsFactory(jsonMapper), new NoopServiceEmitter(), testRequestLogger, - new AuthConfig(null, null), + new AuthConfig(), authMapper ), jsonMapper, jsonMapper, queryManager, - new AuthConfig(null, null), + new AuthConfig(), authMapper, new DefaultGenericQueryMetricsFactory(jsonMapper) ); diff --git a/server/src/test/java/io/druid/server/http/DatasourcesResourceTest.java b/server/src/test/java/io/druid/server/http/DatasourcesResourceTest.java index 96e0b583f2e0..0e5e09939f8e 100644 --- a/server/src/test/java/io/druid/server/http/DatasourcesResourceTest.java +++ b/server/src/test/java/io/druid/server/http/DatasourcesResourceTest.java @@ -236,7 +236,7 @@ public Access authorize(AuthenticationResult authenticationResult1, Resource res inventoryView, null, null, - new AuthConfig(null, null), + new AuthConfig(), authMapper ); Response response = datasourcesResource.getQueryableDataSources("full", null, request); diff --git a/server/src/test/java/io/druid/server/http/security/ResourceFilterTestHelper.java b/server/src/test/java/io/druid/server/http/security/ResourceFilterTestHelper.java index ed39de6452ef..4e66f72d5664 100644 --- a/server/src/test/java/io/druid/server/http/security/ResourceFilterTestHelper.java +++ b/server/src/test/java/io/druid/server/http/security/ResourceFilterTestHelper.java @@ -182,7 +182,7 @@ public void configure(Binder binder) for (Key key : mockableKeys) { binder.bind((Key) key).toInstance(EasyMock.createNiceMock(key.getTypeLiteral().getRawType())); } - binder.bind(AuthConfig.class).toInstance(new AuthConfig(null, null)); + binder.bind(AuthConfig.class).toInstance(new AuthConfig()); } } ); diff --git a/services/src/main/java/io/druid/cli/CliOverlord.java b/services/src/main/java/io/druid/cli/CliOverlord.java index f070c8b5cdbb..d04bc7b175da 100644 --- a/services/src/main/java/io/druid/cli/CliOverlord.java +++ b/services/src/main/java/io/druid/cli/CliOverlord.java @@ -89,6 +89,7 @@ import io.druid.server.http.RedirectInfo; import io.druid.server.initialization.jetty.JettyServerInitUtils; import io.druid.server.initialization.jetty.JettyServerInitializer; +import io.druid.server.security.AuthConfig; import io.druid.server.security.AuthenticationUtils; import io.druid.server.security.Authenticator; import io.druid.server.security.AuthenticatorMapper; @@ -318,6 +319,7 @@ public void initialize(Server server, Injector injector) final ObjectMapper jsonMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class)); final AuthenticatorMapper authenticatorMapper = injector.getInstance(AuthenticatorMapper.class); + final AuthConfig authConfig = injector.getInstance(AuthConfig.class); List authenticators = null; AuthenticationUtils.addSecuritySanityCheckFilter(root, jsonMapper); @@ -328,6 +330,8 @@ public void initialize(Server server, Injector injector) authenticators = authenticatorMapper.getAuthenticatorChain(); AuthenticationUtils.addAuthenticationFilterChain(root, authenticators); + AuthenticationUtils.addAllowOptionsFilter(root, authConfig.isAllowUnauthenticatedHttpOptions()); + JettyServerInitUtils.addExtensionFilters(root, injector); diff --git a/services/src/main/java/io/druid/cli/CoordinatorJettyServerInitializer.java b/services/src/main/java/io/druid/cli/CoordinatorJettyServerInitializer.java index 5aefbd4182c8..572a06296f48 100644 --- a/services/src/main/java/io/druid/cli/CoordinatorJettyServerInitializer.java +++ b/services/src/main/java/io/druid/cli/CoordinatorJettyServerInitializer.java @@ -125,8 +125,9 @@ public void initialize(Server server, Injector injector) authenticators = authenticatorMapper.getAuthenticatorChain(); AuthenticationUtils.addAuthenticationFilterChain(root, authenticators); - JettyServerInitUtils.addExtensionFilters(root, injector); + AuthenticationUtils.addAllowOptionsFilter(root, authConfig.isAllowUnauthenticatedHttpOptions()); + JettyServerInitUtils.addExtensionFilters(root, injector); // Check that requests were authorized before sending responses AuthenticationUtils.addPreResponseAuthorizationCheckFilter( diff --git a/services/src/main/java/io/druid/cli/MiddleManagerJettyServerInitializer.java b/services/src/main/java/io/druid/cli/MiddleManagerJettyServerInitializer.java index 2eaa4d154fe1..bc4d3a54ed92 100644 --- a/services/src/main/java/io/druid/cli/MiddleManagerJettyServerInitializer.java +++ b/services/src/main/java/io/druid/cli/MiddleManagerJettyServerInitializer.java @@ -71,6 +71,7 @@ public void initialize(Server server, Injector injector) authenticators = authenticatorMapper.getAuthenticatorChain(); AuthenticationUtils.addAuthenticationFilterChain(root, authenticators); + AuthenticationUtils.addAllowOptionsFilter(root, authConfig.isAllowUnauthenticatedHttpOptions()); JettyServerInitUtils.addExtensionFilters(root, injector); diff --git a/services/src/main/java/io/druid/cli/QueryJettyServerInitializer.java b/services/src/main/java/io/druid/cli/QueryJettyServerInitializer.java index 0bc34f17fee9..8fa30b1c36cb 100644 --- a/services/src/main/java/io/druid/cli/QueryJettyServerInitializer.java +++ b/services/src/main/java/io/druid/cli/QueryJettyServerInitializer.java @@ -33,6 +33,7 @@ import io.druid.server.initialization.jetty.JettyServerInitUtils; import io.druid.server.initialization.jetty.JettyServerInitializer; import io.druid.server.initialization.jetty.LimitRequestsFilter; +import io.druid.server.security.AuthConfig; import io.druid.server.security.AuthenticationUtils; import io.druid.server.security.Authenticator; import io.druid.server.security.AuthenticatorMapper; @@ -89,6 +90,7 @@ public void initialize(Server server, Injector injector) final ObjectMapper jsonMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class)); final AuthenticatorMapper authenticatorMapper = injector.getInstance(AuthenticatorMapper.class); + final AuthConfig authConfig = injector.getInstance(AuthConfig.class); List authenticators = null; AuthenticationUtils.addSecuritySanityCheckFilter(root, jsonMapper); @@ -99,6 +101,8 @@ public void initialize(Server server, Injector injector) authenticators = authenticatorMapper.getAuthenticatorChain(); AuthenticationUtils.addAuthenticationFilterChain(root, authenticators); + AuthenticationUtils.addAllowOptionsFilter(root, authConfig.isAllowUnauthenticatedHttpOptions()); + JettyServerInitUtils.addExtensionFilters(root, injector); // Check that requests were authorized before sending responses diff --git a/services/src/main/java/io/druid/cli/RouterJettyServerInitializer.java b/services/src/main/java/io/druid/cli/RouterJettyServerInitializer.java index abe20a2d66e2..930d1191e828 100644 --- a/services/src/main/java/io/druid/cli/RouterJettyServerInitializer.java +++ b/services/src/main/java/io/druid/cli/RouterJettyServerInitializer.java @@ -105,6 +105,8 @@ public void initialize(Server server, Injector injector) authenticators = authenticatorMapper.getAuthenticatorChain(); AuthenticationUtils.addAuthenticationFilterChain(root, authenticators); + AuthenticationUtils.addAllowOptionsFilter(root, authConfig.isAllowUnauthenticatedHttpOptions()); + JettyServerInitUtils.addExtensionFilters(root, injector); // Check that requests were authorized before sending responses From b551c9c6c0645ce18870558a3da4c69e5d1972ed Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Mon, 23 Apr 2018 17:59:32 -0700 Subject: [PATCH 64/82] [Backport] Fix loadstatus?full double counting expected segments (#5680) --- .../server/coordinator/DruidCoordinator.java | 1 + .../coordinator/DruidCoordinatorTest.java | 166 ++++++++++++++++-- 2 files changed, 150 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java index 29f6ee12190b..428fc5588cef 100644 --- a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java +++ b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java @@ -254,6 +254,7 @@ public Map getLoadManagementPeons() .computeIfAbsent(tier, ignored -> new Object2LongOpenHashMap<>()) .addTo(segment.getDataSource(), Math.max(ruleReplicants - currentReplicants, 0)); }); + break; // only the first matching rule applies } } diff --git a/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorTest.java b/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorTest.java index d2f871c4e779..bbe5796d96ef 100644 --- a/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorTest.java +++ b/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorTest.java @@ -37,16 +37,18 @@ import io.druid.java.util.common.Intervals; import io.druid.java.util.common.concurrent.Execs; import io.druid.java.util.common.concurrent.ScheduledExecutorFactory; +import io.druid.java.util.emitter.core.Event; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.metadata.MetadataRuleManager; import io.druid.metadata.MetadataSegmentManager; import io.druid.server.DruidNode; import io.druid.server.coordination.DruidServerMetadata; import io.druid.server.coordination.ServerType; import io.druid.server.coordinator.rules.ForeverLoadRule; +import io.druid.server.coordinator.rules.IntervalLoadRule; import io.druid.server.coordinator.rules.Rule; import io.druid.server.initialization.ZkPathsConfig; import io.druid.server.lookup.cache.LookupCoordinatorManager; -import io.druid.server.metrics.NoopServiceEmitter; import io.druid.timeline.DataSegment; import it.unimi.dsi.fastutil.objects.Object2LongMap; import org.apache.curator.framework.CuratorFramework; @@ -96,6 +98,7 @@ public class DruidCoordinatorTest extends CuratorTestBase private ObjectMapper objectMapper; private JacksonConfigManager configManager; private DruidNode druidNode; + private LatchableServiceEmitter serviceEmitter = new LatchableServiceEmitter(); private static final String LOADPATH = "/druid/loadqueue/localhost:1234"; private static final long COORDINATOR_START_DELAY = 1; private static final long COORDINATOR_PERIOD = 100; @@ -178,7 +181,7 @@ public String getBase() serverInventoryView, metadataRuleManager, curator, - new NoopServiceEmitter(), + serviceEmitter, scheduledExecutorFactory, null, null, @@ -378,27 +381,17 @@ public void childEvent( assignSegmentLatch.await(); + final CountDownLatch coordinatorRunLatch = new CountDownLatch(2); + serviceEmitter.latch = coordinatorRunLatch; + coordinatorRunLatch.await(); + Assert.assertEquals(ImmutableMap.of(dataSource, 100.0), coordinator.getLoadStatus()); curator.delete().guaranteed().forPath(ZKPaths.makePath(LOADPATH, dataSegment.getIdentifier())); - // Wait for coordinator thread to run so that replication status is updated - while (coordinator.getSegmentAvailability().getLong(dataSource) != 0) { - Thread.sleep(50); - } + Map segmentAvailability = coordinator.getSegmentAvailability(); Assert.assertEquals(1, segmentAvailability.size()); Assert.assertEquals(0L, segmentAvailability.get(dataSource)); - while (coordinator.hasLoadPending(dataSource)) { - Thread.sleep(50); - } - - // wait historical data to be updated - long startMillis = System.currentTimeMillis(); - long coordinatorRunPeriodMillis = druidCoordinatorConfig.getCoordinatorPeriod().getMillis(); - while (System.currentTimeMillis() - startMillis < coordinatorRunPeriodMillis) { - Thread.sleep(100); - } - Map> replicationStatus = coordinator.getReplicationStatus(); Assert.assertNotNull(replicationStatus); Assert.assertEquals(1, replicationStatus.entrySet().size()); @@ -421,6 +414,127 @@ public void childEvent( EasyMock.verify(metadataRuleManager); } + @Test(timeout = 60_000L) + public void testCoordinatorTieredRun() throws Exception + { + final String dataSource = "dataSource", hotTierName = "hot", coldTierName = "cold"; + final Rule hotTier = new IntervalLoadRule(Intervals.of("2018-01-01/P1M"), ImmutableMap.of(hotTierName, 1)); + final Rule coldTier = new ForeverLoadRule(ImmutableMap.of(coldTierName, 1)); + final String loadPathCold = "/druid/loadqueue/cold:1234"; + final DruidServer hotServer = new DruidServer("hot", "hot", null, 5L, ServerType.HISTORICAL, hotTierName, 0); + final DruidServer coldServer = new DruidServer("cold", "cold", null, 5L, ServerType.HISTORICAL, coldTierName, 0); + + final Map dataSegments = ImmutableMap.of( + "2018-01-02T00:00:00.000Z_2018-01-03T00:00:00.000Z", + new DataSegment(dataSource, Intervals.of("2018-01-02/P1D"), "v1", null, null, null, null, 0x9, 0), + "2018-01-03T00:00:00.000Z_2018-01-04T00:00:00.000Z", + new DataSegment(dataSource, Intervals.of("2018-01-03/P1D"), "v1", null, null, null, null, 0x9, 0), + "2017-01-01T00:00:00.000Z_2017-01-02T00:00:00.000Z", + new DataSegment(dataSource, Intervals.of("2017-01-01/P1D"), "v1", null, null, null, null, 0x9, 0) + ); + + final LoadQueuePeon loadQueuePeonCold = new CuratorLoadQueuePeon( + curator, + loadPathCold, + objectMapper, + Execs.scheduledSingleThreaded("coordinator_test_load_queue_peon_cold_scheduled-%d"), + Execs.singleThreaded("coordinator_test_load_queue_peon_cold-%d"), + druidCoordinatorConfig + ); + final PathChildrenCache pathChildrenCacheCold = new PathChildrenCache( + curator, loadPathCold, true, true, Execs.singleThreaded("coordinator_test_path_children_cache_cold-%d") + ); + loadManagementPeons.putAll(ImmutableMap.of("hot", loadQueuePeon, "cold", loadQueuePeonCold)); + + loadQueuePeonCold.start(); + pathChildrenCache.start(); + pathChildrenCacheCold.start(); + + DruidDataSource[] druidDataSources = {new DruidDataSource(dataSource, Collections.emptyMap())}; + dataSegments.values().forEach(druidDataSources[0]::addSegment); + + EasyMock.expect(metadataRuleManager.getRulesWithDefault(EasyMock.anyString())) + .andReturn(ImmutableList.of(hotTier, coldTier)).atLeastOnce(); + EasyMock.expect(databaseSegmentManager.isStarted()).andReturn(true).anyTimes(); + EasyMock.expect(databaseSegmentManager.getInventory()).andReturn( + ImmutableList.of(druidDataSources[0].toImmutableDruidDataSource()) + ).atLeastOnce(); + EasyMock.expect(serverInventoryView.getInventory()) + .andReturn(ImmutableList.of(hotServer, coldServer)) + .atLeastOnce(); + EasyMock.expect(serverInventoryView.isStarted()).andReturn(true).anyTimes(); + + EasyMock.replay(metadataRuleManager, databaseSegmentManager, serverInventoryView); + + coordinator.start(); + leaderAnnouncerLatch.await(); // Wait for this coordinator to become leader + + final CountDownLatch assignSegmentLatchHot = new CountDownLatch(2); + pathChildrenCache.getListenable().addListener( + (client, event) -> { + if (event.getType().equals(PathChildrenCacheEvent.Type.CHILD_ADDED)) { + DataSegment segment = dataSegments + .entrySet() + .stream() + .filter(x -> event.getData().getPath().contains(x.getKey())) + .map(Map.Entry::getValue) + .findFirst() + .orElse(null); + + if (segment != null) { + hotServer.addDataSegment(segment); + curator.delete().guaranteed().forPath(event.getData().getPath()); + } + + assignSegmentLatchHot.countDown(); + } + } + ); + + final CountDownLatch assignSegmentLatchCold = new CountDownLatch(1); + pathChildrenCacheCold.getListenable().addListener( + (client, event) -> { + if (event.getType().equals(PathChildrenCacheEvent.Type.CHILD_ADDED)) { + DataSegment segment = dataSegments + .entrySet() + .stream() + .filter(x -> event.getData().getPath().contains(x.getKey())) + .map(Map.Entry::getValue) + .findFirst() + .orElse(null); + + if (segment != null) { + coldServer.addDataSegment(segment); + curator.delete().guaranteed().forPath(event.getData().getPath()); + } + + assignSegmentLatchCold.countDown(); + } + } + ); + + assignSegmentLatchHot.await(); + assignSegmentLatchCold.await(); + + final CountDownLatch coordinatorRunLatch = new CountDownLatch(2); + serviceEmitter.latch = coordinatorRunLatch; + coordinatorRunLatch.await(); + + Assert.assertEquals(ImmutableMap.of(dataSource, 100.0), coordinator.getLoadStatus()); + + Map> replicationStatus = coordinator.getReplicationStatus(); + Assert.assertEquals(2, replicationStatus.entrySet().size()); + Assert.assertEquals(0L, replicationStatus.get(hotTierName).getLong(dataSource)); + Assert.assertEquals(0L, replicationStatus.get(coldTierName).getLong(dataSource)); + + coordinator.stop(); + leaderUnannouncerLatch.await(); + + EasyMock.verify(serverInventoryView); + EasyMock.verify(databaseSegmentManager); + EasyMock.verify(metadataRuleManager); + } + @Test public void testOrderedAvailableDataSegments() { @@ -500,4 +614,22 @@ public void unregisterListener() listener.stopBeingLeader(); } } + + private static class LatchableServiceEmitter extends ServiceEmitter + { + private CountDownLatch latch; + + private LatchableServiceEmitter() + { + super("", "", null); + } + + @Override + public void emit(Event event) + { + if (latch != null && "segment/count".equals(event.toMap().get("metric"))) { + latch.countDown(); + } + } + } } From 7489fc97ef5df94b006b49d1df446981076e7e56 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 24 Apr 2018 13:44:02 -0700 Subject: [PATCH 65/82] [Backport] Add missing doc for automatic pendingSegments (#5682) [Backport] Add missing doc for automatic pendingSegments cleanup --- docs/content/configuration/coordinator.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/content/configuration/coordinator.md b/docs/content/configuration/coordinator.md index 2dfd97098df6..867a3043ac2d 100644 --- a/docs/content/configuration/coordinator.md +++ b/docs/content/configuration/coordinator.md @@ -29,6 +29,7 @@ The coordinator node uses several of the global configs in [Configuration](../co |`druid.coordinator.merge.on`|Boolean flag for whether or not the coordinator should try and merge small segments into a more optimal segment size.|false| |`druid.coordinator.conversion.on`|Boolean flag for converting old segment indexing versions to the latest segment indexing version.|false| |`druid.coordinator.load.timeout`|The timeout duration for when the coordinator assigns a segment to a historical node.|PT15M| +|`druid.coordinator.kill.pendingSegments.on`|Boolean flag for whether or not the coordinator clean up old entries in the `pendingSegments` table of metadata store. If set to true, coordinator will check the created time of most recently complete task. If it doesn't exist, it finds the created time of the earlist running/pending/waiting tasks. Once the created time is found, then for all dataSources not in the `killPendingSegmentsSkipList` (see [Dynamic configuration](#dynamic-configuration)), coordinator will ask the overlord to clean up the entries 1 day or more older than the found created time in the `pendingSegments` table. This will be done periodically based on `druid.coordinator.period` specified.|false| |`druid.coordinator.kill.on`|Boolean flag for whether or not the coordinator should submit kill task for unused segments, that is, hard delete them from metadata store and deep storage. If set to true, then for all whitelisted dataSources (or optionally all), coordinator will submit tasks periodically based on `period` specified. These kill tasks will delete all segments except for the last `durationToRetain` period. Whitelist or All can be set via dynamic configuration `killAllDataSources` and `killDataSourceWhitelist` described later.|false| |`druid.coordinator.kill.period`|How often to send kill tasks to the indexing service. Value must be greater than `druid.coordinator.period.indexingPeriod`. Only applies if kill is turned on.|P1D (1 Day)| |`druid.coordinator.kill.durationToRetain`| Do not kill segments in last `durationToRetain`, must be greater or equal to 0. Only applies and MUST be specified if kill is turned on. Note that default value is invalid.|PT-1S (-1 seconds)| @@ -103,8 +104,9 @@ Issuing a GET request at the same URL will return the spec that is currently in |`replicantLifetime`|The maximum number of coordinator runs for a segment to be replicated before we start alerting.|15| |`replicationThrottleLimit`|The maximum number of segments that can be replicated at one time.|10| |`emitBalancingStats`|Boolean flag for whether or not we should emit balancing stats. This is an expensive operation.|false| -|`killDataSourceWhitelist`|List of dataSources for which kill tasks are sent if property `druid.coordinator.kill.on` is true.|none| +|`killDataSourceWhitelist`|List of dataSources for which kill tasks are sent if property `druid.coordinator.kill.on` is true. This can be a list of comma-separated dataSources or a JSON array.|none| |`killAllDataSources`|Send kill tasks for ALL dataSources if property `druid.coordinator.kill.on` is true. If this is set to true then `killDataSourceWhitelist` must not be specified or be empty list.|false| +|`killPendingSegmentsSkipList`|List of dataSources for which pendingSegments are _NOT_ cleaned up if property `druid.coordinator.kill.pendingSegments.on` is true. This can be a list of comma-separated dataSources or a JSON array.|none| |`maxSegmentsInNodeLoadingQueue`|The maximum number of segments that could be queued for loading to any given server. This parameter could be used to speed up segments loading process, especially if there are "slow" nodes in the cluster (with low loading speed) or if too much segments scheduled to be replicated to some particular node (faster loading could be preferred to better segments distribution). Desired value depends on segments loading speed, acceptable replication time and number of nodes. Value 1000 could be a start point for a rather big cluster. Default value is 0 (loading queue is unbounded) |0| To view the audit history of coordinator dynamic config issue a GET request to the URL - From 0738e5c176e1dd7d96fde305b685a9bb6efc6f01 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 24 Apr 2018 13:44:44 -0700 Subject: [PATCH 66/82] [Backport] Fix coordinator's dataSource api with full parameter (#5679) --- .../java/io/druid/client/DruidDataSource.java | 14 ++-- .../client/ImmutableDruidDataSource.java | 42 +++++++++--- .../server/http/DatasourcesResource.java | 13 ++-- .../client/ImmutableDruidDataSourceTest.java | 64 +++++++++++++++++++ .../server/coordinator/DruidClusterTest.java | 9 +-- .../server/coordinator/ServerHolderTest.java | 10 +-- .../server/http/DatasourcesResourceTest.java | 2 +- 7 files changed, 120 insertions(+), 34 deletions(-) create mode 100644 server/src/test/java/io/druid/client/ImmutableDruidDataSourceTest.java diff --git a/server/src/main/java/io/druid/client/DruidDataSource.java b/server/src/main/java/io/druid/client/DruidDataSource.java index 84b04b3b146d..ed2594b67c88 100644 --- a/server/src/main/java/io/druid/client/DruidDataSource.java +++ b/server/src/main/java/io/druid/client/DruidDataSource.java @@ -21,13 +21,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; import io.druid.timeline.DataSegment; import java.util.Collection; import java.util.Collections; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListMap; /** */ @@ -35,7 +34,7 @@ public class DruidDataSource { private final String name; private final Map properties; - private final ConcurrentHashMap idToSegmentMap; + private final ConcurrentSkipListMap idToSegmentMap; public DruidDataSource( String name, @@ -44,7 +43,7 @@ public DruidDataSource( { this.name = Preconditions.checkNotNull(name); this.properties = properties; - this.idToSegmentMap = new ConcurrentHashMap<>(); + this.idToSegmentMap = new ConcurrentSkipListMap<>(); } @JsonProperty @@ -88,11 +87,7 @@ public boolean isEmpty() public ImmutableDruidDataSource toImmutableDruidDataSource() { - return new ImmutableDruidDataSource( - name, - ImmutableMap.copyOf(properties), - ImmutableMap.copyOf(idToSegmentMap) - ); + return new ImmutableDruidDataSource(name, properties, idToSegmentMap); } @Override @@ -107,6 +102,7 @@ public String toString() @Override public boolean equals(Object o) { + //noinspection Contract throw new UnsupportedOperationException("Use ImmutableDruidDataSource instead"); } diff --git a/server/src/main/java/io/druid/client/ImmutableDruidDataSource.java b/server/src/main/java/io/druid/client/ImmutableDruidDataSource.java index 95532f891953..d23dbd628e14 100644 --- a/server/src/main/java/io/druid/client/ImmutableDruidDataSource.java +++ b/server/src/main/java/io/druid/client/ImmutableDruidDataSource.java @@ -19,13 +19,18 @@ package io.druid.client; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedMap; import io.druid.timeline.DataSegment; import java.util.Collection; import java.util.Map; import java.util.Objects; +import java.util.SortedMap; /** */ @@ -33,39 +38,58 @@ public class ImmutableDruidDataSource { private final String name; private final ImmutableMap properties; - private final ImmutableMap idToSegments; + private final ImmutableSortedMap idToSegments; public ImmutableDruidDataSource( String name, - ImmutableMap properties, - ImmutableMap idToSegments + Map properties, + SortedMap idToSegments ) { this.name = Preconditions.checkNotNull(name); - this.properties = properties; - this.idToSegments = idToSegments; + this.properties = ImmutableMap.copyOf(properties); + this.idToSegments = ImmutableSortedMap.copyOfSorted(idToSegments); } + @JsonCreator + public ImmutableDruidDataSource( + @JsonProperty("name") String name, + @JsonProperty("properties") Map properties, + @JsonProperty("segments") Collection segments + ) + { + this.name = Preconditions.checkNotNull(name); + this.properties = ImmutableMap.copyOf(properties); + final ImmutableSortedMap.Builder builder = ImmutableSortedMap.naturalOrder(); + segments.forEach(segment -> builder.put(segment.getIdentifier(), segment)); + this.idToSegments = builder.build(); + } + + @JsonProperty public String getName() { return name; } + @JsonProperty public Map getProperties() { return properties; } - public boolean isEmpty() + @JsonProperty + public Collection getSegments() { - return idToSegments.isEmpty(); + return idToSegments.values(); } - public Collection getSegments() + @JsonIgnore + public boolean isEmpty() { - return idToSegments.values(); + return idToSegments.isEmpty(); } + @JsonIgnore public DataSegment getSegment(String segmentIdentifier) { return idToSegments.get(segmentIdentifier); diff --git a/server/src/main/java/io/druid/server/http/DatasourcesResource.java b/server/src/main/java/io/druid/server/http/DatasourcesResource.java index c53e37158fbd..a763c613cc56 100644 --- a/server/src/main/java/io/druid/server/http/DatasourcesResource.java +++ b/server/src/main/java/io/druid/server/http/DatasourcesResource.java @@ -65,12 +65,15 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.util.Collections; import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; import java.util.stream.Collectors; /** @@ -143,7 +146,7 @@ public Response getTheDataSource( @QueryParam("full") final String full ) { - ImmutableDruidDataSource dataSource = getDataSource(dataSourceName); + final ImmutableDruidDataSource dataSource = getDataSource(dataSourceName); if (dataSource == null) { return Response.noContent().build(); @@ -508,7 +511,7 @@ private ImmutableDruidDataSource getDataSource(final String dataSourceName) return null; } - Map segmentMap = Maps.newHashMap(); + final SortedMap segmentMap = new TreeMap<>(); for (ImmutableDruidDataSource dataSource : dataSources) { Iterable segments = dataSource.getSegments(); for (DataSegment segment : segments) { @@ -516,11 +519,7 @@ private ImmutableDruidDataSource getDataSource(final String dataSourceName) } } - return new ImmutableDruidDataSource( - dataSourceName, - ImmutableMap.of(), - ImmutableMap.copyOf(segmentMap) - ); + return new ImmutableDruidDataSource(dataSourceName, Collections.emptyMap(), segmentMap); } private Pair> getSegment(String segmentId) diff --git a/server/src/test/java/io/druid/client/ImmutableDruidDataSourceTest.java b/server/src/test/java/io/druid/client/ImmutableDruidDataSourceTest.java new file mode 100644 index 000000000000..ad101dbeea31 --- /dev/null +++ b/server/src/test/java/io/druid/client/ImmutableDruidDataSourceTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.client; + +import com.fasterxml.jackson.databind.InjectableValues.Std; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedMap; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.java.util.common.Intervals; +import io.druid.timeline.DataSegment; +import io.druid.timeline.DataSegment.PruneLoadSpecHolder; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; + +public class ImmutableDruidDataSourceTest +{ + @Test + public void testSerde() throws IOException + { + final DataSegment segment = new DataSegment( + "test", + Intervals.of("2017/2018"), + "version", + null, + ImmutableList.of("dim1", "dim2"), + ImmutableList.of("met1", "met2"), + null, + 1, + 100L, + PruneLoadSpecHolder.DEFAULT + ); + final ImmutableDruidDataSource dataSource = new ImmutableDruidDataSource( + "test", + ImmutableMap.of("prop1", "val1", "prop2", "val2"), + ImmutableSortedMap.of(segment.getIdentifier(), segment) + ); + + final ObjectMapper objectMapper = new DefaultObjectMapper() + .setInjectableValues(new Std().addValue(PruneLoadSpecHolder.class, PruneLoadSpecHolder.DEFAULT)); + final String json = objectMapper.writeValueAsString(dataSource); + Assert.assertEquals(dataSource, objectMapper.readValue(json, ImmutableDruidDataSource.class)); + } +} diff --git a/server/src/test/java/io/druid/server/coordinator/DruidClusterTest.java b/server/src/test/java/io/druid/server/coordinator/DruidClusterTest.java index 90eb5d49e305..933c61e060b0 100644 --- a/server/src/test/java/io/druid/server/coordinator/DruidClusterTest.java +++ b/server/src/test/java/io/druid/server/coordinator/DruidClusterTest.java @@ -39,6 +39,7 @@ import java.util.Map; import java.util.NavigableSet; import java.util.Set; +import java.util.TreeMap; import java.util.TreeSet; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -74,14 +75,14 @@ public class DruidClusterTest "src1", new ImmutableDruidDataSource( "src1", - ImmutableMap.of(), - ImmutableMap.of() + Collections.emptyMap(), + new TreeMap<>() ), "src2", new ImmutableDruidDataSource( "src2", - ImmutableMap.of(), - ImmutableMap.of() + Collections.emptyMap(), + new TreeMap<>() ) ); diff --git a/server/src/test/java/io/druid/server/coordinator/ServerHolderTest.java b/server/src/test/java/io/druid/server/coordinator/ServerHolderTest.java index c25f37bc49fa..2affbd11a841 100644 --- a/server/src/test/java/io/druid/server/coordinator/ServerHolderTest.java +++ b/server/src/test/java/io/druid/server/coordinator/ServerHolderTest.java @@ -31,8 +31,10 @@ import org.junit.Assert; import org.junit.Test; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.TreeMap; public class ServerHolderTest { @@ -65,14 +67,14 @@ public class ServerHolderTest "src1", new ImmutableDruidDataSource( "src1", - ImmutableMap.of(), - ImmutableMap.of() + Collections.emptyMap(), + new TreeMap<>() ), "src2", new ImmutableDruidDataSource( "src2", - ImmutableMap.of(), - ImmutableMap.of() + Collections.emptyMap(), + new TreeMap<>() ) ); diff --git a/server/src/test/java/io/druid/server/http/DatasourcesResourceTest.java b/server/src/test/java/io/druid/server/http/DatasourcesResourceTest.java index 0e5e09939f8e..1b97a2807b49 100644 --- a/server/src/test/java/io/druid/server/http/DatasourcesResourceTest.java +++ b/server/src/test/java/io/druid/server/http/DatasourcesResourceTest.java @@ -309,7 +309,7 @@ public void testGetSimpleQueryableDataSources() throws Exception @Test public void testFullGetTheDataSource() throws Exception { - DruidDataSource dataSource1 = new DruidDataSource("datasource1", new HashMap()); + DruidDataSource dataSource1 = new DruidDataSource("datasource1", new HashMap<>()); EasyMock.expect(server.getDataSource("datasource1")).andReturn( dataSource1 ).atLeastOnce(); From a23cd5c90f26174f5e5c59b8f86fa332382eea3a Mon Sep 17 00:00:00 2001 From: David Lim Date: Mon, 30 Apr 2018 04:41:28 -0600 Subject: [PATCH 67/82] Use unique segment paths for Kafka indexing (#5692) (#5718) * support unique segment file paths * forbiddenapis * code review changes * code review changes * code review changes * checkstyle fix --- .../segment/loading/DataSegmentFinder.java | 27 ++++ .../segment/loading/DataSegmentKiller.java | 33 +++- .../segment/loading/DataSegmentPusher.java | 55 ++++--- .../operations/insert-segment-to-db.md | 45 ++++-- .../storage/azure/AzureDataSegmentPusher.java | 18 +-- .../io/druid/storage/azure/AzureStorage.java | 18 +-- .../io/druid/storage/azure/AzureTaskLogs.java | 2 +- .../azure/AzureDataSegmentPusherTest.java | 36 +++-- .../storage/azure/AzureTaskLogsTest.java | 2 +- .../cassandra/CassandraDataSegmentPusher.java | 29 ++-- .../CloudFilesDataSegmentPusher.java | 38 +++-- .../CloudFilesDataSegmentPusherTest.java | 2 +- .../google/GoogleDataSegmentPusher.java | 16 +- .../google/GoogleDataSegmentPusherTest.java | 14 +- .../storage/hdfs/HdfsDataSegmentFinder.java | 50 ++++-- .../storage/hdfs/HdfsDataSegmentKiller.java | 85 +++++----- .../storage/hdfs/HdfsDataSegmentPusher.java | 53 ++++--- .../org/apache/hadoop/fs/HadoopFsWrapper.java | 7 +- .../loading/HdfsDataSegmentFinderTest.java | 25 +++ .../hdfs/HdfsDataSegmentKillerTest.java | 62 +++++--- .../hdfs/HdfsDataSegmentPusherTest.java | 148 +++++++++++------- .../druid/indexing/kafka/KafkaIndexTask.java | 1 + .../indexing/kafka/KafkaIndexTaskTest.java | 7 +- .../druid/storage/s3/S3DataSegmentFinder.java | 16 +- .../druid/storage/s3/S3DataSegmentMover.java | 7 +- .../druid/storage/s3/S3DataSegmentPusher.java | 20 +-- .../storage/s3/S3DataSegmentFinderTest.java | 91 ++++++++--- .../storage/s3/S3DataSegmentPusherTest.java | 23 ++- .../main/java/io/druid/indexer/JobHelper.java | 14 +- .../common/index/YeOldePlumberSchool.java | 5 +- .../common/task/ConvertSegmentTask.java | 5 +- .../druid/indexing/common/task/IndexTask.java | 3 +- .../indexing/common/task/MergeTaskBase.java | 5 +- .../indexing/common/task/IndexTaskTest.java | 20 ++- .../task/SameIntervalMergeTaskTest.java | 6 +- .../IngestSegmentFirehoseFactoryTest.java | 13 +- .../indexing/overlord/TaskLifecycleTest.java | 36 +++-- .../indexing/test/TestDataSegmentPusher.java | 3 +- .../loading/LocalDataSegmentFinder.java | 26 +-- .../loading/LocalDataSegmentKiller.java | 22 +-- .../loading/LocalDataSegmentPusher.java | 40 ++--- .../SegmentLoaderLocalCacheManager.java | 12 +- .../realtime/appenderator/Appenderator.java | 7 +- .../appenderator/AppenderatorImpl.java | 24 +-- .../appenderator/AppenderatorPlumber.java | 2 +- .../appenderator/BaseAppenderatorDriver.java | 32 +++- .../appenderator/BatchAppenderatorDriver.java | 23 +-- .../StreamAppenderatorDriver.java | 12 +- .../realtime/plumber/RealtimePlumber.java | 7 - .../loading/LocalDataSegmentFinderTest.java | 47 +++++- .../loading/LocalDataSegmentKillerTest.java | 23 +++ .../loading/LocalDataSegmentPusherTest.java | 40 ++--- .../appenderator/AppenderatorTest.java | 3 +- .../appenderator/AppenderatorTester.java | 2 +- .../BatchAppenderatorDriverTest.java | 14 +- .../StreamAppenderatorDriverFailTest.java | 110 ++++++++++++- .../StreamAppenderatorDriverTest.java | 39 +++-- .../java/io/druid/cli/CliRealtimeExample.java | 2 +- 58 files changed, 996 insertions(+), 531 deletions(-) diff --git a/api/src/main/java/io/druid/segment/loading/DataSegmentFinder.java b/api/src/main/java/io/druid/segment/loading/DataSegmentFinder.java index 937a42e72c23..66af8353dafb 100644 --- a/api/src/main/java/io/druid/segment/loading/DataSegmentFinder.java +++ b/api/src/main/java/io/druid/segment/loading/DataSegmentFinder.java @@ -20,8 +20,11 @@ package io.druid.segment.loading; import io.druid.guice.annotations.ExtensionPoint; +import io.druid.java.util.common.Pair; +import io.druid.java.util.common.logger.Logger; import io.druid.timeline.DataSegment; +import java.util.Map; import java.util.Set; /** @@ -31,6 +34,8 @@ @ExtensionPoint public interface DataSegmentFinder { + Logger log = new Logger(DataSegmentFinder.class); + /** * This method should first recursively look for descriptor.json (partitionNum_descriptor.json for HDFS data storage) underneath * workingDirPath and then verify that index.zip (partitionNum_index.zip for HDFS data storage) exists in the same folder. @@ -46,4 +51,26 @@ public interface DataSegmentFinder * @return a set of segments that were found underneath workingDirPath */ Set findSegments(String workingDirPath, boolean updateDescriptor) throws SegmentLoadingException; + + /** + * Adds dataSegment if it does not exist in timestampedSegments. If it exists, replaces entry if segmentModifiedAt is + * newer than stored timestamp. + * + * @param timestampedSegments map of > containing segments with modified time + * @param dataSegment segment to add + * @param segmentModifiedAt segment modified timestamp + */ + static void putInMapRetainingNewest( + Map> timestampedSegments, DataSegment dataSegment, long segmentModifiedAt + ) + { + timestampedSegments.merge( + dataSegment.getIdentifier(), + Pair.of(dataSegment, segmentModifiedAt), + (previous, current) -> { + log.warn("Multiple copies of segmentId [%s] found, using newest version", current.lhs.getIdentifier()); + return previous.rhs > current.rhs ? previous : current; + } + ); + } } diff --git a/api/src/main/java/io/druid/segment/loading/DataSegmentKiller.java b/api/src/main/java/io/druid/segment/loading/DataSegmentKiller.java index c26a73daeb10..96bbeb9362d0 100644 --- a/api/src/main/java/io/druid/segment/loading/DataSegmentKiller.java +++ b/api/src/main/java/io/druid/segment/loading/DataSegmentKiller.java @@ -20,16 +20,41 @@ package io.druid.segment.loading; import io.druid.guice.annotations.ExtensionPoint; +import io.druid.java.util.common.logger.Logger; import io.druid.timeline.DataSegment; import java.io.IOException; -/** - */ @ExtensionPoint public interface DataSegmentKiller { - void kill(DataSegment segments) throws SegmentLoadingException; - void killAll() throws IOException; + Logger log = new Logger(DataSegmentKiller.class); + + /** + * Removes segment files (index and metadata) from deep storage. + * @param segment the segment to kill + * @throws SegmentLoadingException if the segment could not be completely removed + */ + void kill(DataSegment segment) throws SegmentLoadingException; + /** + * A more stoic killer who doesn't throw a tantrum if things get messy. Use when killing segments for best-effort + * cleanup. + * @param segment the segment to kill + */ + default void killQuietly(DataSegment segment) + { + try { + kill(segment); + } + catch (Exception e) { + log.debug(e, "Failed to kill segment %s", segment); + } + } + + /** + * Like a nuke. Use wisely. Used by the 'reset-cluster' command, and of the built-in deep storage implementations, it + * is only implemented by local and HDFS. + */ + void killAll() throws IOException; } diff --git a/api/src/main/java/io/druid/segment/loading/DataSegmentPusher.java b/api/src/main/java/io/druid/segment/loading/DataSegmentPusher.java index 2ce046665977..7c4cead40cb2 100644 --- a/api/src/main/java/io/druid/segment/loading/DataSegmentPusher.java +++ b/api/src/main/java/io/druid/segment/loading/DataSegmentPusher.java @@ -30,6 +30,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.UUID; @ExtensionPoint public interface DataSegmentPusher @@ -44,36 +45,48 @@ public interface DataSegmentPusher * Pushes index files and segment descriptor to deep storage. * @param file directory containing index files * @param segment segment descriptor - * @param replaceExisting overwrites existing objects if true, else leaves existing objects unchanged on conflict. - * The behavior of the indexer determines whether this should be true or false. For example, - * since Tranquility does not guarantee that replica tasks will generate indexes with the same - * data, the first segment pushed should be favored since otherwise multiple historicals may - * load segments with the same identifier but different contents which is a bad situation. On - * the other hand, indexers that maintain exactly-once semantics by storing checkpoint data can - * lose or repeat data if it fails to write a segment because it already exists and overwriting - * is not permitted. This situation can occur if a task fails after pushing to deep storage but - * before writing to the metadata storage, see: https://github.com/druid-io/druid/issues/5161. + * @param useUniquePath if true, pushes to a unique file path. This prevents situations where task failures or replica + * tasks can either overwrite or fail to overwrite existing segments leading to the possibility + * of different versions of the same segment ID containing different data. As an example, a Kafka + * indexing task starting at offset A and ending at offset B may push a segment to deep storage + * and then fail before writing the loadSpec to the metadata table, resulting in a replacement + * task being spawned. This replacement will also start at offset A but will read to offset C and + * will then push a segment to deep storage and write the loadSpec metadata. Without unique file + * paths, this can only work correctly if new segments overwrite existing segments. Suppose that + * at this point the task then fails so that the supervisor retries again from offset A. This 3rd + * attempt will overwrite the segments in deep storage before failing to write the loadSpec + * metadata, resulting in inconsistencies in the segment data now in deep storage and copies of + * the segment already loaded by historicals. * - * If replaceExisting is true, existing objects MUST be overwritten, since failure to do so - * will break exactly-once semantics. If replaceExisting is false, existing objects SHOULD be - * prioritized but it is acceptable if they are overwritten (deep storages may be eventually - * consistent or otherwise unable to support transactional writes). + * If unique paths are used, caller is responsible for cleaning up segments that were pushed but + * were not written to the metadata table (for example when using replica tasks). * @return segment descriptor * @throws IOException */ - DataSegment push(File file, DataSegment segment, boolean replaceExisting) throws IOException; + DataSegment push(File file, DataSegment segment, boolean useUniquePath) throws IOException; //use map instead of LoadSpec class to avoid dependency pollution. Map makeLoadSpec(URI finalIndexZipFilePath); + /** + * @deprecated backward-compatibiliy shim that should be removed on next major release; + * use {@link #getStorageDir(DataSegment, boolean)} instead. + */ + @Deprecated default String getStorageDir(DataSegment dataSegment) { - return getDefaultStorageDir(dataSegment); + return getStorageDir(dataSegment, false); + } + + default String getStorageDir(DataSegment dataSegment, boolean useUniquePath) + { + return getDefaultStorageDir(dataSegment, useUniquePath); } default String makeIndexPathName(DataSegment dataSegment, String indexName) { - return StringUtils.format("./%s/%s", getStorageDir(dataSegment), indexName); + // This is only called from Hadoop batch which doesn't require unique segment paths so set useUniquePath=false + return StringUtils.format("./%s/%s", getStorageDir(dataSegment, false), indexName); } /** @@ -89,13 +102,19 @@ default List getAllowedPropertyPrefixesForHadoop() // If above format is ever changed, make sure to change it appropriately in other places // e.g. HDFSDataSegmentKiller uses this information to clean the version, interval and dataSource directories // on segment deletion if segment being deleted was the only segment - static String getDefaultStorageDir(DataSegment segment) + static String getDefaultStorageDir(DataSegment segment, boolean useUniquePath) { return JOINER.join( segment.getDataSource(), StringUtils.format("%s_%s", segment.getInterval().getStart(), segment.getInterval().getEnd()), segment.getVersion(), - segment.getShardSpec().getPartitionNum() + segment.getShardSpec().getPartitionNum(), + useUniquePath ? generateUniquePath() : null ); } + + static String generateUniquePath() + { + return UUID.randomUUID().toString(); + } } diff --git a/docs/content/operations/insert-segment-to-db.md b/docs/content/operations/insert-segment-to-db.md index 60549e69d6c2..cd5ba2264aeb 100644 --- a/docs/content/operations/insert-segment-to-db.md +++ b/docs/content/operations/insert-segment-to-db.md @@ -5,24 +5,43 @@ layout: doc_page `insert-segment-to-db` is a tool that can insert segments into Druid metadata storage. It is intended to be used to update the segment table in metadata storage after people manually migrate segments from one place to another. -It can also be used to insert missing segment into Druid, or even recover metadata storage by telling it where the +It can also be used to insert missing segments into Druid, or even recover metadata storage by telling it where the segments are stored. -Note: This tool expects users to have Druid cluster running in a "safe" mode, where there are no active tasks to interfere -the segments being inserted. Users can optionally bring down the cluster to make 100% sure nothing is interfering. +**Note:** This tool simply scans the deep storage directory to reconstruct the metadata entries used to locate and +identify each segment. It does not have any understanding about whether those segments _should actually_ be written to +the metadata storage. In certain cases, this can lead to undesired or inconsistent results. Some examples of things to +watch out for: + - Dropped datasources will be re-enabled. + - The latest version of each segment set will be loaded by Druid, which in some cases may not be the version you + actually want. An example of this is a bad compaction job that generates segments which need to be manually rolled + back by removing that version from the metadata table. If these segments are not also removed from deep storage, + they will be imported back into the metadata table and overshadow the correct version. + - Some indexers such as the Kafka indexing service have the potential to generate more than one set of segments that + have the same segment ID but different contents. When the metadata is first written, the correct set of segments is + referenced and the other set is normally deleted from deep storage. It is possible however that an unhandled + exception could result in multiple sets of segments with the same segment ID remaining in deep storage. Since this + tool does not know which one is the 'correct' one to use, it will simply select the newest segment set and ignore + the other versions. If the wrong segment set is picked, the exactly-once semantics of the Kafka indexing service + will no longer hold true and you may get duplicated or dropped events. + +With these considerations in mind, it is recommended that data migrations be done by exporting the original metadata +storage directly, since that is the definitive cluster state. This tool should be used as a last resort when a direct +export is not possible. + +**Note:** This tool expects users to have Druid cluster running in a "safe" mode, where there are no active tasks to interfere +with the segments being inserted. Users can optionally bring down the cluster to make 100% sure nothing is interfering. In order to make it work, user will have to provide metadata storage credentials and deep storage type through Java JVM argument -or runtime.properties file. Specifically, this tool needs to know +or runtime.properties file. Specifically, this tool needs to know: -`druid.metadata.storage.type` - -`druid.metadata.storage.connector.connectURI` - -`druid.metadata.storage.connector.user` - -`druid.metadata.storage.connector.password` - -`druid.storage.type` +``` +druid.metadata.storage.type +druid.metadata.storage.connector.connectURI +druid.metadata.storage.connector.user +druid.metadata.storage.connector.password +druid.storage.type +``` Besides the properties above, you also need to specify the location where the segments are stored and whether you want to update descriptor.json (`partitionNum_descriptor.json` for HDFS data storage). These two can be provided through command line arguments. diff --git a/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPusher.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPusher.java index 39ab34258e0d..592b956c260d 100644 --- a/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPusher.java +++ b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureDataSegmentPusher.java @@ -25,7 +25,6 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import com.microsoft.azure.storage.StorageException; - import io.druid.java.util.common.CompressionUtils; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; @@ -92,9 +91,9 @@ public File createSegmentDescriptorFile(final ObjectMapper jsonMapper, final Dat return descriptorFile; } - public Map getAzurePaths(final DataSegment segment) + public Map getAzurePaths(final DataSegment segment, final boolean useUniquePath) { - final String storageDir = this.getStorageDir(segment); + final String storageDir = this.getStorageDir(segment, useUniquePath); return ImmutableMap.of( "index", StringUtils.format("%s/%s", storageDir, AzureStorageDruidModule.INDEX_ZIP_FILE_NAME), @@ -109,13 +108,12 @@ public DataSegment uploadDataSegment( final long size, final File compressedSegmentData, final File descriptorFile, - final Map azurePaths, - final boolean replaceExisting + final Map azurePaths ) throws StorageException, IOException, URISyntaxException { - azureStorage.uploadBlob(compressedSegmentData, config.getContainer(), azurePaths.get("index"), replaceExisting); - azureStorage.uploadBlob(descriptorFile, config.getContainer(), azurePaths.get("descriptor"), replaceExisting); + azureStorage.uploadBlob(compressedSegmentData, config.getContainer(), azurePaths.get("index")); + azureStorage.uploadBlob(descriptorFile, config.getContainer(), azurePaths.get("descriptor")); final DataSegment outSegment = segment .withSize(size) @@ -132,7 +130,7 @@ public DataSegment uploadDataSegment( } @Override - public DataSegment push(final File indexFilesDir, final DataSegment segment, final boolean replaceExisting) + public DataSegment push(final File indexFilesDir, final DataSegment segment, final boolean useUniquePath) throws IOException { log.info("Uploading [%s] to Azure.", indexFilesDir); @@ -146,7 +144,7 @@ public DataSegment push(final File indexFilesDir, final DataSegment segment, fin final long size = CompressionUtils.zip(indexFilesDir, zipOutFile); final File descFile = descriptorFile = createSegmentDescriptorFile(jsonMapper, segment); - final Map azurePaths = getAzurePaths(segment); + final Map azurePaths = getAzurePaths(segment, useUniquePath); return AzureUtils.retryAzureOperation( new Callable() @@ -154,7 +152,7 @@ public DataSegment push(final File indexFilesDir, final DataSegment segment, fin @Override public DataSegment call() throws Exception { - return uploadDataSegment(segment, version, size, outFile, descFile, azurePaths, replaceExisting); + return uploadDataSegment(segment, version, size, outFile, descFile, azurePaths); } }, config.getMaxTries() diff --git a/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureStorage.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureStorage.java index 28fa9dad56c6..25a4764ad845 100644 --- a/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureStorage.java +++ b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureStorage.java @@ -23,7 +23,6 @@ import com.microsoft.azure.storage.blob.CloudBlob; import com.microsoft.azure.storage.blob.CloudBlobClient; import com.microsoft.azure.storage.blob.CloudBlobContainer; -import com.microsoft.azure.storage.blob.CloudBlockBlob; import com.microsoft.azure.storage.blob.ListBlobItem; import io.druid.java.util.common.logger.Logger; @@ -77,27 +76,14 @@ public List emptyCloudBlobDirectory(final String containerName, final St } return deletedFiles; - } - public void uploadBlob( - final File file, - final String containerName, - final String blobPath, - final boolean replaceExisting - ) + public void uploadBlob(final File file, final String containerName, final String blobPath) throws IOException, StorageException, URISyntaxException - { CloudBlobContainer container = getCloudBlobContainer(containerName); try (FileInputStream stream = new FileInputStream(file)) { - CloudBlockBlob blob = container.getBlockBlobReference(blobPath); - - if (!replaceExisting && blob.exists()) { - log.info("Skipping push because blob [%s] exists && replaceExisting == false", blobPath); - } else { - blob.upload(stream, file.length()); - } + container.getBlockBlobReference(blobPath).upload(stream, file.length()); } } diff --git a/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureTaskLogs.java b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureTaskLogs.java index 50092ceff62c..f4b6093ed760 100644 --- a/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureTaskLogs.java +++ b/extensions-contrib/azure-extensions/src/main/java/io/druid/storage/azure/AzureTaskLogs.java @@ -59,7 +59,7 @@ public void pushTaskLog(final String taskid, final File logFile) throws IOExcept try { AzureUtils.retryAzureOperation( (Callable) () -> { - azureStorage.uploadBlob(logFile, config.getContainer(), taskKey, true); + azureStorage.uploadBlob(logFile, config.getContainer(), taskKey); return null; }, config.getMaxTries() diff --git a/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentPusherTest.java b/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentPusherTest.java index 30c571a4f163..3201c4afeca3 100644 --- a/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentPusherTest.java +++ b/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureDataSegmentPusherTest.java @@ -82,6 +82,17 @@ public void before() @Test public void testPush() throws Exception + { + testPushInternal(false, "foo/2015-01-01T00:00:00\\.000Z_2016-01-01T00:00:00\\.000Z/0/0/index\\.zip"); + } + + @Test + public void testPushUseUniquePath() throws Exception + { + testPushInternal(true, "foo/2015-01-01T00:00:00\\.000Z_2016-01-01T00:00:00\\.000Z/0/0/[A-Za-z0-9-]{36}/index\\.zip"); + } + + private void testPushInternal(boolean useUniquePath, String matcher) throws Exception { AzureDataSegmentPusher pusher = new AzureDataSegmentPusher(azureStorage, azureAccountConfig, jsonMapper); @@ -104,7 +115,12 @@ public void testPush() throws Exception size ); - DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush, true); + DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush, useUniquePath); + + Assert.assertTrue( + segment.getLoadSpec().get("blobPath").toString(), + segment.getLoadSpec().get("blobPath").toString().matches(matcher) + ); Assert.assertEquals(segmentToPush.getSize(), segment.getSize()); } @@ -114,10 +130,13 @@ public void getAzurePathsTest() { AzureDataSegmentPusher pusher = new AzureDataSegmentPusher(azureStorage, azureAccountConfig, jsonMapper); - final String storageDir = pusher.getStorageDir(dataSegment); - Map paths = pusher.getAzurePaths(dataSegment); + final String storageDir = pusher.getStorageDir(dataSegment, false); + Map paths = pusher.getAzurePaths(dataSegment, false); - assertEquals(StringUtils.format("%s/%s", storageDir, AzureStorageDruidModule.INDEX_ZIP_FILE_NAME), paths.get("index")); + assertEquals( + StringUtils.format("%s/%s", storageDir, AzureStorageDruidModule.INDEX_ZIP_FILE_NAME), + paths.get("index") + ); assertEquals( StringUtils.format("%s/%s", storageDir, AzureStorageDruidModule.DESCRIPTOR_FILE_NAME), paths.get("descriptor") @@ -131,11 +150,11 @@ public void uploadDataSegmentTest() throws StorageException, IOException, URISyn final int version = 9; final File compressedSegmentData = new File("index.zip"); final File descriptorFile = new File("descriptor.json"); - final Map azurePaths = pusher.getAzurePaths(dataSegment); + final Map azurePaths = pusher.getAzurePaths(dataSegment, false); - azureStorage.uploadBlob(compressedSegmentData, containerName, azurePaths.get("index"), true); + azureStorage.uploadBlob(compressedSegmentData, containerName, azurePaths.get("index")); expectLastCall(); - azureStorage.uploadBlob(descriptorFile, containerName, azurePaths.get("descriptor"), true); + azureStorage.uploadBlob(descriptorFile, containerName, azurePaths.get("descriptor")); expectLastCall(); replayAll(); @@ -146,8 +165,7 @@ public void uploadDataSegmentTest() throws StorageException, IOException, URISyn 0, // empty file compressedSegmentData, descriptorFile, - azurePaths, - true + azurePaths ); assertEquals(compressedSegmentData.length(), pushedDataSegment.getSize()); diff --git a/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureTaskLogsTest.java b/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureTaskLogsTest.java index 5f8394a735a4..26ff677af177 100644 --- a/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureTaskLogsTest.java +++ b/extensions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureTaskLogsTest.java @@ -65,7 +65,7 @@ public void testPushTaskLog() throws Exception try { final File logFile = new File(tmpDir, "log"); - azureStorage.uploadBlob(logFile, container, prefix + "/" + taskid + "/log", true); + azureStorage.uploadBlob(logFile, container, prefix + "/" + taskid + "/log"); expectLastCall(); replayAll(); diff --git a/extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentPusher.java b/extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentPusher.java index 3b3ad8e5539b..16ff585327db 100644 --- a/extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentPusher.java +++ b/extensions-contrib/cassandra-storage/src/main/java/io/druid/storage/cassandra/CassandraDataSegmentPusher.java @@ -75,13 +75,12 @@ public String getPathForHadoop(String dataSource) } @Override - public DataSegment push(final File indexFilesDir, DataSegment segment, final boolean replaceExisting) - throws IOException + public DataSegment push(final File indexFilesDir, DataSegment segment, final boolean useUniquePath) throws IOException { log.info("Writing [%s] to C*", indexFilesDir); String key = JOINER.join( config.getKeyspace().isEmpty() ? null : config.getKeyspace(), - this.getStorageDir(segment) + this.getStorageDir(segment, useUniquePath) ); // Create index @@ -92,20 +91,16 @@ public DataSegment push(final File indexFilesDir, DataSegment segment, final boo int version = SegmentUtils.getVersionFromDir(indexFilesDir); try { - if (!replaceExisting && doesObjectExist(indexStorage, key)) { - log.info("Skipping push because key [%s] exists && replaceExisting == false", key); - } else { - long start = System.currentTimeMillis(); - ChunkedStorage.newWriter(indexStorage, key, new FileInputStream(compressedIndexFile)) - .withConcurrencyLevel(CONCURRENCY).call(); - byte[] json = jsonMapper.writeValueAsBytes(segment); - MutationBatch mutation = this.keyspace.prepareMutationBatch(); - mutation.withRow(descriptorStorage, key) - .putColumn("lastmodified", System.currentTimeMillis(), null) - .putColumn("descriptor", json, null); - mutation.execute(); - log.info("Wrote index to C* in [%s] ms", System.currentTimeMillis() - start); - } + long start = System.currentTimeMillis(); + ChunkedStorage.newWriter(indexStorage, key, new FileInputStream(compressedIndexFile)) + .withConcurrencyLevel(CONCURRENCY).call(); + byte[] json = jsonMapper.writeValueAsBytes(segment); + MutationBatch mutation = this.keyspace.prepareMutationBatch(); + mutation.withRow(descriptorStorage, key) + .putColumn("lastmodified", System.currentTimeMillis(), null) + .putColumn("descriptor", json, null); + mutation.execute(); + log.info("Wrote index to C* in [%s] ms", System.currentTimeMillis() - start); } catch (Exception e) { throw new IOException(e); diff --git a/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusher.java b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusher.java index e224a5b83b78..be3e25472354 100644 --- a/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusher.java +++ b/extensions-contrib/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusher.java @@ -31,7 +31,6 @@ import org.jclouds.rackspace.cloudfiles.v1.CloudFilesApi; import java.io.File; -import java.io.IOException; import java.net.URI; import java.nio.file.Files; import java.util.Map; @@ -74,10 +73,12 @@ public String getPathForHadoop(final String dataSource) } @Override - public DataSegment push(final File indexFilesDir, final DataSegment inSegment, final boolean replaceExisting) - throws IOException + public DataSegment push(final File indexFilesDir, final DataSegment inSegment, final boolean useUniquePath) { - final String segmentPath = CloudFilesUtils.buildCloudFilesPath(this.config.getBasePath(), getStorageDir(inSegment)); + final String segmentPath = CloudFilesUtils.buildCloudFilesPath( + this.config.getBasePath(), + getStorageDir(inSegment, useUniquePath) + ); File descriptorFile = null; File zipOutFile = null; @@ -100,22 +101,19 @@ public DataSegment call() throws Exception objectApi.getContainer() ); - if (!replaceExisting && objectApi.exists(segmentData.getPath())) { - log.info("Skipping push because object [%s] exists && replaceExisting == false", segmentData.getPath()); - } else { - log.info("Pushing %s.", segmentData.getPath()); - objectApi.put(segmentData); - - // Avoid using Guava in DataSegmentPushers because they might be used with very diverse Guava versions in - // runtime, and because Guava deletes methods over time, that causes incompatibilities. - Files.write(descFile.toPath(), jsonMapper.writeValueAsBytes(inSegment)); - CloudFilesObject descriptorData = new CloudFilesObject( - segmentPath, descFile, - objectApi.getRegion(), objectApi.getContainer() - ); - log.info("Pushing %s.", descriptorData.getPath()); - objectApi.put(descriptorData); - } + log.info("Pushing %s.", segmentData.getPath()); + objectApi.put(segmentData); + + // Avoid using Guava in DataSegmentPushers because they might be used with very diverse Guava versions in + // runtime, and because Guava deletes methods over time, that causes incompatibilities. + Files.write(descFile.toPath(), jsonMapper.writeValueAsBytes(inSegment)); + CloudFilesObject descriptorData = new CloudFilesObject( + segmentPath, descFile, + objectApi.getRegion(), objectApi.getContainer() + ); + log.info("Pushing %s.", descriptorData.getPath()); + objectApi.put(descriptorData); + final DataSegment outSegment = inSegment .withSize(indexSize) diff --git a/extensions-contrib/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusherTest.java b/extensions-contrib/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusherTest.java index 73be6ce0e1c1..2be6f584811b 100644 --- a/extensions-contrib/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusherTest.java +++ b/extensions-contrib/cloudfiles-extensions/src/test/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPusherTest.java @@ -84,7 +84,7 @@ public void testPush() throws Exception size ); - DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush, true); + DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush, false); Assert.assertEquals(segmentToPush.getSize(), segment.getSize()); diff --git a/extensions-contrib/google-extensions/src/main/java/io/druid/storage/google/GoogleDataSegmentPusher.java b/extensions-contrib/google-extensions/src/main/java/io/druid/storage/google/GoogleDataSegmentPusher.java index e8489ea8bd86..7a3581462d36 100644 --- a/extensions-contrib/google-extensions/src/main/java/io/druid/storage/google/GoogleDataSegmentPusher.java +++ b/extensions-contrib/google-extensions/src/main/java/io/druid/storage/google/GoogleDataSegmentPusher.java @@ -93,7 +93,7 @@ public File createDescriptorFile(final ObjectMapper jsonMapper, final DataSegmen return descriptorFile; } - public void insert(final File file, final String contentType, final String path, final boolean replaceExisting) + public void insert(final File file, final String contentType, final String path) throws IOException { LOG.info("Inserting [%s] to [%s]", file, path); @@ -103,15 +103,11 @@ public void insert(final File file, final String contentType, final String path, InputStreamContent mediaContent = new InputStreamContent(contentType, fileSteam); mediaContent.setLength(file.length()); - if (!replaceExisting && storage.exists(config.getBucket(), path)) { - LOG.info("Skipping push because path [%s] exists && replaceExisting == false", path); - } else { - storage.insert(config.getBucket(), path, mediaContent); - } + storage.insert(config.getBucket(), path, mediaContent); } @Override - public DataSegment push(final File indexFilesDir, final DataSegment segment, final boolean replaceExisting) + public DataSegment push(final File indexFilesDir, final DataSegment segment, final boolean useUniquePath) throws IOException { LOG.info("Uploading [%s] to Google.", indexFilesDir); @@ -123,7 +119,7 @@ public DataSegment push(final File indexFilesDir, final DataSegment segment, fin try { indexFile = File.createTempFile("index", ".zip"); final long indexSize = CompressionUtils.zip(indexFilesDir, indexFile); - final String storageDir = this.getStorageDir(segment); + final String storageDir = this.getStorageDir(segment, useUniquePath); final String indexPath = buildPath(storageDir + "/" + "index.zip"); final String descriptorPath = buildPath(storageDir + "/" + "descriptor.json"); @@ -134,8 +130,8 @@ public DataSegment push(final File indexFilesDir, final DataSegment segment, fin descriptorFile = createDescriptorFile(jsonMapper, outSegment); - insert(indexFile, "application/zip", indexPath, replaceExisting); - insert(descriptorFile, "application/json", descriptorPath, replaceExisting); + insert(indexFile, "application/zip", indexPath); + insert(descriptorFile, "application/json", descriptorPath); return outSegment; } diff --git a/extensions-contrib/google-extensions/src/test/java/io/druid/storage/google/GoogleDataSegmentPusherTest.java b/extensions-contrib/google-extensions/src/test/java/io/druid/storage/google/GoogleDataSegmentPusherTest.java index 11372c243aa0..b90f0d5f560e 100644 --- a/extensions-contrib/google-extensions/src/test/java/io/druid/storage/google/GoogleDataSegmentPusherTest.java +++ b/extensions-contrib/google-extensions/src/test/java/io/druid/storage/google/GoogleDataSegmentPusherTest.java @@ -89,7 +89,7 @@ public void testPush() throws Exception "foo", Intervals.of("2015/2016"), "0", - Maps.newHashMap(), + Maps.newHashMap(), Lists.newArrayList(), Lists.newArrayList(), new NoneShardSpec(), @@ -103,30 +103,28 @@ public void testPush() throws Exception storage, googleAccountConfig, jsonMapper - ).addMockedMethod("insert", File.class, String.class, String.class, boolean.class).createMock(); + ).addMockedMethod("insert", File.class, String.class, String.class).createMock(); - final String storageDir = pusher.getStorageDir(segmentToPush); + final String storageDir = pusher.getStorageDir(segmentToPush, false); final String indexPath = prefix + "/" + storageDir + "/" + "index.zip"; final String descriptorPath = prefix + "/" + storageDir + "/" + "descriptor.json"; pusher.insert( EasyMock.anyObject(File.class), EasyMock.eq("application/zip"), - EasyMock.eq(indexPath), - EasyMock.eq(true) + EasyMock.eq(indexPath) ); expectLastCall(); pusher.insert( EasyMock.anyObject(File.class), EasyMock.eq("application/json"), - EasyMock.eq(descriptorPath), - EasyMock.eq(true) + EasyMock.eq(descriptorPath) ); expectLastCall(); replayAll(); - DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush, true); + DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush, false); Assert.assertEquals(segmentToPush.getSize(), segment.getSize()); Assert.assertEquals(segmentToPush, segment); diff --git a/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java index 6fba009cf865..8e75b36057c2 100644 --- a/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java +++ b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentFinder.java @@ -20,13 +20,14 @@ package io.druid.storage.hdfs; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Sets; +import com.google.common.base.Preconditions; import com.google.inject.Inject; +import io.druid.java.util.common.Pair; +import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; import io.druid.segment.loading.DataSegmentFinder; import io.druid.segment.loading.SegmentLoadingException; import io.druid.timeline.DataSegment; -import io.druid.java.util.common.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; @@ -34,14 +35,15 @@ import org.apache.hadoop.fs.RemoteIterator; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** */ public class HdfsDataSegmentFinder implements DataSegmentFinder { - private static final Logger log = new Logger(HdfsDataSegmentFinder.class); private final Configuration config; @@ -58,7 +60,7 @@ public HdfsDataSegmentFinder(Configuration config, ObjectMapper mapper) public Set findSegments(String workingDirPathStr, boolean updateDescriptor) throws SegmentLoadingException { - final Set segments = Sets.newHashSet(); + final Map> timestampedSegments = new HashMap<>(); final Path workingDirPath = new Path(workingDirPathStr); FileSystem fs; try { @@ -80,15 +82,31 @@ public Set findSegments(String workingDirPathStr, boolean updateDes final LocatedFileStatus locatedFileStatus = it.next(); final Path path = locatedFileStatus.getPath(); if (path.getName().endsWith("descriptor.json")) { - final Path indexZip; + + // There are 3 supported path formats: + // - hdfs://nn1/hdfs_base_directory/data_source_name/interval/version/shardNum/descriptor.json + // - hdfs://nn1/hdfs_base_directory/data_source_name/interval/version/shardNum_descriptor.json + // - hdfs://nn1/hdfs_base_directory/data_source_name/interval/version/shardNum_UUID_descriptor.json final String descriptorParts[] = path.getName().split("_"); - if (descriptorParts.length == 2 - && descriptorParts[1].equals("descriptor.json") - && org.apache.commons.lang.StringUtils.isNumeric(descriptorParts[0])) { - indexZip = new Path(path.getParent(), StringUtils.format("%s_index.zip", descriptorParts[0])); - } else { - indexZip = new Path(path.getParent(), "index.zip"); + + Path indexZip = new Path(path.getParent(), "index.zip"); + if (descriptorParts.length > 1) { + Preconditions.checkState(descriptorParts.length <= 3 && + org.apache.commons.lang.StringUtils.isNumeric(descriptorParts[0]) && + "descriptor.json".equals(descriptorParts[descriptorParts.length - 1]), + "Unexpected descriptor filename format [%s]", path + ); + + indexZip = new Path( + path.getParent(), + StringUtils.format( + "%s_%sindex.zip", + descriptorParts[0], + descriptorParts.length == 2 ? "" : descriptorParts[1] + "_" + ) + ); } + if (fs.exists(indexZip)) { final DataSegment dataSegment = mapper.readValue(fs.open(path), DataSegment.class); log.info("Found segment [%s] located at [%s]", dataSegment.getIdentifier(), indexZip); @@ -105,7 +123,12 @@ public Set findSegments(String workingDirPathStr, boolean updateDes mapper.writeValue(fs.create(path, true), dataSegment); } } - segments.add(dataSegment); + + DataSegmentFinder.putInMapRetainingNewest( + timestampedSegments, + dataSegment, + locatedFileStatus.getModificationTime() + ); } else { throw new SegmentLoadingException( "index.zip didn't exist at [%s] while descripter.json exists!?", @@ -119,7 +142,6 @@ public Set findSegments(String workingDirPathStr, boolean updateDes throw new SegmentLoadingException(e, "Problems interacting with filesystem[%s].", workingDirPath); } - return segments; + return timestampedSegments.values().stream().map(x -> x.lhs).collect(Collectors.toSet()); } - } diff --git a/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java index 9ce604e7d203..6cbd1eccd7c3 100644 --- a/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java +++ b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentKiller.java @@ -19,6 +19,7 @@ package io.druid.storage.hdfs; +import com.google.common.base.Preconditions; import com.google.inject.Inject; import io.druid.java.util.emitter.EmittingLogger; import io.druid.segment.loading.DataSegmentKiller; @@ -57,65 +58,53 @@ private static Path getPath(DataSegment segment) public void kill(DataSegment segment) throws SegmentLoadingException { final Path segmentPath = getPath(segment); - log.info("killing segment[%s] mapped to path[%s]", segment.getIdentifier(), segmentPath); + log.info("Killing segment[%s] mapped to path[%s]", segment.getIdentifier(), segmentPath); try { - String segmentLocation = segmentPath.getName(); + String filename = segmentPath.getName(); final FileSystem fs = segmentPath.getFileSystem(config); - if (!segmentLocation.endsWith(".zip")) { + if (!filename.endsWith(".zip")) { throw new SegmentLoadingException("Unknown file type[%s]", segmentPath); } else { if (!fs.exists(segmentPath)) { - log.warn("Segment Path [%s] does not exist. It appears to have been deleted already.", segmentPath); + log.warn("Segment path [%s] does not exist", segmentPath); return; } - String[] zipParts = segmentLocation.split("_"); - // for segments stored as hdfs://nn1/hdfs_base_directory/data_source_name/interval/version/shardNum_index.zip - if (zipParts.length == 2 - && zipParts[1].equals("index.zip") - && StringUtils.isNumeric(zipParts[0])) { - if (!fs.delete(segmentPath, false)) { - throw new SegmentLoadingException( - "Unable to kill segment, failed to delete [%s]", - segmentPath.toString() - ); - } - Path descriptorPath = new Path( + // There are 3 supported path formats: + // - hdfs://nn1/hdfs_base_directory/data_source_name/interval/version/shardNum/index.zip + // - hdfs://nn1/hdfs_base_directory/data_source_name/interval/version/shardNum_index.zip + // - hdfs://nn1/hdfs_base_directory/data_source_name/interval/version/shardNum_UUID_index.zip + String[] zipParts = filename.split("_"); + + Path descriptorPath = new Path(segmentPath.getParent(), "descriptor.json"); + if (zipParts.length > 1) { + Preconditions.checkState(zipParts.length <= 3 && + StringUtils.isNumeric(zipParts[0]) && + "index.zip".equals(zipParts[zipParts.length - 1]), + "Unexpected segmentPath format [%s]", segmentPath + ); + + descriptorPath = new Path( segmentPath.getParent(), - io.druid.java.util.common.StringUtils.format("%s_descriptor.json", zipParts[0]) + io.druid.java.util.common.StringUtils.format( + "%s_%sdescriptor.json", + zipParts[0], + zipParts.length == 2 ? "" : zipParts[1] + "_" + ) ); - //delete partitionNumber_descriptor.json - if (!fs.delete(descriptorPath, false)) { - throw new SegmentLoadingException( - "Unable to kill segment, failed to delete [%s]", - descriptorPath.toString() - ); - } - //for segments stored as hdfs://nn1/hdfs_base_directory/data_source_name/interval/version/shardNum_index.zip - // max depth to look is 2, i.e version directory and interval. - mayBeDeleteParentsUpto(fs, segmentPath, 2); - - } else { //for segments stored as hdfs://nn1/hdfs_base_directory/data_source_name/interval/version/shardNum/ - // index.zip - if (!fs.delete(segmentPath, false)) { - throw new SegmentLoadingException( - "Unable to kill segment, failed to delete [%s]", - segmentPath.toString() - ); - } - Path descriptorPath = new Path(segmentPath.getParent(), "descriptor.json"); - if (!fs.delete(descriptorPath, false)) { - throw new SegmentLoadingException( - "Unable to kill segment, failed to delete [%s]", - descriptorPath.toString() - ); - } - //for segments stored as hdfs://nn1/hdfs_base_directory/data_source_name/interval/version/shardNum/index.zip - //max depth to look is 3, i.e partition number directory,version directory and interval. - mayBeDeleteParentsUpto(fs, segmentPath, 3); } + + if (!fs.delete(segmentPath, false)) { + throw new SegmentLoadingException("Unable to kill segment, failed to delete [%s]", segmentPath.toString()); + } + + if (!fs.delete(descriptorPath, false)) { + throw new SegmentLoadingException("Unable to kill segment, failed to delete [%s]", descriptorPath.toString()); + } + + removeEmptyParentDirectories(fs, segmentPath, zipParts.length > 1 ? 2 : 3); } } catch (IOException e) { @@ -131,11 +120,11 @@ public void killAll() throws IOException fs.delete(storageDirectory, true); } - private void mayBeDeleteParentsUpto(final FileSystem fs, final Path segmentPath, final int maxDepthTobeDeleted) + private void removeEmptyParentDirectories(final FileSystem fs, final Path segmentPath, final int depth) { Path path = segmentPath; try { - for (int i = 1; i <= maxDepthTobeDeleted; i++) { + for (int i = 1; i <= depth; i++) { path = path.getParent(); if (fs.listStatus(path).length != 0 || !fs.delete(path, false)) { break; diff --git a/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPusher.java b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPusher.java index 3374d4333329..94b41e70e199 100644 --- a/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPusher.java +++ b/extensions-core/hdfs-storage/src/main/java/io/druid/storage/hdfs/HdfsDataSegmentPusher.java @@ -20,6 +20,7 @@ package io.druid.storage.hdfs; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteSink; import com.google.common.io.ByteSource; @@ -57,11 +58,8 @@ public class HdfsDataSegmentPusher implements DataSegmentPusher private final String fullyQualifiedStorageDirectory; @Inject - public HdfsDataSegmentPusher( - HdfsDataSegmentPusherConfig config, - Configuration hadoopConfig, - ObjectMapper jsonMapper - ) throws IOException + public HdfsDataSegmentPusher(HdfsDataSegmentPusherConfig config, Configuration hadoopConfig, ObjectMapper jsonMapper) + throws IOException { this.config = config; this.hadoopConfig = hadoopConfig; @@ -89,9 +87,11 @@ public String getPathForHadoop() } @Override - public DataSegment push(File inDir, DataSegment segment, boolean replaceExisting) throws IOException + public DataSegment push(final File inDir, final DataSegment segment, final boolean useUniquePath) throws IOException { - final String storageDir = this.getStorageDir(segment); + // For HDFS, useUniquePath does not affect the directory tree but instead affects the filename, which is of the form + // '{partitionNum}_index.zip' without unique paths and '{partitionNum}_{UUID}_index.zip' with unique paths. + final String storageDir = this.getStorageDir(segment, false); log.info( "Copying segment[%s] to HDFS at location[%s/%s]", @@ -116,17 +116,20 @@ public DataSegment push(File inDir, DataSegment segment, boolean replaceExisting final DataSegment dataSegment; try (FSDataOutputStream out = fs.create(tmpIndexFile)) { size = CompressionUtils.zip(inDir, out); + final String uniquePrefix = useUniquePath ? DataSegmentPusher.generateUniquePath() + "_" : ""; final Path outIndexFile = new Path(StringUtils.format( - "%s/%s/%d_index.zip", + "%s/%s/%d_%sindex.zip", fullyQualifiedStorageDirectory, storageDir, - segment.getShardSpec().getPartitionNum() + segment.getShardSpec().getPartitionNum(), + uniquePrefix )); final Path outDescriptorFile = new Path(StringUtils.format( - "%s/%s/%d_descriptor.json", + "%s/%s/%d_%sdescriptor.json", fullyQualifiedStorageDirectory, storageDir, - segment.getShardSpec().getPartitionNum() + segment.getShardSpec().getPartitionNum(), + uniquePrefix )); dataSegment = segment.withLoadSpec(makeLoadSpec(outIndexFile.toUri())) @@ -145,8 +148,8 @@ public DataSegment push(File inDir, DataSegment segment, boolean replaceExisting // Create parent if it does not exist, recreation is not an error fs.mkdirs(outIndexFile.getParent()); - copyFilesWithChecks(fs, tmpDescriptorFile, outDescriptorFile, replaceExisting); - copyFilesWithChecks(fs, tmpIndexFile, outIndexFile, replaceExisting); + copyFilesWithChecks(fs, tmpDescriptorFile, outDescriptorFile); + copyFilesWithChecks(fs, tmpIndexFile, outIndexFile); } finally { try { @@ -162,19 +165,17 @@ public DataSegment push(File inDir, DataSegment segment, boolean replaceExisting return dataSegment; } - private void copyFilesWithChecks(final FileSystem fs, final Path from, final Path to, final boolean replaceExisting) - throws IOException + private void copyFilesWithChecks(final FileSystem fs, final Path from, final Path to) throws IOException { - if (!HadoopFsWrapper.rename(fs, from, to, replaceExisting)) { + if (!HadoopFsWrapper.rename(fs, from, to)) { if (fs.exists(to)) { log.info( - "Unable to rename temp Index file[%s] to final segment path [%s]. " - + "It is already pushed by a replica task.", + "Unable to rename temp file [%s] to segment path [%s], it may have already been pushed by a replica task.", from, to ); } else { - throw new IOE("Failed to rename temp Index file[%s] and final segment path[%s] is not present.", from, to); + throw new IOE("Failed to rename temp file [%s] and final segment path [%s] is not present.", from, to); } } } @@ -209,8 +210,17 @@ public Map makeLoadSpec(URI finalIndexZipFilePath) */ @Override - public String getStorageDir(DataSegment segment) + public String getStorageDir(DataSegment segment, boolean useUniquePath) { + // This is only called by HdfsDataSegmentPusher.push(), which will always set useUniquePath to false since any + // 'uniqueness' will be applied not to the directory but to the filename along with the shard number. This is done + // to avoid performance issues due to excessive HDFS directories. Hence useUniquePath is ignored here and we + // expect it to be false. + Preconditions.checkArgument( + !useUniquePath, + "useUniquePath must be false for HdfsDataSegmentPusher.getStorageDir()" + ); + return JOINER.join( segment.getDataSource(), StringUtils.format( @@ -225,9 +235,10 @@ public String getStorageDir(DataSegment segment) @Override public String makeIndexPathName(DataSegment dataSegment, String indexName) { + // This is only called from Hadoop batch which doesn't require unique segment paths so set useUniquePath=false return StringUtils.format( "./%s/%d_%s", - this.getStorageDir(dataSegment), + this.getStorageDir(dataSegment, false), dataSegment.getShardSpec().getPartitionNum(), indexName ); diff --git a/extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java b/extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java index 4604241dd76a..913adb0977bc 100644 --- a/extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java +++ b/extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java @@ -41,16 +41,15 @@ private HadoopFsWrapper() {} * * @param from * @param to - * @param replaceExisting if existing files should be overwritten * - * @return true if operation succeeded, false if replaceExisting == false and destination already exists + * @return true if operation succeeded, false if destination already exists * * @throws IOException if trying to overwrite a non-empty directory */ - public static boolean rename(FileSystem fs, Path from, Path to, boolean replaceExisting) throws IOException + public static boolean rename(FileSystem fs, Path from, Path to) throws IOException { try { - fs.rename(from, to, replaceExisting ? Options.Rename.OVERWRITE : Options.Rename.NONE); + fs.rename(from, to, Options.Rename.NONE); return true; } catch (FileAlreadyExistsException ex) { diff --git a/extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentFinderTest.java b/extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentFinderTest.java index 626124622527..5e052df1e6d1 100644 --- a/extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentFinderTest.java +++ b/extensions-core/hdfs-storage/src/test/java/io/druid/segment/loading/HdfsDataSegmentFinderTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableMap; import io.druid.java.util.common.IOE; import io.druid.java.util.common.Intervals; +import io.druid.java.util.common.StringUtils; import io.druid.segment.TestHelper; import io.druid.storage.hdfs.HdfsDataSegmentFinder; import io.druid.timeline.DataSegment; @@ -278,6 +279,30 @@ public void testFindSegmentsFail() throws Exception hdfsDataSegmentFinder.findSegments(dataSourceDir.toString(), false); } + @Test + public void testPreferNewestSegment() throws Exception + { + dataSourceDir = new Path(new Path(uriBase), "/usr/replicaDataSource"); + descriptor1 = new Path(dataSourceDir, StringUtils.format("interval1/v1/%d_%s_%s", 0, "older", DESCRIPTOR_JSON)); + descriptor2 = new Path(dataSourceDir, StringUtils.format("interval1/v1/%d_%s_%s", 0, "newer", DESCRIPTOR_JSON)); + indexZip1 = new Path(descriptor1.getParent(), StringUtils.format("%d_%s_%s", 0, "older", INDEX_ZIP)); + indexZip2 = new Path(descriptor2.getParent(), StringUtils.format("%d_%s_%s", 0, "newer", INDEX_ZIP)); + + mapper.writeValue(fs.create(descriptor1), SEGMENT_1); + mapper.writeValue(fs.create(descriptor2), SEGMENT_1); + + create(indexZip1); + Thread.sleep(1000); + create(indexZip2); + + final Set segments = new HdfsDataSegmentFinder(conf, mapper).findSegments( + dataSourceDir.toString(), false + ); + + Assert.assertEquals(1, segments.size()); + Assert.assertEquals(indexZip2.toUri().getPath(), segments.iterator().next().getLoadSpec().get("path")); + } + private String getDescriptorPath(DataSegment segment) { final Path indexzip = new Path(String.valueOf(segment.getLoadSpec().get("path"))); diff --git a/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentKillerTest.java b/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentKillerTest.java index bdf499dcd904..dba43507a699 100644 --- a/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentKillerTest.java +++ b/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentKillerTest.java @@ -26,13 +26,13 @@ import io.druid.timeline.DataSegment; import io.druid.timeline.partition.NoneShardSpec; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.junit.Assert; import org.junit.Test; import java.io.IOException; +import java.util.UUID; /** */ @@ -129,7 +129,10 @@ public String getStorageDirectory() Path interval1Dir = new Path(dataSourceDir, "intervalNew"); Path version11Dir = new Path(interval1Dir, "v1"); - makePartitionDirWithIndexWitNewFormat(fs, version11Dir, 3); + Assert.assertTrue(fs.mkdirs(version11Dir)); + fs.createNewFile(new Path(version11Dir, StringUtils.format("%s_index.zip", 3))); + fs.createNewFile(new Path(version11Dir, StringUtils.format("%s_descriptor.json", 3))); + killer.kill(getSegmentWithPath(new Path(version11Dir, "3_index.zip").toString())); Assert.assertFalse(fs.exists(version11Dir)); @@ -141,7 +144,7 @@ public String getStorageDirectory() } @Test - public void testKillNonExistingSegment() throws Exception + public void testKillForSegmentWithUniquePath() throws Exception { Configuration config = new Configuration(); HdfsDataSegmentKiller killer = new HdfsDataSegmentKiller( @@ -155,28 +158,51 @@ public String getStorageDirectory() } } ); - killer.kill(getSegmentWithPath(new Path("/xxx/", "index.zip").toString())); + + FileSystem fs = FileSystem.get(config); + Path dataSourceDir = new Path("/tmp/dataSourceNew"); + + Path interval1Dir = new Path(dataSourceDir, "intervalNew"); + Path version11Dir = new Path(interval1Dir, "v1"); + String uuid = UUID.randomUUID().toString().substring(0, 5); + + Assert.assertTrue(fs.mkdirs(version11Dir)); + fs.createNewFile(new Path(version11Dir, StringUtils.format("%s_%s_index.zip", 3, uuid))); + fs.createNewFile(new Path(version11Dir, StringUtils.format("%s_%s_descriptor.json", 3, uuid))); + + killer.kill(getSegmentWithPath(new Path(version11Dir, StringUtils.format("%s_%s_index.zip", 3, uuid)).toString())); + + Assert.assertFalse(fs.exists(version11Dir)); + Assert.assertFalse(fs.exists(interval1Dir)); + Assert.assertTrue(fs.exists(dataSourceDir)); + Assert.assertTrue(fs.exists(new Path("/tmp"))); + Assert.assertTrue(fs.exists(dataSourceDir)); + Assert.assertTrue(fs.delete(dataSourceDir, false)); } - private void makePartitionDirWithIndex(FileSystem fs, Path path) throws IOException + @Test + public void testKillNonExistingSegment() throws Exception { - Assert.assertTrue(fs.mkdirs(path)); - try (FSDataOutputStream os = fs.create(new Path(path, "index.zip")); FSDataOutputStream oos = fs.create(new Path( - path, - "descriptor.json" - ))) { - } + Configuration config = new Configuration(); + HdfsDataSegmentKiller killer = new HdfsDataSegmentKiller( + config, + new HdfsDataSegmentPusherConfig() + { + @Override + public String getStorageDirectory() + { + return "/tmp"; + } + } + ); + killer.kill(getSegmentWithPath(new Path("/xxx/", "index.zip").toString())); } - private void makePartitionDirWithIndexWitNewFormat(FileSystem fs, Path path, Integer partitionNumber) - throws IOException + private void makePartitionDirWithIndex(FileSystem fs, Path path) throws IOException { Assert.assertTrue(fs.mkdirs(path)); - try (FSDataOutputStream os = fs.create(new Path( - path, - StringUtils.format("%s_index.zip", partitionNumber) - )); FSDataOutputStream oos = fs.create(new Path(path, StringUtils.format("%s_descriptor.json", partitionNumber)))) { - } + fs.createNewFile(new Path(path, "index.zip")); + fs.createNewFile(new Path(path, "descriptor.json")); } private DataSegment getSegmentWithPath(String path) diff --git a/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java b/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java index bfc119f6498a..9bb25f90707f 100644 --- a/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java +++ b/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java @@ -127,7 +127,8 @@ public void testPushWithMultipleSegments() throws Exception testUsingSchemeForMultipleSegments("file", 3); } - private void testUsingScheme(final String scheme) throws Exception + @Test + public void testUsingUniqueFilePath() throws Exception { Configuration conf = new Configuration(true); @@ -142,11 +143,7 @@ private void testUsingScheme(final String scheme) throws Exception HdfsDataSegmentPusherConfig config = new HdfsDataSegmentPusherConfig(); final File storageDirectory = tempFolder.newFolder(); - config.setStorageDirectory( - scheme != null - ? StringUtils.format("%s://%s", scheme, storageDirectory.getAbsolutePath()) - : storageDirectory.getAbsolutePath() - ); + config.setStorageDirectory(StringUtils.format("file://%s", storageDirectory.getAbsolutePath())); HdfsDataSegmentPusher pusher = new HdfsDataSegmentPusher(config, conf, new DefaultObjectMapper()); DataSegment segmentToPush = new DataSegment( @@ -163,49 +160,11 @@ private void testUsingScheme(final String scheme) throws Exception DataSegment segment = pusher.push(segmentDir, segmentToPush, true); - - String indexUri = StringUtils.format( - "%s/%s/%d_index.zip", - FileSystem.newInstance(conf).makeQualified(new Path(config.getStorageDirectory())).toUri().toString(), - pusher.getStorageDir(segmentToPush), - segmentToPush.getShardSpec().getPartitionNum() + String matcher = ".*/foo/20150101T000000\\.000Z_20160101T000000\\.000Z/0/0_[A-Za-z0-9-]{36}_index\\.zip"; + Assert.assertTrue( + segment.getLoadSpec().get("path").toString(), + segment.getLoadSpec().get("path").toString().matches(matcher) ); - - Assert.assertEquals(segmentToPush.getSize(), segment.getSize()); - Assert.assertEquals(segmentToPush, segment); - Assert.assertEquals(ImmutableMap.of( - "type", - "hdfs", - "path", - indexUri - ), segment.getLoadSpec()); - // rename directory after push - final String segmentPath = pusher.getStorageDir(segment); - - File indexFile = new File(StringUtils.format( - "%s/%s/%d_index.zip", - storageDirectory, - segmentPath, - segment.getShardSpec().getPartitionNum() - )); - Assert.assertTrue(indexFile.exists()); - File descriptorFile = new File(StringUtils.format( - "%s/%s/%d_descriptor.json", - storageDirectory, - segmentPath, - segment.getShardSpec().getPartitionNum() - )); - Assert.assertTrue(descriptorFile.exists()); - - // push twice will fail and temp dir cleaned - File outDir = new File(StringUtils.format("%s/%s", config.getStorageDirectory(), segmentPath)); - outDir.setReadOnly(); - try { - pusher.push(segmentDir, segmentToPush, true); - } - catch (IOException e) { - Assert.fail("should not throw exception"); - } } private void testUsingSchemeForMultipleSegments(final String scheme, final int numberOfSegments) throws Exception @@ -246,12 +205,12 @@ private void testUsingSchemeForMultipleSegments(final String scheme, final int n } for (int i = 0; i < numberOfSegments; i++) { - final DataSegment pushedSegment = pusher.push(segmentDir, segments[i], true); + final DataSegment pushedSegment = pusher.push(segmentDir, segments[i], false); String indexUri = StringUtils.format( "%s/%s/%d_index.zip", FileSystem.newInstance(conf).makeQualified(new Path(config.getStorageDirectory())).toUri().toString(), - pusher.getStorageDir(segments[i]), + pusher.getStorageDir(segments[i], false), segments[i].getShardSpec().getPartitionNum() ); @@ -264,7 +223,7 @@ private void testUsingSchemeForMultipleSegments(final String scheme, final int n indexUri ), pushedSegment.getLoadSpec()); // rename directory after push - String segmentPath = pusher.getStorageDir(pushedSegment); + String segmentPath = pusher.getStorageDir(pushedSegment, false); File indexFile = new File(StringUtils.format( "%s/%s/%d_index.zip", @@ -293,7 +252,7 @@ private void testUsingSchemeForMultipleSegments(final String scheme, final int n indexUri ), fromDescriptorFileDataSegment.getLoadSpec()); // rename directory after push - segmentPath = pusher.getStorageDir(fromDescriptorFileDataSegment); + segmentPath = pusher.getStorageDir(fromDescriptorFileDataSegment, false); indexFile = new File(StringUtils.format( "%s/%s/%d_index.zip", @@ -308,7 +267,7 @@ private void testUsingSchemeForMultipleSegments(final String scheme, final int n File outDir = new File(StringUtils.format("%s/%s", config.getStorageDirectory(), segmentPath)); outDir.setReadOnly(); try { - pusher.push(segmentDir, segments[i], true); + pusher.push(segmentDir, segments[i], false); } catch (IOException e) { Assert.fail("should not throw exception"); @@ -316,6 +275,87 @@ private void testUsingSchemeForMultipleSegments(final String scheme, final int n } } + private void testUsingScheme(final String scheme) throws Exception + { + Configuration conf = new Configuration(true); + + // Create a mock segment on disk + File segmentDir = tempFolder.newFolder(); + File tmp = new File(segmentDir, "version.bin"); + + final byte[] data = new byte[]{0x0, 0x0, 0x0, 0x1}; + Files.write(data, tmp); + final long size = data.length; + + HdfsDataSegmentPusherConfig config = new HdfsDataSegmentPusherConfig(); + final File storageDirectory = tempFolder.newFolder(); + + config.setStorageDirectory( + scheme != null + ? StringUtils.format("%s://%s", scheme, storageDirectory.getAbsolutePath()) + : storageDirectory.getAbsolutePath() + ); + HdfsDataSegmentPusher pusher = new HdfsDataSegmentPusher(config, conf, new DefaultObjectMapper()); + + DataSegment segmentToPush = new DataSegment( + "foo", + Intervals.of("2015/2016"), + "0", + Maps.newHashMap(), + Lists.newArrayList(), + Lists.newArrayList(), + NoneShardSpec.instance(), + 0, + size + ); + + DataSegment segment = pusher.push(segmentDir, segmentToPush, false); + + + String indexUri = StringUtils.format( + "%s/%s/%d_index.zip", + FileSystem.newInstance(conf).makeQualified(new Path(config.getStorageDirectory())).toUri().toString(), + pusher.getStorageDir(segmentToPush, false), + segmentToPush.getShardSpec().getPartitionNum() + ); + + Assert.assertEquals(segmentToPush.getSize(), segment.getSize()); + Assert.assertEquals(segmentToPush, segment); + Assert.assertEquals(ImmutableMap.of( + "type", + "hdfs", + "path", + indexUri + ), segment.getLoadSpec()); + // rename directory after push + final String segmentPath = pusher.getStorageDir(segment, false); + + File indexFile = new File(StringUtils.format( + "%s/%s/%d_index.zip", + storageDirectory, + segmentPath, + segment.getShardSpec().getPartitionNum() + )); + Assert.assertTrue(indexFile.exists()); + File descriptorFile = new File(StringUtils.format( + "%s/%s/%d_descriptor.json", + storageDirectory, + segmentPath, + segment.getShardSpec().getPartitionNum() + )); + Assert.assertTrue(descriptorFile.exists()); + + // push twice will fail and temp dir cleaned + File outDir = new File(StringUtils.format("%s/%s", config.getStorageDirectory(), segmentPath)); + outDir.setReadOnly(); + try { + pusher.push(segmentDir, segmentToPush, false); + } + catch (IOException e) { + Assert.fail("should not throw exception"); + } + } + public static class TestObjectMapper extends ObjectMapper { public TestObjectMapper() @@ -371,7 +411,7 @@ public void shouldNotHaveColonsInHdfsStorageDir() throws Exception 1 ); - String storageDir = hdfsDataSegmentPusher.getStorageDir(segment); + String storageDir = hdfsDataSegmentPusher.getStorageDir(segment, false); Assert.assertEquals("something/20111001T000000.000Z_20111002T000000.000Z/brand_new_version", storageDir); } diff --git a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java index 17d742a7589c..d7bffedea17b 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java +++ b/extensions-core/kafka-indexing-service/src/main/java/io/druid/indexing/kafka/KafkaIndexTask.java @@ -1811,6 +1811,7 @@ private StreamAppenderatorDriver newDriver( new ActionBasedSegmentAllocator(toolbox.getTaskActionClient(), dataSchema), toolbox.getSegmentHandoffNotifierFactory(), new ActionBasedUsedSegmentChecker(toolbox.getTaskActionClient()), + toolbox.getDataSegmentKiller(), toolbox.getObjectMapper(), metrics ); diff --git a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java index ad6459f526f6..45f3003638f0 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/io/druid/indexing/kafka/KafkaIndexTaskTest.java @@ -2001,9 +2001,9 @@ private File getSegmentDirectory() private List readSegmentColumn(final String column, final SegmentDescriptor descriptor) throws IOException { - File indexZip = new File( + File indexBasePath = new File( StringUtils.format( - "%s/%s/%s_%s/%s/%d/index.zip", + "%s/%s/%s_%s/%s/%d", getSegmentDirectory(), DATA_SCHEMA.getDataSource(), descriptor.getInterval().getStart(), @@ -2012,6 +2012,7 @@ private List readSegmentColumn(final String column, final SegmentDescrip descriptor.getPartitionNumber() ) ); + File outputLocation = new File( directory, StringUtils.format( @@ -2024,7 +2025,7 @@ private List readSegmentColumn(final String column, final SegmentDescrip ); outputLocation.mkdir(); CompressionUtils.unzip( - Files.asByteSource(indexZip), + Files.asByteSource(new File(indexBasePath.listFiles()[0], "index.zip")), outputLocation, Predicates.alwaysFalse(), false diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentFinder.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentFinder.java index d6d773640e8e..3f4529ec99a0 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentFinder.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentFinder.java @@ -21,9 +21,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; -import com.google.common.collect.Sets; import com.google.inject.Inject; - +import io.druid.java.util.common.Pair; import io.druid.java.util.common.logger.Logger; import io.druid.segment.loading.DataSegmentFinder; import io.druid.segment.loading.SegmentLoadingException; @@ -35,9 +34,11 @@ import java.io.IOException; import java.io.InputStream; +import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; public class S3DataSegmentFinder implements DataSegmentFinder { @@ -62,7 +63,7 @@ public S3DataSegmentFinder( @Override public Set findSegments(String workingDirPath, boolean updateDescriptor) throws SegmentLoadingException { - final Set segments = Sets.newHashSet(); + final Map> timestampedSegments = new HashMap<>(); try { Iterator objectsIterator = S3Utils.storageObjectsIterator( @@ -103,7 +104,12 @@ public Set findSegments(String workingDirPath, boolean updateDescri s3Client.putObject(config.getBucket(), newDescJsonObject); } } - segments.add(dataSegment); + + DataSegmentFinder.putInMapRetainingNewest( + timestampedSegments, + dataSegment, + indexObject.getLastModifiedDate() == null ? 0 : indexObject.getLastModifiedDate().getTime() + ); } } else { throw new SegmentLoadingException( @@ -124,6 +130,6 @@ public Set findSegments(String workingDirPath, boolean updateDescri Throwables.propagateIfInstanceOf(e, SegmentLoadingException.class); Throwables.propagate(e); } - return segments; + return timestampedSegments.values().stream().map(x -> x.lhs).collect(Collectors.toSet()); } } diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java index 983837865e8b..643b30407d9f 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentMover.java @@ -72,8 +72,11 @@ public DataSegment move(DataSegment segment, Map targetLoadSpec) final String targetS3Bucket = MapUtils.getString(targetLoadSpec, "bucket"); final String targetS3BaseKey = MapUtils.getString(targetLoadSpec, "baseKey"); - final String targetS3Path = S3Utils.constructSegmentPath(targetS3BaseKey, DataSegmentPusher.getDefaultStorageDir(segment)); - String targetS3DescriptorPath = S3Utils.descriptorPathForSegmentPath(targetS3Path); + final String targetS3Path = S3Utils.constructSegmentPath( + targetS3BaseKey, + DataSegmentPusher.getDefaultStorageDir(segment, false) + ); + final String targetS3DescriptorPath = S3Utils.descriptorPathForSegmentPath(targetS3Path); if (targetS3Bucket.isEmpty()) { throw new SegmentLoadingException("Target S3 bucket is not specified"); diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java index 3473cffe4a80..eb5c4a9c10dd 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3DataSegmentPusher.java @@ -24,9 +24,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; -import io.druid.java.util.emitter.EmittingLogger; import io.druid.java.util.common.CompressionUtils; import io.druid.java.util.common.StringUtils; +import io.druid.java.util.emitter.EmittingLogger; import io.druid.segment.SegmentUtils; import io.druid.segment.loading.DataSegmentPusher; import io.druid.timeline.DataSegment; @@ -88,10 +88,10 @@ public List getAllowedPropertyPrefixesForHadoop() } @Override - public DataSegment push(final File indexFilesDir, final DataSegment inSegment, final boolean replaceExisting) + public DataSegment push(final File indexFilesDir, final DataSegment inSegment, final boolean useUniquePath) throws IOException { - final String s3Path = S3Utils.constructSegmentPath(config.getBaseKey(), getStorageDir(inSegment)); + final String s3Path = S3Utils.constructSegmentPath(config.getBaseKey(), getStorageDir(inSegment, useUniquePath)); log.info("Copying segment[%s] to S3 at location[%s]", inSegment.getIdentifier(), s3Path); @@ -106,7 +106,7 @@ public DataSegment push(final File indexFilesDir, final DataSegment inSegment, f public DataSegment call() throws Exception { S3Object toPush = new S3Object(zipOutFile); - putObject(config.getBucket(), s3Path, toPush, replaceExisting); + putObject(config.getBucket(), s3Path, toPush); final DataSegment outSegment = inSegment.withSize(indexSize) .withLoadSpec(makeLoadSpec(config.getBucket(), toPush.getKey())) @@ -121,8 +121,7 @@ public DataSegment call() throws Exception putObject( config.getBucket(), S3Utils.descriptorPathForSegmentPath(s3Path), - descriptorObject, - replaceExisting + descriptorObject ); log.info("Deleting zipped index File[%s]", zipOutFile); @@ -169,8 +168,7 @@ private Map makeLoadSpec(String bucket, String key) ); } - private void putObject(String bucketName, String path, S3Object object, boolean replaceExisting) - throws ServiceException + private void putObject(String bucketName, String path, S3Object object) throws ServiceException { object.setBucketName(bucketName); object.setKey(path); @@ -180,10 +178,6 @@ private void putObject(String bucketName, String path, S3Object object, boolean log.info("Pushing %s.", object); - if (!replaceExisting && S3Utils.isObjectInBucket(s3Client, bucketName, object.getKey())) { - log.info("Skipping push because key [%s] exists && replaceExisting == false", object.getKey()); - } else { - s3Client.putObject(bucketName, object); - } + s3Client.putObject(bucketName, object); } } diff --git a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentFinderTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentFinderTest.java index f0df427ccb29..55ad817003d2 100644 --- a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentFinderTest.java +++ b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentFinderTest.java @@ -124,7 +124,6 @@ public class S3DataSegmentFinderTest private String indexZip4_1; - @BeforeClass public static void setUpStatic() { @@ -210,31 +209,51 @@ public void testFindSegments() throws Exception final String serializedSegment4_0 = mapper.writeValueAsString(updatedSegment4_0); final String serializedSegment4_1 = mapper.writeValueAsString(updatedSegment4_1); - Assert.assertNotEquals(serializedSegment1, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor1).getDataInputStream())); - Assert.assertNotEquals(serializedSegment2, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor2).getDataInputStream())); - Assert.assertNotEquals(serializedSegment3, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor3).getDataInputStream())); - Assert.assertNotEquals(serializedSegment4_0, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor4_0).getDataInputStream())); - Assert.assertNotEquals(serializedSegment4_1, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor4_1).getDataInputStream())); + Assert.assertNotEquals( + serializedSegment1, + IOUtils.toString(mockS3Client.getObject(bucket, descriptor1).getDataInputStream()) + ); + Assert.assertNotEquals( + serializedSegment2, + IOUtils.toString(mockS3Client.getObject(bucket, descriptor2).getDataInputStream()) + ); + Assert.assertNotEquals( + serializedSegment3, + IOUtils.toString(mockS3Client.getObject(bucket, descriptor3).getDataInputStream()) + ); + Assert.assertNotEquals( + serializedSegment4_0, + IOUtils.toString(mockS3Client.getObject(bucket, descriptor4_0).getDataInputStream()) + ); + Assert.assertNotEquals( + serializedSegment4_1, + IOUtils.toString(mockS3Client.getObject(bucket, descriptor4_1).getDataInputStream()) + ); final Set segments2 = s3DataSegmentFinder.findSegments("", true); Assert.assertEquals(segments, segments2); - Assert.assertEquals(serializedSegment1, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor1).getDataInputStream())); - Assert.assertEquals(serializedSegment2, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor2).getDataInputStream())); - Assert.assertEquals(serializedSegment3, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor3).getDataInputStream())); - Assert.assertEquals(serializedSegment4_0, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor4_0).getDataInputStream())); - Assert.assertEquals(serializedSegment4_1, - IOUtils.toString(mockS3Client.getObject(bucket, descriptor4_1).getDataInputStream())); + Assert.assertEquals( + serializedSegment1, + IOUtils.toString(mockS3Client.getObject(bucket, descriptor1).getDataInputStream()) + ); + Assert.assertEquals( + serializedSegment2, + IOUtils.toString(mockS3Client.getObject(bucket, descriptor2).getDataInputStream()) + ); + Assert.assertEquals( + serializedSegment3, + IOUtils.toString(mockS3Client.getObject(bucket, descriptor3).getDataInputStream()) + ); + Assert.assertEquals( + serializedSegment4_0, + IOUtils.toString(mockS3Client.getObject(bucket, descriptor4_0).getDataInputStream()) + ); + Assert.assertEquals( + serializedSegment4_1, + IOUtils.toString(mockS3Client.getObject(bucket, descriptor4_1).getDataInputStream()) + ); } @Test(expected = SegmentLoadingException.class) @@ -268,9 +287,7 @@ public void testFindSegmentsWithworkingDirPath() throws SegmentLoadingException public void testFindSegmentsUpdateLoadSpec() throws Exception { config.setBucket("amazing"); - final DataSegment segmentMissingLoadSpec = DataSegment.builder(SEGMENT_1) - .loadSpec(ImmutableMap.of()) - .build(); + final DataSegment segmentMissingLoadSpec = DataSegment.builder(SEGMENT_1).loadSpec(ImmutableMap.of()).build(); final S3DataSegmentFinder s3DataSegmentFinder = new S3DataSegmentFinder(mockS3Client, config, mapper); final String segmentPath = baseKey + "/interval_missing_load_spec/v1/1/"; final String descriptorPath = S3Utils.descriptorPathForSegmentPath(segmentPath); @@ -304,6 +321,32 @@ public void testFindSegmentsUpdateLoadSpec() throws Exception Assert.assertEquals(indexPath, testLoadSpec.get("key")); } + @Test + public void testPreferNewestSegment() throws Exception + { + baseKey = "replicaDataSource"; + + config = new S3DataSegmentPusherConfig(); + config.setBucket(bucket); + config.setBaseKey(baseKey); + + descriptor1 = S3Utils.descriptorPathForSegmentPath(baseKey + "/interval10/v1/0/older/"); + descriptor2 = S3Utils.descriptorPathForSegmentPath(baseKey + "/interval10/v1/0/newer/"); + + indexZip1 = S3Utils.indexZipForSegmentPath(descriptor1); + indexZip2 = S3Utils.indexZipForSegmentPath(descriptor2); + + mockS3Client.putObject(bucket, new S3Object(descriptor1, mapper.writeValueAsString(SEGMENT_1))); + mockS3Client.putObject(bucket, new S3Object(indexZip1, "dummy")); + mockS3Client.putObject(bucket, new S3Object(descriptor2, mapper.writeValueAsString(SEGMENT_1))); + mockS3Client.putObject(bucket, new S3Object(indexZip2, "dummy")); + + final S3DataSegmentFinder s3DataSegmentFinder = new S3DataSegmentFinder(mockS3Client, config, mapper); + final Set segments = s3DataSegmentFinder.findSegments("", false); + + Assert.assertEquals(1, segments.size()); + } + private String getDescriptorPath(DataSegment segment) { return S3Utils.descriptorPathForSegmentPath(String.valueOf(segment.getLoadSpec().get("key"))); diff --git a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java index f26bd1610398..841f10e1c663 100644 --- a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java +++ b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java @@ -64,6 +64,20 @@ public void setValue(T value) @Test public void testPush() throws Exception + { + testPushInternal(false, "key/foo/2015-01-01T00:00:00\\.000Z_2016-01-01T00:00:00\\.000Z/0/0/index\\.zip"); + } + + @Test + public void testPushUseUniquePath() throws Exception + { + testPushInternal( + true, + "key/foo/2015-01-01T00:00:00\\.000Z_2016-01-01T00:00:00\\.000Z/0/0/[A-Za-z0-9-]{36}/index\\.zip" + ); + } + + private void testPushInternal(boolean useUniquePath, String matcher) throws Exception { RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class); @@ -113,14 +127,15 @@ public S3Object answer() throws Throwable size ); - DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush, true); + DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush, useUniquePath); Assert.assertEquals(segmentToPush.getSize(), segment.getSize()); Assert.assertEquals(1, (int) segment.getBinaryVersion()); Assert.assertEquals("bucket", segment.getLoadSpec().get("bucket")); - Assert.assertEquals( - "key/foo/2015-01-01T00:00:00.000Z_2016-01-01T00:00:00.000Z/0/0/index.zip", - segment.getLoadSpec().get("key")); + Assert.assertTrue( + segment.getLoadSpec().get("key").toString(), + segment.getLoadSpec().get("key").toString().matches(matcher) + ); Assert.assertEquals("s3_zip", segment.getLoadSpec().get("type")); // Verify that the pushed S3Object contains the correct data diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java b/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java index 7ce678045c7f..8975c01d86a0 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/JobHelper.java @@ -91,6 +91,7 @@ public static Path distributedClassPath(Path base) { return new Path(base, "classpath"); } + public static final String INDEX_ZIP = "index.zip"; public static final String DESCRIPTOR_JSON = "descriptor.json"; @@ -568,8 +569,10 @@ public static Path makeFileNamePath( DataSegmentPusher dataSegmentPusher ) { - return new Path(prependFSIfNullScheme(fs, basePath), - dataSegmentPusher.makeIndexPathName(segmentTemplate, baseFileName)); + return new Path( + prependFSIfNullScheme(fs, basePath), + dataSegmentPusher.makeIndexPathName(segmentTemplate, baseFileName) + ); } public static Path makeTmpPath( @@ -582,9 +585,10 @@ public static Path makeTmpPath( { return new Path( prependFSIfNullScheme(fs, basePath), - StringUtils.format("./%s.%d", - dataSegmentPusher.makeIndexPathName(segmentTemplate, JobHelper.INDEX_ZIP), - taskAttemptID.getId() + StringUtils.format( + "./%s.%d", + dataSegmentPusher.makeIndexPathName(segmentTemplate, JobHelper.INDEX_ZIP), + taskAttemptID.getId() ) ); } diff --git a/indexing-service/src/main/java/io/druid/indexing/common/index/YeOldePlumberSchool.java b/indexing-service/src/main/java/io/druid/indexing/common/index/YeOldePlumberSchool.java index 0bba29fbe881..4364f50f40a8 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/index/YeOldePlumberSchool.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/index/YeOldePlumberSchool.java @@ -199,10 +199,7 @@ public void finishJob() .withDimensions(ImmutableList.copyOf(mappedSegment.getAvailableDimensions())) .withBinaryVersion(SegmentUtils.getVersionFromDir(fileToUpload)); - // This plumber is only used in batch ingestion situations where you do not have replica tasks pushing - // segments with the same identifier but potentially different contents. In case of conflict, favor the most - // recently pushed segment (replaceExisting == true). - dataSegmentPusher.push(fileToUpload, segmentToUpload, true); + dataSegmentPusher.push(fileToUpload, segmentToUpload, false); log.info( "Uploaded segment[%s]", diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/ConvertSegmentTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/ConvertSegmentTask.java index 0e752cb988ce..460225af1fe0 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/ConvertSegmentTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/ConvertSegmentTask.java @@ -441,10 +441,7 @@ private void convertSegment(TaskToolbox toolbox) throws SegmentLoadingException, // version, but is "newer" than said original version. DataSegment updatedSegment = segment.withVersion(StringUtils.format("%s_v%s", segment.getVersion(), outVersion)); - // The convert segment task does not support replicas where different tasks could generate segments with the - // same identifier but potentially different contents. In case of conflict, favor the most recently pushed - // segment (replaceExisting == true). - updatedSegment = toolbox.getSegmentPusher().push(outLocation, updatedSegment, true); + updatedSegment = toolbox.getSegmentPusher().push(outLocation, updatedSegment, false); actionClient.submit(new SegmentInsertAction(Sets.newHashSet(updatedSegment))); } else { diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java index f0987b21ea8c..1d88733f9522 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/IndexTask.java @@ -792,7 +792,8 @@ private static BatchAppenderatorDriver newDriver( return new BatchAppenderatorDriver( appenderator, segmentAllocator, - new ActionBasedUsedSegmentChecker(toolbox.getTaskActionClient()) + new ActionBasedUsedSegmentChecker(toolbox.getTaskActionClient()), + toolbox.getDataSegmentKiller() ); } diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java b/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java index 747cf58d4162..454899f5c774 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/MergeTaskBase.java @@ -186,10 +186,7 @@ public String apply(DataSegment input) // Upload file - // The merge task does not support replicas where different tasks could generate segments with the - // same identifier but potentially different contents. In case of conflict, favor the most recently pushed - // segment (replaceExisting == true). - final DataSegment uploadedSegment = toolbox.getSegmentPusher().push(fileToUpload, mergedSegment, true); + final DataSegment uploadedSegment = toolbox.getSegmentPusher().push(fileToUpload, mergedSegment, false); emitter.emit(builder.build("merger/uploadTime", System.currentTimeMillis() - uploadStart)); emitter.emit(builder.build("merger/mergeSize", uploadedSegment.getSize())); diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java index f813220e25d2..7113efa6aa50 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java @@ -60,6 +60,7 @@ import io.druid.segment.indexing.granularity.ArbitraryGranularitySpec; import io.druid.segment.indexing.granularity.GranularitySpec; import io.druid.segment.indexing.granularity.UniformGranularitySpec; +import io.druid.segment.loading.DataSegmentKiller; import io.druid.segment.loading.DataSegmentPusher; import io.druid.segment.realtime.appenderator.SegmentIdentifier; import io.druid.segment.realtime.firehose.LocalFirehoseFactory; @@ -1006,7 +1007,7 @@ public String getPathForHadoop() } @Override - public DataSegment push(File file, DataSegment segment, boolean replaceExisting) throws IOException + public DataSegment push(File file, DataSegment segment, boolean useUniquePath) { segments.add(segment); return segment; @@ -1019,12 +1020,27 @@ public Map makeLoadSpec(URI uri) } }; + final DataSegmentKiller killer = new DataSegmentKiller() + { + @Override + public void kill(DataSegment segment) + { + + } + + @Override + public void killAll() + { + + } + }; + final TaskToolbox box = new TaskToolbox( null, actionClient, null, pusher, - null, + killer, null, null, null, diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/SameIntervalMergeTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/SameIntervalMergeTaskTest.java index e42f55d9f605..50932dbb2311 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/SameIntervalMergeTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/SameIntervalMergeTaskTest.java @@ -185,7 +185,8 @@ public RetType submit(TaskAction taskAction) throws IOExcepti return null; } }, - new NoopServiceEmitter(), new DataSegmentPusher() + new NoopServiceEmitter(), + new DataSegmentPusher() { @Deprecated @Override @@ -201,12 +202,13 @@ public String getPathForHadoop() } @Override - public DataSegment push(File file, DataSegment segment, boolean replaceExisting) throws IOException + public DataSegment push(File file, DataSegment segment, boolean useUniquePath) { // the merged segment is pushed to storage segments.add(segment); return segment; } + @Override public Map makeLoadSpec(URI finalIndexZipFilePath) { diff --git a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java index 13f2d4f1e88a..9767ebf513ac 100644 --- a/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java @@ -31,7 +31,6 @@ import com.google.common.io.Files; import com.google.inject.Binder; import com.google.inject.Module; -import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.data.input.InputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.InputRowParser; @@ -61,6 +60,7 @@ import io.druid.java.util.common.JodaUtils; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.logger.Logger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.math.expr.ExprMacroTable; import io.druid.metadata.IndexerSQLMetadataStorageCoordinator; import io.druid.query.aggregation.DoubleSumAggregatorFactory; @@ -184,7 +184,8 @@ public List getUsedSegmentsForInterval(String dataSource, Interval } @Override - public List getUsedSegmentsForIntervals(String dataSource, List interval) throws IOException + public List getUsedSegmentsForIntervals(String dataSource, List interval) + throws IOException { return ImmutableList.copyOf(segmentSet); } @@ -249,7 +250,7 @@ public String getPathForHadoop() } @Override - public DataSegment push(File file, DataSegment segment, boolean replaceExisting) throws IOException + public DataSegment push(File file, DataSegment segment, boolean useUniquePath) { return segment; } @@ -537,7 +538,11 @@ public void simpleFirehoseReadingTest() throws IOException Assert.assertArrayEquals(new String[]{DIM_NAME}, row.getDimensions().toArray()); Assert.assertArrayEquals(new String[]{DIM_VALUE}, row.getDimension(DIM_NAME).toArray()); Assert.assertEquals(METRIC_LONG_VALUE.longValue(), row.getMetric(METRIC_LONG_NAME)); - Assert.assertEquals(METRIC_FLOAT_VALUE, row.getMetric(METRIC_FLOAT_NAME).floatValue(), METRIC_FLOAT_VALUE * 0.0001); + Assert.assertEquals( + METRIC_FLOAT_VALUE, + row.getMetric(METRIC_FLOAT_NAME).floatValue(), + METRIC_FLOAT_VALUE * 0.0001 + ); ++rowcount; } } diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java index 6ce005867133..e583a85b2671 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java @@ -34,10 +34,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Ordering; import com.google.common.util.concurrent.MoreExecutors; -import io.druid.java.util.emitter.EmittingLogger; -import io.druid.java.util.emitter.service.ServiceEmitter; -import io.druid.java.util.metrics.Monitor; -import io.druid.java.util.metrics.MonitorScheduler; import io.druid.client.cache.MapCache; import io.druid.data.input.Firehose; import io.druid.data.input.FirehoseFactory; @@ -82,6 +78,10 @@ import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.guava.Comparators; +import io.druid.java.util.emitter.EmittingLogger; +import io.druid.java.util.emitter.service.ServiceEmitter; +import io.druid.java.util.metrics.Monitor; +import io.druid.java.util.metrics.MonitorScheduler; import io.druid.metadata.DerbyMetadataStorageActionHandlerFactory; import io.druid.metadata.TestDerbyConnector; import io.druid.query.QueryRunnerFactoryConglomerate; @@ -485,7 +485,7 @@ public String getPathForHadoop(String dataSource) } @Override - public DataSegment push(File file, DataSegment segment, boolean replaceExisting) throws IOException + public DataSegment push(File file, DataSegment segment, boolean useUniquePath) { pushedSegments++; return segment; @@ -527,8 +527,11 @@ private TaskToolboxFactory setUpTaskToolboxFactory( Preconditions.checkNotNull(emitter); taskLockbox = new TaskLockbox(taskStorage); - tac = new LocalTaskActionClientFactory(taskStorage, new TaskActionToolbox(taskLockbox, mdc, emitter, EasyMock.createMock( - SupervisorManager.class))); + tac = new LocalTaskActionClientFactory( + taskStorage, + new TaskActionToolbox(taskLockbox, mdc, emitter, EasyMock.createMock( + SupervisorManager.class)) + ); File tmpDir = temporaryFolder.newFolder(); taskConfig = new TaskConfig(tmpDir.toString(), null, null, 50000, null, false, null, null); @@ -1034,7 +1037,7 @@ public String getPathForHadoop() } @Override - public DataSegment push(File file, DataSegment dataSegment, boolean replaceExisting) throws IOException + public DataSegment push(File file, DataSegment dataSegment, boolean useUniquePath) { throw new RuntimeException("FAILURE"); } @@ -1094,7 +1097,22 @@ public void testResumeTasks() throws Exception mapper ), new IndexIOConfig(new MockFirehoseFactory(false), false), - new IndexTuningConfig(10000, 10, null, null, null, indexSpec, null, false, null, null, null, null, null, null) + new IndexTuningConfig( + 10000, + 10, + null, + null, + null, + indexSpec, + null, + false, + null, + null, + null, + null, + null, + null + ) ), null ); diff --git a/indexing-service/src/test/java/io/druid/indexing/test/TestDataSegmentPusher.java b/indexing-service/src/test/java/io/druid/indexing/test/TestDataSegmentPusher.java index d518931a5faa..b9b27f48d07a 100644 --- a/indexing-service/src/test/java/io/druid/indexing/test/TestDataSegmentPusher.java +++ b/indexing-service/src/test/java/io/druid/indexing/test/TestDataSegmentPusher.java @@ -25,7 +25,6 @@ import io.druid.timeline.DataSegment; import java.io.File; -import java.io.IOException; import java.net.URI; import java.util.Map; import java.util.Set; @@ -48,7 +47,7 @@ public String getPathForHadoop() } @Override - public DataSegment push(File file, DataSegment segment, boolean replaceExisting) throws IOException + public DataSegment push(File file, DataSegment segment, boolean useUniquePath) { pushedSegments.add(segment); return segment; diff --git a/server/src/main/java/io/druid/segment/loading/LocalDataSegmentFinder.java b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentFinder.java index 018dcce3e281..177461c39f5f 100644 --- a/server/src/main/java/io/druid/segment/loading/LocalDataSegmentFinder.java +++ b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentFinder.java @@ -20,24 +20,24 @@ package io.druid.segment.loading; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Sets; import com.google.inject.Inject; - import io.druid.guice.LocalDataStorageDruidModule; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.logger.Logger; import io.druid.timeline.DataSegment; import org.apache.commons.io.FileUtils; import java.io.File; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** */ public class LocalDataSegmentFinder implements DataSegmentFinder { - private static final Logger log = new Logger(LocalDataSegmentFinder.class); private final ObjectMapper mapper; @@ -49,25 +49,26 @@ public LocalDataSegmentFinder(ObjectMapper mapper) } @Override - public Set findSegments(String workingDirPath, boolean updateDescriptor) - throws SegmentLoadingException + public Set findSegments(String workingDirPath, boolean updateDescriptor) throws SegmentLoadingException { + final Map> timestampedSegments = new HashMap<>(); - final Set segments = Sets.newHashSet(); final File workingDir = new File(workingDirPath); if (!workingDir.isDirectory()) { throw new SegmentLoadingException("Working directory [%s] didn't exist !?", workingDir); } - recursiveSearchSegments(segments, workingDir, updateDescriptor); - return segments; + recursiveSearchSegments(timestampedSegments, workingDir, updateDescriptor); + + return timestampedSegments.values().stream().map(x -> x.lhs).collect(Collectors.toSet()); } - private void recursiveSearchSegments(Set segments, File workingDir, boolean updateDescriptor) - throws SegmentLoadingException + private void recursiveSearchSegments( + Map> timestampedSegments, File workingDir, boolean updateDescriptor + ) throws SegmentLoadingException { for (File file : workingDir.listFiles()) { if (file.isDirectory()) { - recursiveSearchSegments(segments, file, updateDescriptor); + recursiveSearchSegments(timestampedSegments, file, updateDescriptor); } else if (file.getName().equals("descriptor.json")) { final File indexZip = new File(file.getParentFile(), "index.zip"); if (indexZip.exists()) { @@ -88,7 +89,8 @@ private void recursiveSearchSegments(Set segments, File workingDir, FileUtils.writeStringToFile(file, mapper.writeValueAsString(dataSegment)); } } - segments.add(dataSegment); + + DataSegmentFinder.putInMapRetainingNewest(timestampedSegments, dataSegment, indexZip.lastModified()); } catch (IOException e) { throw new SegmentLoadingException( diff --git a/server/src/main/java/io/druid/segment/loading/LocalDataSegmentKiller.java b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentKiller.java index 8c8715d1f9d9..770c89a4a023 100644 --- a/server/src/main/java/io/druid/segment/loading/LocalDataSegmentKiller.java +++ b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentKiller.java @@ -52,19 +52,21 @@ public void kill(DataSegment segment) throws SegmentLoadingException try { if (path.getName().endsWith(".zip")) { - // path format -- > .../dataSource/interval/version/partitionNum/xxx.zip - File partitionNumDir = path.getParentFile(); - FileUtils.deleteDirectory(partitionNumDir); + // or .../dataSource/interval/version/partitionNum/UUID/xxx.zip + + File parentDir = path.getParentFile(); + FileUtils.deleteDirectory(parentDir); - //try to delete other directories if possible - File versionDir = partitionNumDir.getParentFile(); - if (versionDir.delete()) { - File intervalDir = versionDir.getParentFile(); - if (intervalDir.delete()) { - File dataSourceDir = intervalDir.getParentFile(); - dataSourceDir.delete(); + // possibly recursively delete empty parent directories up to 'dataSource' + parentDir = parentDir.getParentFile(); + int maxDepth = 4; // if for some reason there's no datasSource directory, stop recursing somewhere reasonable + while (parentDir != null && --maxDepth >= 0) { + if (!parentDir.delete() || segment.getDataSource().equals(parentDir.getName())) { + break; } + + parentDir = parentDir.getParentFile(); } } else { throw new SegmentLoadingException("Unknown file type[%s]", path); diff --git a/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java index f60228dfeb3e..830af44be3d9 100644 --- a/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java +++ b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java @@ -32,7 +32,6 @@ import java.io.File; import java.io.IOException; import java.net.URI; -import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.StandardOpenOption; import java.util.Map; @@ -71,10 +70,11 @@ public String getPathForHadoop(String dataSource) } @Override - public DataSegment push(File dataSegmentFile, DataSegment segment, boolean replaceExisting) throws IOException + public DataSegment push(final File dataSegmentFile, final DataSegment segment, final boolean useUniquePath) + throws IOException { final File baseStorageDir = config.getStorageDirectory(); - final File outDir = new File(baseStorageDir, this.getStorageDir(segment)); + final File outDir = new File(baseStorageDir, this.getStorageDir(segment, useUniquePath)); log.info("Copying segment[%s] to local filesystem at location[%s]", segment.getIdentifier(), outDir.toString()); @@ -109,31 +109,15 @@ public DataSegment push(File dataSegmentFile, DataSegment segment, boolean repla ); FileUtils.forceMkdir(outDir); - if (replaceExisting) { - final File indexFileTarget = new File(outDir, tmpIndexFile.getName()); - final File descriptorFileTarget = new File(outDir, tmpDescriptorFile.getName()); - - if (!tmpIndexFile.renameTo(indexFileTarget)) { - throw new IOE("Failed to rename [%s] to [%s]", tmpIndexFile, indexFileTarget); - } - - if (!tmpDescriptorFile.renameTo(descriptorFileTarget)) { - throw new IOE("Failed to rename [%s] to [%s]", tmpDescriptorFile, descriptorFileTarget); - } - } else { - try { - Files.move(tmpIndexFile.toPath(), outDir.toPath().resolve(tmpIndexFile.toPath().getFileName())); - } - catch (FileAlreadyExistsException e) { - log.info("[%s] already exists at [%s], ignore if replication is configured", INDEX_FILENAME, outDir); - } - try { - Files.move(tmpDescriptorFile.toPath(), outDir.toPath().resolve(tmpDescriptorFile.toPath().getFileName())); - } - catch (FileAlreadyExistsException e) { - log.info("[%s] already exists at [%s], ignore if replication is configured", DESCRIPTOR_FILENAME, outDir); - dataSegment = jsonMapper.readValue(new File(outDir, DESCRIPTOR_FILENAME), DataSegment.class); - } + final File indexFileTarget = new File(outDir, tmpIndexFile.getName()); + final File descriptorFileTarget = new File(outDir, tmpDescriptorFile.getName()); + + if (!tmpIndexFile.renameTo(indexFileTarget)) { + throw new IOE("Failed to rename [%s] to [%s]", tmpIndexFile, indexFileTarget); + } + + if (!tmpDescriptorFile.renameTo(descriptorFileTarget)) { + throw new IOE("Failed to rename [%s] to [%s]", tmpDescriptorFile, descriptorFileTarget); } return dataSegment; diff --git a/server/src/main/java/io/druid/segment/loading/SegmentLoaderLocalCacheManager.java b/server/src/main/java/io/druid/segment/loading/SegmentLoaderLocalCacheManager.java index 83c95f595c7e..75377a463ec0 100644 --- a/server/src/main/java/io/druid/segment/loading/SegmentLoaderLocalCacheManager.java +++ b/server/src/main/java/io/druid/segment/loading/SegmentLoaderLocalCacheManager.java @@ -91,10 +91,10 @@ public boolean isSegmentLoaded(final DataSegment segment) return findStorageLocationIfLoaded(segment) != null; } - public StorageLocation findStorageLocationIfLoaded(final DataSegment segment) + private StorageLocation findStorageLocationIfLoaded(final DataSegment segment) { for (StorageLocation location : getSortedList(locations)) { - File localStorageDir = new File(location.getPath(), DataSegmentPusher.getDefaultStorageDir(segment)); + File localStorageDir = new File(location.getPath(), DataSegmentPusher.getDefaultStorageDir(segment, false)); if (localStorageDir.exists()) { return location; } @@ -127,7 +127,7 @@ public Segment getSegment(DataSegment segment) throws SegmentLoadingException public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException { StorageLocation loc = findStorageLocationIfLoaded(segment); - String storageDir = DataSegmentPusher.getDefaultStorageDir(segment); + String storageDir = DataSegmentPusher.getDefaultStorageDir(segment, false); if (loc == null) { loc = loadSegmentWithRetry(segment, storageDir); @@ -232,11 +232,11 @@ public void cleanup(DataSegment segment) throws SegmentLoadingException // in this case, findStorageLocationIfLoaded() will think segment is located in the failed storageDir which is actually not. // So we should always clean all possible locations here for (StorageLocation location : getSortedList(locations)) { - File localStorageDir = new File(location.getPath(), DataSegmentPusher.getDefaultStorageDir(segment)); + File localStorageDir = new File(location.getPath(), DataSegmentPusher.getDefaultStorageDir(segment, false)); if (localStorageDir.exists()) { // Druid creates folders of the form dataSource/interval/version/partitionNum. // We need to clean up all these directories if they are all empty. - File cacheFile = new File(location.getPath(), DataSegmentPusher.getDefaultStorageDir(segment)); + File cacheFile = new File(location.getPath(), DataSegmentPusher.getDefaultStorageDir(segment, false)); cleanupCacheFiles(location.getPath(), cacheFile); location.removeSegment(segment); } @@ -272,7 +272,7 @@ public void cleanupCacheFiles(File baseFile, File cacheFile) throws IOException } } - public List getSortedList(List locs) + private List getSortedList(List locs) { List locations = new ArrayList<>(locs); Collections.sort(locations, COMPARATOR); diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderator.java b/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderator.java index dbd1ed831378..51708f4f31cd 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderator.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderator.java @@ -194,11 +194,16 @@ default ListenableFuture persistAll(@Nullable Committer committer) * * @param identifiers list of segments to push * @param committer a committer associated with all data that has been added so far + * @param useUniquePath true if the segment should be written to a path with a unique identifier * * @return future that resolves when all segments have been pushed. The segment list will be the list of segments * that have been pushed and the commit metadata from the Committer. */ - ListenableFuture push(Collection identifiers, @Nullable Committer committer); + ListenableFuture push( + Collection identifiers, + @Nullable Committer committer, + boolean useUniquePath + ); /** * Stop any currently-running processing and clean up after ourselves. This allows currently running persists and pushes diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java index 2f4a851bc57e..3d872e25e6af 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java @@ -506,7 +506,8 @@ public ListenableFuture persistAll(@Nullable final Committer committer) @Override public ListenableFuture push( final Collection identifiers, - @Nullable final Committer committer + @Nullable final Committer committer, + final boolean useUniquePath ) { final Map theSinks = Maps.newHashMap(); @@ -530,7 +531,7 @@ public ListenableFuture push( continue; } - final DataSegment dataSegment = mergeAndPush(entry.getKey(), entry.getValue()); + final DataSegment dataSegment = mergeAndPush(entry.getKey(), entry.getValue(), useUniquePath); if (dataSegment != null) { dataSegments.add(dataSegment); } else { @@ -560,17 +561,13 @@ private ListenableFuture pushBarrier() * Merge segment, push to deep storage. Should only be used on segments that have been fully persisted. Must only * be run in the single-threaded pushExecutor. * - * Note that this calls DataSegmentPusher.push() with replaceExisting == true which is appropriate for the indexing - * tasks it is currently being used for (local indexing and Kafka indexing). If this is going to be used by an - * indexing task type that requires replaceExisting == false, this setting will need to be pushed to the caller. - * * @param identifier sink identifier * @param sink sink to push + * @param useUniquePath true if the segment should be written to a path with a unique identifier * * @return segment descriptor, or null if the sink is no longer valid */ - - private DataSegment mergeAndPush(final SegmentIdentifier identifier, final Sink sink) + private DataSegment mergeAndPush(final SegmentIdentifier identifier, final Sink sink, final boolean useUniquePath) { // Bail out if this sink is null or otherwise not what we expect. if (sinks.get(identifier) != sink) { @@ -642,17 +639,12 @@ private DataSegment mergeAndPush(final SegmentIdentifier identifier, final Sink // Retry pushing segments because uploading to deep storage might fail especially for cloud storage types final DataSegment segment = RetryUtils.retry( // The appenderator is currently being used for the local indexing task and the Kafka indexing task. For the - // Kafka indexing task, pushers MUST overwrite any existing objects in deep storage with the same identifier - // in order to maintain exactly-once semantics. If they do not and instead favor existing objects, in case of - // failure during publishing, the indexed data may not represent the checkpointed state and data loss or - // duplication may occur. See: https://github.com/druid-io/druid/issues/5161. The local indexing task does not - // support replicas where different tasks could generate segments with the same identifier but potentially - // different contents so it is okay if existing objects are overwritten. In both of these cases, we want to - // favor the most recently pushed segment so replaceExisting == true. + // Kafka indexing task, pushers must use unique file paths in deep storage in order to maintain exactly-once + // semantics. () -> dataSegmentPusher.push( mergedFile, sink.getSegment().withDimensions(IndexMerger.getMergedDimensionsFromQueryableIndexes(indexes)), - true + useUniquePath ), exception -> exception instanceof Exception, 5 diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorPlumber.java b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorPlumber.java index 2edfa4ed6cbc..a3683d0950a2 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorPlumber.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorPlumber.java @@ -463,7 +463,7 @@ public String apply(SegmentIdentifier input) // WARNING: Committers.nil() here means that on-disk data can get out of sync with committing. Futures.addCallback( - appenderator.push(segmentsToPush, Committers.nil()), + appenderator.push(segmentsToPush, Committers.nil(), false), new FutureCallback() { @Override diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/BaseAppenderatorDriver.java b/server/src/main/java/io/druid/segment/realtime/appenderator/BaseAppenderatorDriver.java index 24482d19c96c..c578a65d6cd6 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/BaseAppenderatorDriver.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/BaseAppenderatorDriver.java @@ -39,6 +39,7 @@ import io.druid.java.util.common.ISE; import io.druid.java.util.common.concurrent.Execs; import io.druid.java.util.common.logger.Logger; +import io.druid.segment.loading.DataSegmentKiller; import io.druid.segment.realtime.appenderator.SegmentWithState.SegmentState; import org.joda.time.DateTime; @@ -128,6 +129,7 @@ Stream segmentStateStream() private final SegmentAllocator segmentAllocator; private final UsedSegmentChecker usedSegmentChecker; + private final DataSegmentKiller dataSegmentKiller; protected final Appenderator appenderator; // sequenceName -> segmentsForSequence @@ -141,12 +143,14 @@ Stream segmentStateStream() BaseAppenderatorDriver( Appenderator appenderator, SegmentAllocator segmentAllocator, - UsedSegmentChecker usedSegmentChecker + UsedSegmentChecker usedSegmentChecker, + DataSegmentKiller dataSegmentKiller ) { this.appenderator = Preconditions.checkNotNull(appenderator, "appenderator"); this.segmentAllocator = Preconditions.checkNotNull(segmentAllocator, "segmentAllocator"); this.usedSegmentChecker = Preconditions.checkNotNull(usedSegmentChecker, "usedSegmentChecker"); + this.dataSegmentKiller = Preconditions.checkNotNull(dataSegmentKiller, "dataSegmentKiller"); this.executor = MoreExecutors.listeningDecorator(Execs.singleThreaded("publish-%d")); } @@ -331,24 +335,32 @@ Stream getSegmentWithStates(Collection sequenceNames) * * @param wrappedCommitter should not be null if you want to persist intermediate states * @param segmentIdentifiers identifiers of the segments to be pushed + * @param useUniquePath true if the segment should be written to a path with a unique identifier * * @return a future for pushing segments */ ListenableFuture pushInBackground( @Nullable final WrappedCommitter wrappedCommitter, - final Collection segmentIdentifiers + final Collection segmentIdentifiers, + final boolean useUniquePath ) { log.info("Pushing segments in background: [%s]", Joiner.on(", ").join(segmentIdentifiers)); return Futures.transform( - appenderator.push(segmentIdentifiers, wrappedCommitter), + appenderator.push(segmentIdentifiers, wrappedCommitter, useUniquePath), (Function) segmentsAndMetadata -> { // Sanity check final Set pushedSegments = segmentsAndMetadata.getSegments().stream() .map(SegmentIdentifier::fromDataSegment) .collect(Collectors.toSet()); if (!pushedSegments.equals(Sets.newHashSet(segmentIdentifiers))) { + log.warn( + "Removing segments from deep storage because sanity check failed: %s", segmentsAndMetadata.getSegments() + ); + + segmentsAndMetadata.getSegments().forEach(dataSegmentKiller::killQuietly); + throw new ISE( "WTF?! Pushed different segments than requested. Pushed[%s], requested[%s].", pushedSegments, @@ -436,13 +448,25 @@ ListenableFuture publishInBackground( .collect(Collectors.toSet()); if (usedSegmentChecker.findUsedSegments(segmentsIdentifiers) .equals(Sets.newHashSet(segmentsAndMetadata.getSegments()))) { + log.info( + "Removing our segments from deep storage because someone else already published them: %s", + segmentsAndMetadata.getSegments() + ); + segmentsAndMetadata.getSegments().forEach(dataSegmentKiller::killQuietly); + log.info("Our segments really do exist, awaiting handoff."); } else { throw new ISE("Failed to publish segments[%s]", segmentsAndMetadata.getSegments()); } } } - catch (IOException e) { + catch (Exception e) { + log.warn( + "Removing segments from deep storage after failed publish: %s", + segmentsAndMetadata.getSegments() + ); + segmentsAndMetadata.getSegments().forEach(dataSegmentKiller::killQuietly); + throw Throwables.propagate(e); } } diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriver.java b/server/src/main/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriver.java index c67b31054bf1..4ceb5264097f 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriver.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriver.java @@ -26,6 +26,7 @@ import com.google.common.util.concurrent.ListenableFuture; import io.druid.data.input.InputRow; import io.druid.java.util.common.ISE; +import io.druid.segment.loading.DataSegmentKiller; import io.druid.segment.realtime.appenderator.SegmentWithState.SegmentState; import io.druid.timeline.DataSegment; @@ -41,11 +42,11 @@ /** * This class is specifialized for batch ingestion. In batch ingestion, the segment lifecycle is like: - * + *

    * 

      * APPENDING -> PUSHED_AND_DROPPED -> PUBLISHED
  *
    - * + *

    *

      *
    • APPENDING: Segment is available for appending.
      • *
    • PUSHED_AND_DROPPED: Segment is pushed to deep storage and dropped from the local storage.
      • @@ -57,22 +58,23 @@ public class BatchAppenderatorDriver extends BaseAppenderatorDriver /** * Create a driver. * - * @param appenderator appenderator - * @param segmentAllocator segment allocator - * @param usedSegmentChecker used segment checker + * @param appenderator appenderator + * @param segmentAllocator segment allocator + * @param usedSegmentChecker used segment checker */ public BatchAppenderatorDriver( Appenderator appenderator, SegmentAllocator segmentAllocator, - UsedSegmentChecker usedSegmentChecker + UsedSegmentChecker usedSegmentChecker, + DataSegmentKiller dataSegmentKiller ) { - super(appenderator, segmentAllocator, usedSegmentChecker); + super(appenderator, segmentAllocator, usedSegmentChecker, dataSegmentKiller); } /** * This method always returns null because batch ingestion doesn't support restoring tasks on failures. - + * * @return always null */ @Override @@ -133,7 +135,7 @@ private SegmentsAndMetadata pushAndClear( .collect(Collectors.toList()); final ListenableFuture future = Futures.transform( - pushInBackground(null, segmentIdentifierList), + pushInBackground(null, segmentIdentifierList, false), this::dropInBackground ); @@ -198,7 +200,8 @@ public ListenableFuture publishAll(final TransactionalSegme .map(segmentWithState -> Preconditions.checkNotNull( segmentWithState.getDataSegment(), "dataSegment for segmentId[%s]", - segmentWithState.getSegmentIdentifier()) + segmentWithState.getSegmentIdentifier() + ) ) .collect(Collectors.toList()), null diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriver.java b/server/src/main/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriver.java index a5b1a3fbdd65..4b0c171ef8a9 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriver.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriver.java @@ -35,6 +35,7 @@ import io.druid.java.util.common.ISE; import io.druid.java.util.common.logger.Logger; import io.druid.query.SegmentDescriptor; +import io.druid.segment.loading.DataSegmentKiller; import io.druid.segment.realtime.FireDepartmentMetrics; import io.druid.segment.realtime.appenderator.SegmentWithState.SegmentState; import io.druid.segment.realtime.plumber.SegmentHandoffNotifier; @@ -54,11 +55,11 @@ /** * This class is specialized for streaming ingestion. In streaming ingestion, the segment lifecycle is like: - * + *

      * 

        * APPENDING -> APPEND_FINISHED -> PUBLISHED
  *
      - * + *

      *

        *
      • APPENDING: Segment is available for appending.
        • *
      • APPEND_FINISHED: Segment cannot be updated (data cannot be added anymore) and is waiting for being published.
        • @@ -89,11 +90,12 @@ public StreamAppenderatorDriver( SegmentAllocator segmentAllocator, SegmentHandoffNotifierFactory handoffNotifierFactory, UsedSegmentChecker usedSegmentChecker, + DataSegmentKiller dataSegmentKiller, ObjectMapper objectMapper, FireDepartmentMetrics metrics ) { - super(appenderator, segmentAllocator, usedSegmentChecker); + super(appenderator, segmentAllocator, usedSegmentChecker, dataSegmentKiller); this.handoffNotifier = Preconditions.checkNotNull(handoffNotifierFactory, "handoffNotifierFactory") .createSegmentHandoffNotifier(appenderator.getDataSource()); @@ -261,7 +263,9 @@ public ListenableFuture publish( .collect(Collectors.toList()); final ListenableFuture publishFuture = Futures.transform( - pushInBackground(wrapCommitter(committer), theSegments), + // useUniquePath=true prevents inconsistencies in segment data when task failures or replicas leads to a second + // version of a segment with the same identifier containing different data; see DataSegmentPusher.push() docs + pushInBackground(wrapCommitter(committer), theSegments, true), (AsyncFunction) segmentsAndMetadata -> publishInBackground( segmentsAndMetadata, publisher diff --git a/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java b/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java index 2f6a4a1dd995..91ccae79407a 100644 --- a/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java +++ b/server/src/main/java/io/druid/segment/realtime/plumber/RealtimePlumber.java @@ -444,13 +444,6 @@ public void doRun() log.info("Pushing [%s] to deep storage", sink.getSegment().getIdentifier()); - // The realtime plumber can generate segments with the same identifier (i.e. replica tasks) but does not - // have any strict requirement that the contents of these segments be identical. It is possible that two - // tasks generate a segment with the same identifier containing different data, and in this situation we - // want to favor the data from the task which pushed first. This is because it is possible that one - // historical could load the segment after the first task pushed and another historical load the same - // segment after the second task pushed. If the second task's segment overwrote the first one, the second - // historical node would be serving different data from the first. Hence set replaceExisting == false. DataSegment segment = dataSegmentPusher.push( mergedFile, sink.getSegment().withDimensions(IndexMerger.getMergedDimensionsFromQueryableIndexes(indexes)), diff --git a/server/src/test/java/io/druid/segment/loading/LocalDataSegmentFinderTest.java b/server/src/test/java/io/druid/segment/loading/LocalDataSegmentFinderTest.java index 78b48a0ab6aa..b399bb8c5985 100644 --- a/server/src/test/java/io/druid/segment/loading/LocalDataSegmentFinderTest.java +++ b/server/src/test/java/io/druid/segment/loading/LocalDataSegmentFinderTest.java @@ -36,6 +36,7 @@ import org.junit.rules.TemporaryFolder; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.util.Set; @@ -204,12 +205,6 @@ public void testFindSegments() throws SegmentLoadingException, IOException Assert.assertEquals(serializedSegment4_1, FileUtils.readFileToString(descriptor4_1)); } - private String getDescriptorPath(DataSegment segment) - { - final File indexzip = new File(String.valueOf(segment.getLoadSpec().get("path"))); - return indexzip.getParent() + "/" + DESCRIPTOR_JSON; - } - @Test(expected = SegmentLoadingException.class) public void testFindSegmentsFail() throws SegmentLoadingException { @@ -219,4 +214,44 @@ public void testFindSegmentsFail() throws SegmentLoadingException final LocalDataSegmentFinder localDataSegmentFinder = new LocalDataSegmentFinder(mapper); localDataSegmentFinder.findSegments(dataSourceDir.getAbsolutePath(), false); } + + @Test + public void testPreferNewestSegment() throws Exception + { + dataSourceDir = temporaryFolder.newFolder(); + descriptor1 = new File(dataSourceDir.getAbsolutePath() + "/interval10/v10/0/older", DESCRIPTOR_JSON); + descriptor2 = new File(dataSourceDir.getAbsolutePath() + "/interval10/v10/0/newer", DESCRIPTOR_JSON); + + descriptor1.getParentFile().mkdirs(); + descriptor2.getParentFile().mkdirs(); + + mapper.writeValue(descriptor1, SEGMENT_1); + mapper.writeValue(descriptor2, SEGMENT_1); + + indexZip1 = new File(descriptor1.getParentFile(), INDEX_ZIP); + indexZip2 = new File(descriptor2.getParentFile(), INDEX_ZIP); + + FileOutputStream fos1 = new FileOutputStream(indexZip1); + fos1.getFD().sync(); + fos1.close(); + + Thread.sleep(1000); + + FileOutputStream fos2 = new FileOutputStream(indexZip2); + fos2.getFD().sync(); + fos2.close(); + + final Set segments = new LocalDataSegmentFinder(mapper).findSegments( + dataSourceDir.getAbsolutePath(), false + ); + + Assert.assertEquals(1, segments.size()); + Assert.assertEquals(indexZip2.getAbsolutePath(), segments.iterator().next().getLoadSpec().get("path")); + } + + private String getDescriptorPath(DataSegment segment) + { + final File indexzip = new File(String.valueOf(segment.getLoadSpec().get("path"))); + return indexzip.getParent() + "/" + DESCRIPTOR_JSON; + } } diff --git a/server/src/test/java/io/druid/segment/loading/LocalDataSegmentKillerTest.java b/server/src/test/java/io/druid/segment/loading/LocalDataSegmentKillerTest.java index b5eaad7fc9c1..4c75b4d5f681 100644 --- a/server/src/test/java/io/druid/segment/loading/LocalDataSegmentKillerTest.java +++ b/server/src/test/java/io/druid/segment/loading/LocalDataSegmentKillerTest.java @@ -31,6 +31,7 @@ import java.io.File; import java.io.IOException; +import java.util.UUID; public class LocalDataSegmentKillerTest { @@ -93,6 +94,28 @@ public void testKill() throws Exception Assert.assertFalse(dataSourceDir.exists()); } + @Test + public void testKillUniquePath() throws Exception + { + final LocalDataSegmentKiller killer = new LocalDataSegmentKiller(new LocalDataSegmentPusherConfig()); + final String uuid = UUID.randomUUID().toString().substring(0, 5); + final File dataSourceDir = temporaryFolder.newFolder("dataSource"); + final File intervalDir = new File(dataSourceDir, "interval"); + final File versionDir = new File(intervalDir, "1"); + final File partitionDir = new File(versionDir, "0"); + final File uuidDir = new File(partitionDir, uuid); + + makePartitionDirWithIndex(uuidDir); + + killer.kill(getSegmentWithPath(new File(uuidDir, "index.zip").toString())); + + Assert.assertFalse(uuidDir.exists()); + Assert.assertFalse(partitionDir.exists()); + Assert.assertFalse(versionDir.exists()); + Assert.assertFalse(intervalDir.exists()); + Assert.assertFalse(dataSourceDir.exists()); + } + private void makePartitionDirWithIndex(File path) throws IOException { Assert.assertTrue(path.mkdirs()); diff --git a/server/src/test/java/io/druid/segment/loading/LocalDataSegmentPusherTest.java b/server/src/test/java/io/druid/segment/loading/LocalDataSegmentPusherTest.java index e8168cb668fe..6ff0ab06ec63 100644 --- a/server/src/test/java/io/druid/segment/loading/LocalDataSegmentPusherTest.java +++ b/server/src/test/java/io/druid/segment/loading/LocalDataSegmentPusherTest.java @@ -92,8 +92,8 @@ public void testPush() throws IOException */ final DataSegment dataSegment2 = dataSegment.withVersion("v2"); - DataSegment returnSegment1 = localDataSegmentPusher.push(dataSegmentFiles, dataSegment, true); - DataSegment returnSegment2 = localDataSegmentPusher.push(dataSegmentFiles, dataSegment2, true); + DataSegment returnSegment1 = localDataSegmentPusher.push(dataSegmentFiles, dataSegment, false); + DataSegment returnSegment2 = localDataSegmentPusher.push(dataSegmentFiles, dataSegment2, false); Assert.assertNotNull(returnSegment1); Assert.assertEquals(dataSegment, returnSegment1); @@ -102,14 +102,14 @@ public void testPush() throws IOException Assert.assertEquals(dataSegment2, returnSegment2); Assert.assertNotEquals( - localDataSegmentPusher.getStorageDir(dataSegment), - localDataSegmentPusher.getStorageDir(dataSegment2) + localDataSegmentPusher.getStorageDir(dataSegment, false), + localDataSegmentPusher.getStorageDir(dataSegment2, false) ); for (DataSegment returnSegment : ImmutableList.of(returnSegment1, returnSegment2)) { File outDir = new File( config.getStorageDirectory(), - localDataSegmentPusher.getStorageDir(returnSegment) + localDataSegmentPusher.getStorageDir(returnSegment, false) ); File versionFile = new File(outDir, "index.zip"); File descriptorJson = new File(outDir, "descriptor.json"); @@ -119,33 +119,23 @@ public void testPush() throws IOException } @Test - public void testFirstPushWinsForConcurrentPushesWhenReplaceExistingFalse() throws IOException + public void testPushUseUniquePath() throws IOException { - File replicatedDataSegmentFiles = temporaryFolder.newFolder(); - Files.asByteSink(new File(replicatedDataSegmentFiles, "version.bin")).write(Ints.toByteArray(0x8)); - DataSegment returnSegment1 = localDataSegmentPusher.push(dataSegmentFiles, dataSegment, false); - DataSegment returnSegment2 = localDataSegmentPusher.push(replicatedDataSegmentFiles, dataSegment2, false); - - Assert.assertEquals(dataSegment.getDimensions(), returnSegment1.getDimensions()); - Assert.assertEquals(dataSegment.getDimensions(), returnSegment2.getDimensions()); - - File unzipDir = new File(config.storageDirectory, "unzip"); - FileUtils.forceMkdir(unzipDir); - CompressionUtils.unzip( - new File(config.storageDirectory, "/ds/1970-01-01T00:00:00.000Z_1970-01-01T00:00:00.001Z/v1/0/index.zip"), - unzipDir - ); + DataSegment segment = localDataSegmentPusher.push(dataSegmentFiles, dataSegment, true); - Assert.assertEquals(0x9, Ints.fromByteArray(Files.toByteArray(new File(unzipDir, "version.bin")))); + String path = segment.getLoadSpec().get("path").toString(); + String matcher = ".*/ds/1970-01-01T00:00:00\\.000Z_1970-01-01T00:00:00\\.001Z/v1/0/[A-Za-z0-9-]{36}/index\\.zip"; + Assert.assertTrue(path, path.matches(matcher)); + Assert.assertTrue(new File(path).exists()); } @Test - public void testLastPushWinsForConcurrentPushesWhenReplaceExistingTrue() throws IOException + public void testLastPushWinsForConcurrentPushes() throws IOException { File replicatedDataSegmentFiles = temporaryFolder.newFolder(); Files.asByteSink(new File(replicatedDataSegmentFiles, "version.bin")).write(Ints.toByteArray(0x8)); - DataSegment returnSegment1 = localDataSegmentPusher.push(dataSegmentFiles, dataSegment, true); - DataSegment returnSegment2 = localDataSegmentPusher.push(replicatedDataSegmentFiles, dataSegment2, true); + DataSegment returnSegment1 = localDataSegmentPusher.push(dataSegmentFiles, dataSegment, false); + DataSegment returnSegment2 = localDataSegmentPusher.push(replicatedDataSegmentFiles, dataSegment2, false); Assert.assertEquals(dataSegment.getDimensions(), returnSegment1.getDimensions()); Assert.assertEquals(dataSegment2.getDimensions(), returnSegment2.getDimensions()); @@ -168,7 +158,7 @@ public void testPushCannotCreateDirectory() throws IOException config.storageDirectory = new File(config.storageDirectory, "xxx"); Assert.assertTrue(config.storageDirectory.mkdir()); config.storageDirectory.setWritable(false); - localDataSegmentPusher.push(dataSegmentFiles, dataSegment, true); + localDataSegmentPusher.push(dataSegmentFiles, dataSegment, false); } @Test diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTest.java index 8a5bbda46c19..85bcd22ced33 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTest.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTest.java @@ -118,7 +118,8 @@ public void testSimpleIngestion() throws Exception // push all final SegmentsAndMetadata segmentsAndMetadata = appenderator.push( appenderator.getSegments(), - committerSupplier.get() + committerSupplier.get(), + false ).get(); Assert.assertEquals(ImmutableMap.of("x", "3"), (Map) segmentsAndMetadata.getCommitMetadata()); Assert.assertEquals( diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTester.java b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTester.java index 91ea83cc449e..ac0af8334487 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTester.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTester.java @@ -193,7 +193,7 @@ public String getPathForHadoop() } @Override - public DataSegment push(File file, DataSegment segment, boolean replaceExisting) throws IOException + public DataSegment push(File file, DataSegment segment, boolean useUniquePath) throws IOException { if (enablePushFailure && mustFail) { mustFail = false; diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriverTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriverTest.java index c69c8e443a7d..5ca72100a945 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriverTest.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriverTest.java @@ -27,10 +27,13 @@ import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.Intervals; import io.druid.java.util.common.granularity.Granularities; +import io.druid.segment.loading.DataSegmentKiller; import io.druid.segment.realtime.appenderator.BaseAppenderatorDriver.SegmentsForSequence; import io.druid.segment.realtime.appenderator.StreamAppenderatorDriverTest.TestSegmentAllocator; import io.druid.segment.realtime.appenderator.SegmentWithState.SegmentState; import io.druid.timeline.partition.NumberedShardSpec; +import org.easymock.EasyMock; +import org.easymock.EasyMockSupport; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -41,7 +44,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -public class BatchAppenderatorDriverTest +public class BatchAppenderatorDriverTest extends EasyMockSupport { private static final String DATA_SOURCE = "foo"; private static final String VERSION = "abc123"; @@ -69,22 +72,29 @@ public class BatchAppenderatorDriverTest private SegmentAllocator allocator; private AppenderatorTester appenderatorTester; private BatchAppenderatorDriver driver; + private DataSegmentKiller dataSegmentKiller; @Before public void setup() { appenderatorTester = new AppenderatorTester(MAX_ROWS_IN_MEMORY); allocator = new TestSegmentAllocator(DATA_SOURCE, Granularities.HOUR); + dataSegmentKiller = createStrictMock(DataSegmentKiller.class); driver = new BatchAppenderatorDriver( appenderatorTester.getAppenderator(), allocator, - new TestUsedSegmentChecker(appenderatorTester) + new TestUsedSegmentChecker(appenderatorTester), + dataSegmentKiller ); + + EasyMock.replay(dataSegmentKiller); } @After public void tearDown() throws Exception { + EasyMock.verify(dataSegmentKiller); + driver.clear(); driver.close(); } diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverFailTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverFailTest.java index a249ccb7f088..a89d9565c8c8 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverFailTest.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverFailTest.java @@ -34,16 +34,20 @@ import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.ISE; +import io.druid.java.util.common.Intervals; import io.druid.java.util.common.granularity.Granularities; import io.druid.query.Query; import io.druid.query.QueryRunner; import io.druid.query.SegmentDescriptor; -import io.druid.segment.incremental.IndexSizeExceededException; +import io.druid.segment.loading.DataSegmentKiller; import io.druid.segment.realtime.FireDepartmentMetrics; import io.druid.segment.realtime.appenderator.StreamAppenderatorDriverTest.TestCommitterSupplier; import io.druid.segment.realtime.appenderator.StreamAppenderatorDriverTest.TestSegmentAllocator; import io.druid.segment.realtime.appenderator.StreamAppenderatorDriverTest.TestSegmentHandoffNotifierFactory; import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NumberedShardSpec; +import org.easymock.EasyMock; +import org.easymock.EasyMockSupport; import org.hamcrest.CoreMatchers; import org.joda.time.Interval; import org.junit.After; @@ -65,7 +69,7 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; -public class StreamAppenderatorDriverFailTest +public class StreamAppenderatorDriverFailTest extends EasyMockSupport { private static final String DATA_SOURCE = "foo"; private static final ObjectMapper OBJECT_MAPPER = new DefaultObjectMapper(); @@ -92,6 +96,7 @@ public class StreamAppenderatorDriverFailTest SegmentAllocator allocator; TestSegmentHandoffNotifierFactory segmentHandoffNotifierFactory; StreamAppenderatorDriver driver; + DataSegmentKiller dataSegmentKiller; @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -101,6 +106,7 @@ public void setUp() { allocator = new TestSegmentAllocator(DATA_SOURCE, Granularities.HOUR); segmentHandoffNotifierFactory = new TestSegmentHandoffNotifierFactory(); + dataSegmentKiller = createStrictMock(DataSegmentKiller.class); } @After @@ -126,6 +132,7 @@ public void testFailDuringPersist() throws IOException, InterruptedException, Ti allocator, segmentHandoffNotifierFactory, new NoopUsedSegmentChecker(), + dataSegmentKiller, OBJECT_MAPPER, new FireDepartmentMetrics() ); @@ -163,6 +170,7 @@ public void testFailDuringPush() throws IOException, InterruptedException, Timeo allocator, segmentHandoffNotifierFactory, new NoopUsedSegmentChecker(), + dataSegmentKiller, OBJECT_MAPPER, new FireDepartmentMetrics() ); @@ -200,6 +208,7 @@ public void testFailDuringDrop() throws IOException, InterruptedException, Timeo allocator, segmentHandoffNotifierFactory, new NoopUsedSegmentChecker(), + dataSegmentKiller, OBJECT_MAPPER, new FireDepartmentMetrics() ); @@ -225,6 +234,94 @@ public void testFailDuringDrop() throws IOException, InterruptedException, Timeo driver.registerHandoff(published).get(); } + @Test + public void testFailDuringPublish() throws Exception + { + expectedException.expect(ExecutionException.class); + expectedException.expectCause(CoreMatchers.instanceOf(ISE.class)); + expectedException.expectMessage( + "Failed to publish segments[[DataSegment{size=0, shardSpec=NumberedShardSpec{partitionNum=0, partitions=0}, metrics=[], dimensions=[], version='abc123', loadSpec={}, interval=2000-01-01T00:00:00.000Z/2000-01-01T01:00:00.000Z, dataSource='foo', binaryVersion='0'}, DataSegment{size=0, shardSpec=NumberedShardSpec{partitionNum=0, partitions=0}, metrics=[], dimensions=[], version='abc123', loadSpec={}, interval=2000-01-01T01:00:00.000Z/2000-01-01T02:00:00.000Z, dataSource='foo', binaryVersion='0'}]]"); + + testFailDuringPublishInternal(false); + } + + @Test + public void testFailWithExceptionDuringPublish() throws Exception + { + expectedException.expect(ExecutionException.class); + expectedException.expectCause(CoreMatchers.instanceOf(RuntimeException.class)); + expectedException.expectMessage("test"); + + testFailDuringPublishInternal(true); + } + + private void testFailDuringPublishInternal(boolean failWithException) throws Exception + { + driver = new StreamAppenderatorDriver( + new FailableAppenderator(), + allocator, + segmentHandoffNotifierFactory, + new NoopUsedSegmentChecker(), + dataSegmentKiller, + OBJECT_MAPPER, + new FireDepartmentMetrics() + ); + + driver.startJob(); + + final TestCommitterSupplier committerSupplier = new TestCommitterSupplier<>(); + segmentHandoffNotifierFactory.setHandoffDelay(100); + + Assert.assertNull(driver.startJob()); + + for (int i = 0; i < ROWS.size(); i++) { + committerSupplier.setMetadata(i + 1); + Assert.assertTrue(driver.add(ROWS.get(i), "dummy", committerSupplier, false, true).isOk()); + } + + dataSegmentKiller.killQuietly(new DataSegment( + "foo", + Intervals.of("2000-01-01T00:00:00.000Z/2000-01-01T01:00:00.000Z"), + "abc123", + ImmutableMap.of(), + ImmutableList.of(), + ImmutableList.of(), + new NumberedShardSpec(0, 0), + 0, + 0 + )); + EasyMock.expectLastCall().once(); + + dataSegmentKiller.killQuietly(new DataSegment( + "foo", + Intervals.of("2000-01-01T01:00:00.000Z/2000-01-01T02:00:00.000Z"), + "abc123", + ImmutableMap.of(), + ImmutableList.of(), + ImmutableList.of(), + new NumberedShardSpec(0, 0), + 0, + 0 + )); + EasyMock.expectLastCall().once(); + + EasyMock.replay(dataSegmentKiller); + + try { + driver.publish( + StreamAppenderatorDriverTest.makeFailingPublisher(failWithException), + committerSupplier.get(), + ImmutableList.of("dummy") + ).get(PUBLISH_TIMEOUT, TimeUnit.MILLISECONDS); + } + catch (Exception e) { + throw e; + } + finally { + EasyMock.verify(dataSegmentKiller); + } + } + private static class NoopUsedSegmentChecker implements UsedSegmentChecker { @Override @@ -305,8 +402,11 @@ public Object startJob() @Override public AppenderatorAddResult add( - SegmentIdentifier identifier, InputRow row, Supplier committerSupplier, boolean allowIncrementalPersists - ) throws IndexSizeExceededException, SegmentNotWritableException + SegmentIdentifier identifier, + InputRow row, + Supplier committerSupplier, + boolean allowIncrementalPersists + ) { rows.computeIfAbsent(identifier, k -> new ArrayList<>()).add(row); numRows++; @@ -368,7 +468,7 @@ public ListenableFuture persist( @Override public ListenableFuture push( - Collection identifiers, Committer committer + Collection identifiers, Committer committer, boolean useUniquePath ) { if (pushEnabled) { diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverTest.java index ee92d2e2d7d0..aff1e020c530 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverTest.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverTest.java @@ -20,7 +20,6 @@ package io.druid.segment.realtime.appenderator; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Function; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -38,11 +37,14 @@ import io.druid.java.util.common.granularity.Granularities; import io.druid.java.util.common.granularity.Granularity; import io.druid.query.SegmentDescriptor; +import io.druid.segment.loading.DataSegmentKiller; import io.druid.segment.realtime.FireDepartmentMetrics; import io.druid.segment.realtime.plumber.SegmentHandoffNotifier; import io.druid.segment.realtime.plumber.SegmentHandoffNotifierFactory; import io.druid.timeline.DataSegment; import io.druid.timeline.partition.NumberedShardSpec; +import org.easymock.EasyMock; +import org.easymock.EasyMockSupport; import org.joda.time.DateTime; import org.junit.After; import org.junit.Assert; @@ -61,7 +63,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; -public class StreamAppenderatorDriverTest +public class StreamAppenderatorDriverTest extends EasyMockSupport { private static final String DATA_SOURCE = "foo"; private static final String VERSION = "abc123"; @@ -93,26 +95,33 @@ public class StreamAppenderatorDriverTest private AppenderatorTester appenderatorTester; private TestSegmentHandoffNotifierFactory segmentHandoffNotifierFactory; private StreamAppenderatorDriver driver; + private DataSegmentKiller dataSegmentKiller; @Before - public void setUp() + public void setUp() throws Exception { appenderatorTester = new AppenderatorTester(MAX_ROWS_IN_MEMORY); allocator = new TestSegmentAllocator(DATA_SOURCE, Granularities.HOUR); segmentHandoffNotifierFactory = new TestSegmentHandoffNotifierFactory(); + dataSegmentKiller = createStrictMock(DataSegmentKiller.class); driver = new StreamAppenderatorDriver( appenderatorTester.getAppenderator(), allocator, segmentHandoffNotifierFactory, new TestUsedSegmentChecker(appenderatorTester), + dataSegmentKiller, OBJECT_MAPPER, new FireDepartmentMetrics() ); + + EasyMock.replay(dataSegmentKiller); } @After public void tearDown() throws Exception { + EasyMock.verify(dataSegmentKiller); + driver.clear(); driver.close(); } @@ -345,19 +354,7 @@ public void testIncrementalHandoff() throws Exception private Set asIdentifiers(Iterable segments) { - return ImmutableSet.copyOf( - Iterables.transform( - segments, - new Function() - { - @Override - public SegmentIdentifier apply(DataSegment input) - { - return SegmentIdentifier.fromDataSegment(input); - } - } - ) - ); + return ImmutableSet.copyOf(Iterables.transform(segments, SegmentIdentifier::fromDataSegment)); } static TransactionalSegmentPublisher makeOkPublisher() @@ -372,6 +369,16 @@ public boolean publishSegments(Set segments, Object commitMetadata) }; } + static TransactionalSegmentPublisher makeFailingPublisher(boolean failWithException) + { + return (segments, commitMetadata) -> { + if (failWithException) { + throw new RuntimeException("test"); + } + return false; + }; + } + static class TestCommitterSupplier implements Supplier { private final AtomicReference metadata = new AtomicReference<>(); diff --git a/services/src/main/java/io/druid/cli/CliRealtimeExample.java b/services/src/main/java/io/druid/cli/CliRealtimeExample.java index a50f51bf0204..7ec1c32ac4db 100644 --- a/services/src/main/java/io/druid/cli/CliRealtimeExample.java +++ b/services/src/main/java/io/druid/cli/CliRealtimeExample.java @@ -155,7 +155,7 @@ public String getPathForHadoop(String dataSource) } @Override - public DataSegment push(File file, DataSegment segment, boolean replaceExisting) throws IOException + public DataSegment push(File file, DataSegment segment, boolean useUniquePath) { return segment; } From e2932fe4f7faec5e411adcf72d1fcc8d935f291a Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 3 May 2018 17:06:00 -0700 Subject: [PATCH 68/82] [maven-release-plugin] prepare release druid-0.12.1-rc1 --- api/pom.xml | 2 +- aws-common/pom.xml | 2 +- benchmarks/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- extendedset/pom.xml | 2 +- extensions-contrib/ambari-metrics-emitter/pom.xml | 2 +- extensions-contrib/azure-extensions/pom.xml | 2 +- extensions-contrib/cassandra-storage/pom.xml | 2 +- extensions-contrib/cloudfiles-extensions/pom.xml | 2 +- extensions-contrib/distinctcount/pom.xml | 2 +- extensions-contrib/druid-rocketmq/pom.xml | 2 +- extensions-contrib/google-extensions/pom.xml | 2 +- extensions-contrib/graphite-emitter/pom.xml | 2 +- extensions-contrib/kafka-eight-simpleConsumer/pom.xml | 2 +- extensions-contrib/kafka-emitter/pom.xml | 2 +- extensions-contrib/orc-extensions/pom.xml | 2 +- extensions-contrib/parquet-extensions/pom.xml | 2 +- extensions-contrib/rabbitmq/pom.xml | 2 +- extensions-contrib/redis-cache/pom.xml | 2 +- extensions-contrib/sqlserver-metadata-storage/pom.xml | 2 +- extensions-contrib/statsd-emitter/pom.xml | 2 +- extensions-contrib/thrift-extensions/pom.xml | 2 +- extensions-contrib/time-min-max/pom.xml | 2 +- extensions-contrib/virtual-columns/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 2 +- extensions-core/datasketches/pom.xml | 2 +- extensions-core/druid-basic-security/pom.xml | 2 +- extensions-core/druid-kerberos/pom.xml | 2 +- extensions-core/hdfs-storage/pom.xml | 2 +- extensions-core/histogram/pom.xml | 2 +- extensions-core/kafka-eight/pom.xml | 2 +- extensions-core/kafka-extraction-namespace/pom.xml | 2 +- extensions-core/kafka-indexing-service/pom.xml | 2 +- extensions-core/lookups-cached-global/pom.xml | 2 +- extensions-core/lookups-cached-single/pom.xml | 2 +- extensions-core/mysql-metadata-storage/pom.xml | 2 +- extensions-core/postgresql-metadata-storage/pom.xml | 2 +- extensions-core/protobuf-extensions/pom.xml | 2 +- extensions-core/s3-extensions/pom.xml | 2 +- extensions-core/simple-client-sslcontext/pom.xml | 2 +- extensions-core/stats/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- integration-tests/pom.xml | 2 +- java-util/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- sql/pom.xml | 2 +- 53 files changed, 54 insertions(+), 54 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 0e8333e27709..67fc7694afb8 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 diff --git a/aws-common/pom.xml b/aws-common/pom.xml index 9ce40e3951ea..d46f80722017 100644 --- a/aws-common/pom.xml +++ b/aws-common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 539d3df54524..76f8c74ebc9c 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 diff --git a/common/pom.xml b/common/pom.xml index 234f21af5bb2..dc3deee778de 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 diff --git a/distribution/pom.xml b/distribution/pom.xml index d5d046f1b336..b9de9a08958a 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -28,7 +28,7 @@ druid io.druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 diff --git a/examples/pom.xml b/examples/pom.xml index a5804484bedf..fd981c745266 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 diff --git a/extendedset/pom.xml b/extendedset/pom.xml index aac97e228602..d54b433c8c01 100755 --- a/extendedset/pom.xml +++ b/extendedset/pom.xml @@ -31,7 +31,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index fdad1c65963c..d211b5b097f5 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-contrib/azure-extensions/pom.xml b/extensions-contrib/azure-extensions/pom.xml index 9bdf4a8a6a1a..944b8946b881 100644 --- a/extensions-contrib/azure-extensions/pom.xml +++ b/extensions-contrib/azure-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-contrib/cassandra-storage/pom.xml b/extensions-contrib/cassandra-storage/pom.xml index 56c425c4520b..6114e6439cfd 100644 --- a/extensions-contrib/cassandra-storage/pom.xml +++ b/extensions-contrib/cassandra-storage/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index 5369f088ce3d..c43135cf5811 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-contrib/distinctcount/pom.xml b/extensions-contrib/distinctcount/pom.xml index 22be879de80b..02661447a1c0 100644 --- a/extensions-contrib/distinctcount/pom.xml +++ b/extensions-contrib/distinctcount/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-contrib/druid-rocketmq/pom.xml b/extensions-contrib/druid-rocketmq/pom.xml index 9bc68287152e..15e587fab777 100644 --- a/extensions-contrib/druid-rocketmq/pom.xml +++ b/extensions-contrib/druid-rocketmq/pom.xml @@ -22,7 +22,7 @@ druid io.druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-contrib/google-extensions/pom.xml b/extensions-contrib/google-extensions/pom.xml index 889fd343d1a6..87f53b21cd40 100644 --- a/extensions-contrib/google-extensions/pom.xml +++ b/extensions-contrib/google-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index c3b32e7bc072..4e376f1a0e06 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml index ec933db08366..0b3b7d0bbc08 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml +++ b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index fcc78fc7eebb..5060090e20e4 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-contrib/orc-extensions/pom.xml b/extensions-contrib/orc-extensions/pom.xml index 0ebc6fd8d19c..db73f8ac1648 100644 --- a/extensions-contrib/orc-extensions/pom.xml +++ b/extensions-contrib/orc-extensions/pom.xml @@ -24,7 +24,7 @@ druid io.druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/parquet-extensions/pom.xml b/extensions-contrib/parquet-extensions/pom.xml index ead3c3bf78b1..12d911688f23 100644 --- a/extensions-contrib/parquet-extensions/pom.xml +++ b/extensions-contrib/parquet-extensions/pom.xml @@ -8,7 +8,7 @@ druid io.druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/rabbitmq/pom.xml b/extensions-contrib/rabbitmq/pom.xml index dd68599a48ba..89c5d793220e 100644 --- a/extensions-contrib/rabbitmq/pom.xml +++ b/extensions-contrib/rabbitmq/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-contrib/redis-cache/pom.xml b/extensions-contrib/redis-cache/pom.xml index 47be36eb3d73..6d137d1b13ae 100644 --- a/extensions-contrib/redis-cache/pom.xml +++ b/extensions-contrib/redis-cache/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-contrib/sqlserver-metadata-storage/pom.xml b/extensions-contrib/sqlserver-metadata-storage/pom.xml index 7d91c2d88207..71f866e166c9 100644 --- a/extensions-contrib/sqlserver-metadata-storage/pom.xml +++ b/extensions-contrib/sqlserver-metadata-storage/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index 0fc55b102427..dd5e427774da 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index e815d046b13f..f3dcc234389b 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -9,7 +9,7 @@ druid io.druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/time-min-max/pom.xml b/extensions-contrib/time-min-max/pom.xml index 1b79eb519f7b..6962178a6b06 100644 --- a/extensions-contrib/time-min-max/pom.xml +++ b/extensions-contrib/time-min-max/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/virtual-columns/pom.xml b/extensions-contrib/virtual-columns/pom.xml index 6095f37a87b5..fbff5bb5daf2 100644 --- a/extensions-contrib/virtual-columns/pom.xml +++ b/extensions-contrib/virtual-columns/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index d57142176726..6ed3a0be8ed4 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index 5db3f194b20a..9db0c072e425 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-core/druid-basic-security/pom.xml b/extensions-core/druid-basic-security/pom.xml index 5367bfb93ef9..a108ef43a771 100644 --- a/extensions-core/druid-basic-security/pom.xml +++ b/extensions-core/druid-basic-security/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index 6f06a47b3d1b..c843981df80f 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index f737da39c3e1..2e9df56000f7 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-core/histogram/pom.xml b/extensions-core/histogram/pom.xml index ef4604a257a2..1a6a2717abae 100644 --- a/extensions-core/histogram/pom.xml +++ b/extensions-core/histogram/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-core/kafka-eight/pom.xml b/extensions-core/kafka-eight/pom.xml index 155aae49763b..d5454d114cd6 100644 --- a/extensions-core/kafka-eight/pom.xml +++ b/extensions-core/kafka-eight/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index cc158c8d9a9f..1421594a0b9d 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index 13b15ca16478..c7fcd279decc 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-core/lookups-cached-global/pom.xml b/extensions-core/lookups-cached-global/pom.xml index 80c84328b87a..67c430e8b92e 100644 --- a/extensions-core/lookups-cached-global/pom.xml +++ b/extensions-core/lookups-cached-global/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-core/lookups-cached-single/pom.xml b/extensions-core/lookups-cached-single/pom.xml index c0bfd3abfb87..c710a36f8a96 100644 --- a/extensions-core/lookups-cached-single/pom.xml +++ b/extensions-core/lookups-cached-single/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index a3b8fa7cd80a..d3990f495de9 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-core/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml index 05591967d3c0..c9250ba2ee5e 100644 --- a/extensions-core/postgresql-metadata-storage/pom.xml +++ b/extensions-core/postgresql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-core/protobuf-extensions/pom.xml b/extensions-core/protobuf-extensions/pom.xml index 0c9f4cad1693..abf49ec4c8c2 100644 --- a/extensions-core/protobuf-extensions/pom.xml +++ b/extensions-core/protobuf-extensions/pom.xml @@ -27,7 +27,7 @@ druid io.druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index dd84f21a182e..2cbc12861418 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/extensions-core/simple-client-sslcontext/pom.xml b/extensions-core/simple-client-sslcontext/pom.xml index c9037c4bde02..09abba31926b 100644 --- a/extensions-core/simple-client-sslcontext/pom.xml +++ b/extensions-core/simple-client-sslcontext/pom.xml @@ -3,7 +3,7 @@ druid io.druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml 4.0.0 diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml index 2e989c1c07f7..7bb7f41fc8af 100644 --- a/extensions-core/stats/pom.xml +++ b/extensions-core/stats/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 ../../pom.xml diff --git a/hll/pom.xml b/hll/pom.xml index 676a4eb92b7b..4e3ce5533487 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 druid-hll diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index e2bf002c3d55..1e47579584ec 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index cd15dc607b2d..cde434ab15df 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 7e1de402aec4..e93c2f49869a 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 diff --git a/java-util/pom.xml b/java-util/pom.xml index 1efd28e039ac..4ca29450b5f2 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 java-util diff --git a/pom.xml b/pom.xml index b25f1c7d5252..92884c6a59d9 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 pom ${project.groupId}:${project.artifactId} @@ -53,7 +53,7 @@ scm:git:ssh://git@github.com/druid-io/druid.git scm:git:ssh://git@github.com/druid-io/druid.git https://github.com/druid-io/druid.git - 0.12.0-SNAPSHOT + druid-0.12.1-rc1 diff --git a/processing/pom.xml b/processing/pom.xml index 220abc4192f3..41eaaee05e2d 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 diff --git a/server/pom.xml b/server/pom.xml index 0b9a07a3b9ef..0246bf3054ba 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 diff --git a/services/pom.xml b/services/pom.xml index 4444c2ad9297..32bd7da6cc78 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -25,7 +25,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 diff --git a/sql/pom.xml b/sql/pom.xml index d4eae7224b84..9c6a9b53fbb0 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.13.0-SNAPSHOT + 0.12.1-rc1 From fc716fa090b31091a406f621b21d6b398f17def8 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 3 May 2018 17:06:11 -0700 Subject: [PATCH 69/82] [maven-release-plugin] prepare for next development iteration --- api/pom.xml | 2 +- aws-common/pom.xml | 2 +- benchmarks/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- extendedset/pom.xml | 2 +- extensions-contrib/ambari-metrics-emitter/pom.xml | 2 +- extensions-contrib/azure-extensions/pom.xml | 2 +- extensions-contrib/cassandra-storage/pom.xml | 2 +- extensions-contrib/cloudfiles-extensions/pom.xml | 2 +- extensions-contrib/distinctcount/pom.xml | 2 +- extensions-contrib/druid-rocketmq/pom.xml | 2 +- extensions-contrib/google-extensions/pom.xml | 2 +- extensions-contrib/graphite-emitter/pom.xml | 2 +- extensions-contrib/kafka-eight-simpleConsumer/pom.xml | 2 +- extensions-contrib/kafka-emitter/pom.xml | 2 +- extensions-contrib/orc-extensions/pom.xml | 2 +- extensions-contrib/parquet-extensions/pom.xml | 2 +- extensions-contrib/rabbitmq/pom.xml | 2 +- extensions-contrib/redis-cache/pom.xml | 2 +- extensions-contrib/sqlserver-metadata-storage/pom.xml | 2 +- extensions-contrib/statsd-emitter/pom.xml | 2 +- extensions-contrib/thrift-extensions/pom.xml | 2 +- extensions-contrib/time-min-max/pom.xml | 2 +- extensions-contrib/virtual-columns/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 2 +- extensions-core/datasketches/pom.xml | 2 +- extensions-core/druid-basic-security/pom.xml | 2 +- extensions-core/druid-kerberos/pom.xml | 2 +- extensions-core/hdfs-storage/pom.xml | 2 +- extensions-core/histogram/pom.xml | 2 +- extensions-core/kafka-eight/pom.xml | 2 +- extensions-core/kafka-extraction-namespace/pom.xml | 2 +- extensions-core/kafka-indexing-service/pom.xml | 2 +- extensions-core/lookups-cached-global/pom.xml | 2 +- extensions-core/lookups-cached-single/pom.xml | 2 +- extensions-core/mysql-metadata-storage/pom.xml | 2 +- extensions-core/postgresql-metadata-storage/pom.xml | 2 +- extensions-core/protobuf-extensions/pom.xml | 2 +- extensions-core/s3-extensions/pom.xml | 2 +- extensions-core/simple-client-sslcontext/pom.xml | 2 +- extensions-core/stats/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- integration-tests/pom.xml | 2 +- java-util/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- sql/pom.xml | 2 +- 53 files changed, 54 insertions(+), 54 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 67fc7694afb8..d262eeb898d9 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT diff --git a/aws-common/pom.xml b/aws-common/pom.xml index d46f80722017..665e277d74f2 100644 --- a/aws-common/pom.xml +++ b/aws-common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 76f8c74ebc9c..8e62ab45f0d5 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index dc3deee778de..978aac817f06 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT diff --git a/distribution/pom.xml b/distribution/pom.xml index b9de9a08958a..04a145251485 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -28,7 +28,7 @@ druid io.druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index fd981c745266..9136b1276ccc 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT diff --git a/extendedset/pom.xml b/extendedset/pom.xml index d54b433c8c01..c0c2329a9f03 100755 --- a/extendedset/pom.xml +++ b/extendedset/pom.xml @@ -31,7 +31,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index d211b5b097f5..802c35af8aa7 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/azure-extensions/pom.xml b/extensions-contrib/azure-extensions/pom.xml index 944b8946b881..10e7d0b542a9 100644 --- a/extensions-contrib/azure-extensions/pom.xml +++ b/extensions-contrib/azure-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/cassandra-storage/pom.xml b/extensions-contrib/cassandra-storage/pom.xml index 6114e6439cfd..e2d32197de5d 100644 --- a/extensions-contrib/cassandra-storage/pom.xml +++ b/extensions-contrib/cassandra-storage/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index c43135cf5811..33902392cfd8 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/distinctcount/pom.xml b/extensions-contrib/distinctcount/pom.xml index 02661447a1c0..417c16fbb464 100644 --- a/extensions-contrib/distinctcount/pom.xml +++ b/extensions-contrib/distinctcount/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/druid-rocketmq/pom.xml b/extensions-contrib/druid-rocketmq/pom.xml index 15e587fab777..6647eae9f95f 100644 --- a/extensions-contrib/druid-rocketmq/pom.xml +++ b/extensions-contrib/druid-rocketmq/pom.xml @@ -22,7 +22,7 @@ druid io.druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/google-extensions/pom.xml b/extensions-contrib/google-extensions/pom.xml index 87f53b21cd40..b7c21eff0530 100644 --- a/extensions-contrib/google-extensions/pom.xml +++ b/extensions-contrib/google-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index 4e376f1a0e06..9db9405886ef 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml index 0b3b7d0bbc08..087323589c86 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml +++ b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index 5060090e20e4..d631f6d25595 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/orc-extensions/pom.xml b/extensions-contrib/orc-extensions/pom.xml index db73f8ac1648..d1d62c6e71b4 100644 --- a/extensions-contrib/orc-extensions/pom.xml +++ b/extensions-contrib/orc-extensions/pom.xml @@ -24,7 +24,7 @@ druid io.druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/parquet-extensions/pom.xml b/extensions-contrib/parquet-extensions/pom.xml index 12d911688f23..117b523ce3ec 100644 --- a/extensions-contrib/parquet-extensions/pom.xml +++ b/extensions-contrib/parquet-extensions/pom.xml @@ -8,7 +8,7 @@ druid io.druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/rabbitmq/pom.xml b/extensions-contrib/rabbitmq/pom.xml index 89c5d793220e..f16a43044358 100644 --- a/extensions-contrib/rabbitmq/pom.xml +++ b/extensions-contrib/rabbitmq/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/redis-cache/pom.xml b/extensions-contrib/redis-cache/pom.xml index 6d137d1b13ae..e500311159b0 100644 --- a/extensions-contrib/redis-cache/pom.xml +++ b/extensions-contrib/redis-cache/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/sqlserver-metadata-storage/pom.xml b/extensions-contrib/sqlserver-metadata-storage/pom.xml index 71f866e166c9..d38b04b66991 100644 --- a/extensions-contrib/sqlserver-metadata-storage/pom.xml +++ b/extensions-contrib/sqlserver-metadata-storage/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index dd5e427774da..673fa9bca873 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index f3dcc234389b..f23d4c132780 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -9,7 +9,7 @@ druid io.druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/time-min-max/pom.xml b/extensions-contrib/time-min-max/pom.xml index 6962178a6b06..6ad82d3eddf8 100644 --- a/extensions-contrib/time-min-max/pom.xml +++ b/extensions-contrib/time-min-max/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/virtual-columns/pom.xml b/extensions-contrib/virtual-columns/pom.xml index fbff5bb5daf2..269702e5476f 100644 --- a/extensions-contrib/virtual-columns/pom.xml +++ b/extensions-contrib/virtual-columns/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index 6ed3a0be8ed4..c5c3f582ef97 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index 9db0c072e425..bf5f0982343c 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-basic-security/pom.xml b/extensions-core/druid-basic-security/pom.xml index a108ef43a771..fe0c45983c78 100644 --- a/extensions-core/druid-basic-security/pom.xml +++ b/extensions-core/druid-basic-security/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index c843981df80f..226f8d3b89e4 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index 2e9df56000f7..c874c1d56ed9 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/histogram/pom.xml b/extensions-core/histogram/pom.xml index 1a6a2717abae..bfb960160a05 100644 --- a/extensions-core/histogram/pom.xml +++ b/extensions-core/histogram/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-eight/pom.xml b/extensions-core/kafka-eight/pom.xml index d5454d114cd6..511bba3f7d15 100644 --- a/extensions-core/kafka-eight/pom.xml +++ b/extensions-core/kafka-eight/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index 1421594a0b9d..8d0b7094f636 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index c7fcd279decc..11a806c78f6a 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/lookups-cached-global/pom.xml b/extensions-core/lookups-cached-global/pom.xml index 67c430e8b92e..7bdc3cb77ff6 100644 --- a/extensions-core/lookups-cached-global/pom.xml +++ b/extensions-core/lookups-cached-global/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/lookups-cached-single/pom.xml b/extensions-core/lookups-cached-single/pom.xml index c710a36f8a96..e6fc760e14df 100644 --- a/extensions-core/lookups-cached-single/pom.xml +++ b/extensions-core/lookups-cached-single/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index d3990f495de9..3c33d71c7fa6 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml index c9250ba2ee5e..36529ed36196 100644 --- a/extensions-core/postgresql-metadata-storage/pom.xml +++ b/extensions-core/postgresql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/protobuf-extensions/pom.xml b/extensions-core/protobuf-extensions/pom.xml index abf49ec4c8c2..053058eb53b8 100644 --- a/extensions-core/protobuf-extensions/pom.xml +++ b/extensions-core/protobuf-extensions/pom.xml @@ -27,7 +27,7 @@ druid io.druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index 2cbc12861418..744a00589443 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/simple-client-sslcontext/pom.xml b/extensions-core/simple-client-sslcontext/pom.xml index 09abba31926b..26a6f093fc3f 100644 --- a/extensions-core/simple-client-sslcontext/pom.xml +++ b/extensions-core/simple-client-sslcontext/pom.xml @@ -3,7 +3,7 @@ druid io.druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml index 7bb7f41fc8af..82f9b592f43f 100644 --- a/extensions-core/stats/pom.xml +++ b/extensions-core/stats/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT ../../pom.xml diff --git a/hll/pom.xml b/hll/pom.xml index 4e3ce5533487..77abf7cf0e9a 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT druid-hll diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 1e47579584ec..79c722fc8b89 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index cde434ab15df..994dd25b3d1e 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index e93c2f49869a..6496b9d244ea 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT diff --git a/java-util/pom.xml b/java-util/pom.xml index 4ca29450b5f2..0dad4e22d814 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT java-util diff --git a/pom.xml b/pom.xml index 92884c6a59d9..70f2e7a96655 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT pom ${project.groupId}:${project.artifactId} @@ -53,7 +53,7 @@ scm:git:ssh://git@github.com/druid-io/druid.git scm:git:ssh://git@github.com/druid-io/druid.git https://github.com/druid-io/druid.git - druid-0.12.1-rc1 + 0.12.0-SNAPSHOT diff --git a/processing/pom.xml b/processing/pom.xml index 41eaaee05e2d..f8745662af84 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 0246bf3054ba..feeb71dd0cd9 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index 32bd7da6cc78..1b2185343fb0 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -25,7 +25,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT diff --git a/sql/pom.xml b/sql/pom.xml index 9c6a9b53fbb0..85be9ec6b139 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc1 + 0.12.1-rc2-SNAPSHOT From 7dc4f4b6b4b71cd16475aa1973ac357468f3a331 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 8 May 2018 13:36:13 -0700 Subject: [PATCH 70/82] [Backport] Fix Appenderator.push() to commit the metadata of all segments (#5756) --- .../realtime/appenderator/Appenderator.java | 50 ++++++------------- .../appenderator/AppenderatorImpl.java | 15 ++---- .../appenderator/AppenderatorTest.java | 11 ++-- .../StreamAppenderatorDriverFailTest.java | 8 ++- 4 files changed, 29 insertions(+), 55 deletions(-) diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderator.java b/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderator.java index 51708f4f31cd..7f06f86269e3 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderator.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/Appenderator.java @@ -36,9 +36,9 @@ * both of those. It can also push data to deep storage. But, it does not decide which segments data should go into. * It also doesn't publish segments to the metadata store or monitor handoff; you have to do that yourself! *

        - * You can provide a {@link Committer} or a Supplier of one when you call one of the methods that adds, persists, or - * pushes data. The Committer should represent all data you have given to the Appenderator so far. This Committer will - * be used when that data has been persisted to disk. + * You can provide a {@link Committer} or a Supplier of one when you call one of the methods that {@link #add}, + * {@link #persistAll}, or {@link #push}. The Committer should represent all data you have given to the Appenderator so + * far. This Committer will be used when that data has been persisted to disk. */ public interface Appenderator extends QuerySegmentWalker, Closeable { @@ -73,8 +73,9 @@ default AppenderatorAddResult add(SegmentIdentifier identifier, InputRow row, Su * Committer is guaranteed to be *created* synchronously with the call to add, but will actually be used * asynchronously. *

        - * If committer is not provided, no metadata is persisted. If it's provided, the add, clear, persist, persistAll, - * and push methods should all be called from the same thread to keep the metadata committed by Committer in sync. + * If committer is not provided, no metadata is persisted. If it's provided, {@link #add}, {@link #clear}, + * {@link #persistAll}, and {@link #push} methods should all be called from the same thread to keep the metadata + * committed by Committer in sync. * * @param identifier the segment into which this row should be added * @param row the row to add @@ -128,8 +129,8 @@ AppenderatorAddResult add( * for some reason, rows have been added that we do not actually want to hand off. Blocks until all data has been * cleared. This may take some time, since all pending persists must finish first. *

        - * The add, clear, persist, persistAll, and push methods should all be called from the same thread to keep the - * metadata committed by Committer in sync. + * {@link #add}, {@link #clear}, {@link #persistAll}, and {@link #push} methods should all be called from the same + * thread to keep the metadata committed by Committer in sync. */ void clear() throws InterruptedException; @@ -147,50 +148,31 @@ AppenderatorAddResult add( */ ListenableFuture drop(SegmentIdentifier identifier); - /** - * Persist any in-memory indexed data for segments of the given identifiers to durable storage. This may be only - * somewhat durable, e.g. the machine's local disk. The Committer will be made synchronously with the call to - * persist, but will actually be used asynchronously. Any metadata returned by the committer will be associated with - * the data persisted to disk. - *

        - * If committer is not provided, no metadata is persisted. If it's provided, the add, clear, persist, persistAll, - * and push methods should all be called from the same thread to keep the metadata committed by Committer in sync. - * - * @param identifiers segment identifiers to be persisted - * @param committer a committer associated with all data that has been added to segments of the given identifiers so - * far - * - * @return future that resolves when all pending data to segments of the identifiers has been persisted, contains - * commit metadata for this persist - */ - ListenableFuture persist(Collection identifiers, @Nullable Committer committer); - /** * Persist any in-memory indexed data to durable storage. This may be only somewhat durable, e.g. the * machine's local disk. The Committer will be made synchronously with the call to persistAll, but will actually * be used asynchronously. Any metadata returned by the committer will be associated with the data persisted to * disk. *

        - * If committer is not provided, no metadata is persisted. If it's provided, the add, clear, persist, persistAll, - * and push methods should all be called from the same thread to keep the metadata committed by Committer in sync. + * If committer is not provided, no metadata is persisted. If it's provided, {@link #add}, {@link #clear}, + * {@link #persistAll}, and {@link #push} methods should all be called from the same thread to keep the metadata + * committed by Committer in sync. * * @param committer a committer associated with all data that has been added so far * * @return future that resolves when all pending data has been persisted, contains commit metadata for this persist */ - default ListenableFuture persistAll(@Nullable Committer committer) - { - return persist(getSegments(), committer); - } + ListenableFuture persistAll(@Nullable Committer committer); /** * Merge and push particular segments to deep storage. This will trigger an implicit - * {@link #persist(Collection, Committer)} using the provided Committer. + * {@link #persistAll(Committer)} using the provided Committer. *

        * After this method is called, you cannot add new data to any segments that were previously under construction. *

        - * If committer is not provided, no metadata is persisted. If it's provided, the add, clear, persist, persistAll, - * and push methods should all be called from the same thread to keep the metadata committed by Committer in sync. + * If committer is not provided, no metadata is persisted. If it's provided, {@link #add}, {@link #clear}, + * {@link #persistAll}, and {@link #push} methods should all be called from the same thread to keep the metadata + * committed by Committer in sync. * * @param identifiers list of segments to push * @param committer a committer associated with all data that has been added so far diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java index 3d872e25e6af..70c22187d8dd 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/AppenderatorImpl.java @@ -389,12 +389,12 @@ public ListenableFuture drop(final SegmentIdentifier identifier) } @Override - public ListenableFuture persist(Collection identifiers, @Nullable Committer committer) + public ListenableFuture persistAll(@Nullable final Committer committer) { final Map currentHydrants = Maps.newHashMap(); final List> indexesToPersist = Lists.newArrayList(); int numPersistedRows = 0; - for (SegmentIdentifier identifier : identifiers) { + for (SegmentIdentifier identifier : sinks.keySet()) { final Sink sink = sinks.get(identifier); if (sink == null) { throw new ISE("No sink for identifier: %s", identifier); @@ -496,13 +496,6 @@ public Object doCall() return future; } - @Override - public ListenableFuture persistAll(@Nullable final Committer committer) - { - // Submit persistAll task to the persistExecutor - return persist(sinks.keySet(), committer); - } - @Override public ListenableFuture push( final Collection identifiers, @@ -521,7 +514,9 @@ public ListenableFuture push( } return Futures.transform( - persist(identifiers, committer), + // We should always persist all segments regardless of the input because metadata should be committed for all + // segments. + persistAll(committer), (Function) commitMetadata -> { final List dataSegments = Lists.newArrayList(); diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTest.java index 85bcd22ced33..17b32218fad9 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTest.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/AppenderatorTest.java @@ -191,10 +191,9 @@ public void run() Assert.assertEquals(1, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "bob", 1), committerSupplier); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); - appenderator.persist(ImmutableList.of(IDENTIFIERS.get(1)), committerSupplier.get()); - Assert.assertEquals(1, ((AppenderatorImpl) appenderator).getRowsInMemory()); - appenderator.close(); + appenderator.persistAll(committerSupplier.get()); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); + appenderator.close(); } } @@ -238,12 +237,12 @@ public void run() Assert.assertEquals(4, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.add(IDENTIFIERS.get(0), IR("2000", "bob", 1), committerSupplier, false); Assert.assertEquals(5, ((AppenderatorImpl) appenderator).getRowsInMemory()); - appenderator.persist(ImmutableList.of(IDENTIFIERS.get(1)), committerSupplier.get()); - Assert.assertEquals(3, ((AppenderatorImpl) appenderator).getRowsInMemory()); - appenderator.close(); + appenderator.persistAll(committerSupplier.get()); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); + appenderator.close(); } } + @Test public void testRestoreFromDisk() throws Exception { diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverFailTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverFailTest.java index a89d9565c8c8..9535bb3868da 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverFailTest.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriverFailTest.java @@ -454,15 +454,13 @@ public ListenableFuture drop(SegmentIdentifier identifier) } @Override - public ListenableFuture persist( - Collection identifiers, Committer committer - ) + public ListenableFuture persistAll(Committer committer) { if (persistEnabled) { // do nothing return Futures.immediateFuture(committer.getMetadata()); } else { - return Futures.immediateFailedFuture(new ISE("Fail test while persisting segments[%s]", identifiers)); + return Futures.immediateFailedFuture(new ISE("Fail test while persisting segments[%s]", rows.keySet())); } } @@ -488,7 +486,7 @@ public ListenableFuture push( ) .collect(Collectors.toList()); return Futures.transform( - persist(identifiers, committer), + persistAll(committer), (Function) commitMetadata -> new SegmentsAndMetadata(segments, commitMetadata) ); } else { From 8b7a2da1940a3ec191d20dd126bb3d862a6ea8ae Mon Sep 17 00:00:00 2001 From: Slim Bouguerra Date: Tue, 8 May 2018 14:49:46 -0700 Subject: [PATCH 71/82] Kerberos Spnego Authentication Router Issue (#5706) (#5757) * Adding decoration method to proxy servlet Change-Id: I872f9282fb60bfa20524271535980a36a87b9621 * moving the proxy request decoration to authenticators Change-Id: I7f94b9ff5ecf08e8abf7169b58bc410f33148448 * added docs Change-Id: I901543e52f0faf4666bfea6256a7c05593b1ae70 * use the authentication result to decorate request Change-Id: I052650de9cd02b4faefdbcdaf2332dd3b2966af5 * adding authenticated by name Change-Id: I074d2933460165feeddb19352eac9bd0f96f42ca * ensure that authenticator is not null Change-Id: Idb58e308f90db88224a06f3759114872165b24f5 * fix types and minor bug Change-Id: I6801d49a05d5d8324406fc0280286954eb66db10 * fix typo Change-Id: I390b12af74f44d760d0812a519125fbf0df4e97b * use actual type names Change-Id: I62c3ee763363781e52809ec912aafd50b8486b8e * set authenitcatedBy to null for AutheticationResults created by Escalator. Change-Id: I4a675c372f59ebd8a8d19c61b85a1e4bf227a8ba --- .../BasicHTTPAuthenticator.java | 4 +- .../authentication/BasicHTTPEscalator.java | 4 +- .../BasicRoleBasedAuthorizerTest.java | 2 +- .../kerberos/KerberosAuthenticator.java | 111 +++++++++++++++++- .../security/kerberos/KerberosEscalator.java | 7 +- .../overlord/http/OverlordResourceTest.java | 2 +- .../indexing/overlord/http/OverlordTest.java | 2 +- .../supervisor/SupervisorResourceTest.java | 12 +- .../server/AsyncQueryForwardingServlet.java | 24 +++- .../security/AllowAllAuthenticator.java | 2 +- .../security/AllowOptionsResourceFilter.java | 2 +- .../server/security/AuthenticationResult.java | 16 +++ .../druid/server/security/Authenticator.java | 22 ++++ .../io/druid/server/security/Escalator.java | 1 + .../security/UnsecuredResourceFilter.java | 2 +- .../AsyncQueryForwardingServletTest.java | 7 +- .../io/druid/server/QueryResourceTest.java | 2 +- .../server/http/DatasourcesResourceTest.java | 8 +- .../server/http/IntervalsResourceTest.java | 8 +- ...eResponseAuthorizationCheckFilterTest.java | 6 +- .../security/ResourceFilterTestHelper.java | 2 +- .../SecuritySanityCheckFilterTest.java | 4 +- .../druid/sql/calcite/util/CalciteTests.java | 6 +- 23 files changed, 216 insertions(+), 40 deletions(-) diff --git a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BasicHTTPAuthenticator.java b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BasicHTTPAuthenticator.java index b127687999bd..eeb406ee681b 100644 --- a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BasicHTTPAuthenticator.java +++ b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BasicHTTPAuthenticator.java @@ -105,7 +105,7 @@ public AuthenticationResult authenticateJDBCContext(Map context) } if (checkCredentials(user, password.toCharArray())) { - return new AuthenticationResult(user, name, null); + return new AuthenticationResult(user, authorizerName, name, null); } else { return null; } @@ -173,7 +173,7 @@ public void doFilter( char[] password = splits[1].toCharArray(); if (checkCredentials(user, password)) { - AuthenticationResult authenticationResult = new AuthenticationResult(user, authorizerName, null); + AuthenticationResult authenticationResult = new AuthenticationResult(user, authorizerName, name, null); servletRequest.setAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT, authenticationResult); } diff --git a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BasicHTTPEscalator.java b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BasicHTTPEscalator.java index 45e3a5db7bf6..d2993f6509df 100644 --- a/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BasicHTTPEscalator.java +++ b/extensions-core/druid-basic-security/src/main/java/io/druid/security/basic/authentication/BasicHTTPEscalator.java @@ -59,6 +59,8 @@ public HttpClient createEscalatedClient(HttpClient baseClient) @Override public AuthenticationResult createEscalatedAuthenticationResult() { - return new AuthenticationResult(internalClientUsername, authorizerName, null); + // if you found your self asking why the authenticatedBy field is set to null please read this: + // https://github.com/druid-io/druid/pull/5706#discussion_r185940889 + return new AuthenticationResult(internalClientUsername, authorizerName, null, null); } } diff --git a/extensions-core/druid-basic-security/src/test/java/io/druid/security/authorization/BasicRoleBasedAuthorizerTest.java b/extensions-core/druid-basic-security/src/test/java/io/druid/security/authorization/BasicRoleBasedAuthorizerTest.java index 6132753c684d..aafac305525a 100644 --- a/extensions-core/druid-basic-security/src/test/java/io/druid/security/authorization/BasicRoleBasedAuthorizerTest.java +++ b/extensions-core/druid-basic-security/src/test/java/io/druid/security/authorization/BasicRoleBasedAuthorizerTest.java @@ -116,7 +116,7 @@ public void testAuth() updater.setPermissions(AUTHORIZER_NAME, "druidRole", permissions); - AuthenticationResult authenticationResult = new AuthenticationResult("druid", "druid", null); + AuthenticationResult authenticationResult = new AuthenticationResult("druid", "druid", null, null); Access access = authorizer.authorize( authenticationResult, diff --git a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java index d0b7bcaabfbd..6b5d044f26a3 100644 --- a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java +++ b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java @@ -23,6 +23,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import io.druid.guice.annotations.Self; import io.druid.java.util.common.StringUtils; @@ -41,6 +43,8 @@ import org.apache.hadoop.security.authentication.util.Signer; import org.apache.hadoop.security.authentication.util.SignerException; import org.apache.hadoop.security.authentication.util.SignerSecretProvider; +import org.eclipse.jetty.client.api.Request; +import org.eclipse.jetty.http.HttpHeader; import sun.security.krb5.EncryptedData; import sun.security.krb5.EncryptionKey; import sun.security.krb5.internal.APReq; @@ -72,18 +76,25 @@ import javax.servlet.http.HttpServletResponse; import java.io.File; import java.io.IOException; +import java.net.HttpCookie; import java.security.Principal; +import java.text.SimpleDateFormat; import java.util.Collections; +import java.util.Date; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.Random; import java.util.Set; +import java.util.TimeZone; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; + @JsonTypeName("kerberos") public class KerberosAuthenticator implements Authenticator @@ -91,6 +102,7 @@ public class KerberosAuthenticator implements Authenticator private static final Logger log = new Logger(KerberosAuthenticator.class); private static final Pattern HADOOP_AUTH_COOKIE_REGEX = Pattern.compile(".*p=(\\S+)&t=.*"); public static final List DEFAULT_EXCLUDED_PATHS = Collections.emptyList(); + public static final String SIGNED_TOKEN_ATTRIBUTE = "signedToken"; private final DruidNode node; private final String serverPrincipal; @@ -99,6 +111,7 @@ public class KerberosAuthenticator implements Authenticator private final List excludedPaths; private final String cookieSignatureSecret; private final String authorizerName; + private final String name; private LoginContext loginContext; @JsonCreator @@ -109,6 +122,7 @@ public KerberosAuthenticator( @JsonProperty("excludedPaths") List excludedPaths, @JsonProperty("cookieSignatureSecret") String cookieSignatureSecret, @JsonProperty("authorizerName") String authorizerName, + @JsonProperty("name") String name, @JacksonInject @Self DruidNode node ) { @@ -119,6 +133,7 @@ public KerberosAuthenticator( this.excludedPaths = excludedPaths == null ? DEFAULT_EXCLUDED_PATHS : excludedPaths; this.cookieSignatureSecret = cookieSignatureSecret; this.authorizerName = authorizerName; + this.name = Preconditions.checkNotNull(name); } @Override @@ -253,7 +268,7 @@ public void doFilter( if (clientPrincipal != null) { request.setAttribute( AuthConfig.DRUID_AUTHENTICATION_RESULT, - new AuthenticationResult(clientPrincipal, authorizerName, null) + new AuthenticationResult(clientPrincipal, authorizerName, name, null) ); } } @@ -327,11 +342,19 @@ public Principal getUserPrincipal() createAuthCookie(httpResponse, signedToken, getCookieDomain(), getCookiePath(), token.getExpires(), isHttps ); + request.setAttribute(SIGNED_TOKEN_ATTRIBUTE, tokenToCookieString( + signedToken, + getCookieDomain(), + getCookiePath(), + token.getExpires(), + !token.isExpired() && token.getExpires() > 0, + isHttps + )); } // Since this request is validated also set DRUID_AUTHENTICATION_RESULT request.setAttribute( AuthConfig.DRUID_AUTHENTICATION_RESULT, - new AuthenticationResult(token.getName(), authorizerName, null) + new AuthenticationResult(token.getName(), authorizerName, name, null) ); doFilter(filterChain, httpRequest, httpResponse); } @@ -344,7 +367,7 @@ public Principal getUserPrincipal() errCode = HttpServletResponse.SC_FORBIDDEN; authenticationEx = ex; if (log.isDebugEnabled()) { - log.debug("Authentication exception: " + ex.getMessage(), ex); + log.debug(ex, "Authentication exception: " + ex.getMessage()); } else { log.warn("Authentication exception: " + ex.getMessage()); } @@ -439,6 +462,22 @@ private boolean isExcluded(String path) return false; } + @Override + public void decorateProxyRequest( + HttpServletRequest clientRequest, HttpServletResponse proxyResponse, Request proxyRequest + ) + { + Object cookieToken = clientRequest.getAttribute(SIGNED_TOKEN_ATTRIBUTE); + if (cookieToken != null && cookieToken instanceof String) { + log.debug("Found cookie token will attache it to proxyRequest as cookie"); + String authResult = (String) cookieToken; + String existingCookies = proxyRequest.getCookies() + .stream() + .map(HttpCookie::toString) + .collect(Collectors.joining(";")); + proxyRequest.header(HttpHeader.COOKIE, Joiner.on(";").join(authResult, existingCookies)); + } + } /** * Kerberos context configuration for the JDK GSS library. Copied from hadoop-auth's KerberosAuthenticationHandler. @@ -604,4 +643,70 @@ private void initializeKerberosLogin() throws ServletException throw new ServletException(ex); } } + + /** + * Creates the Hadoop authentication HTTP cookie. + * + * @param resp the response object. + * @param token authentication token for the cookie. + * @param domain the cookie domain. + * @param path the cookie path. + * @param expires UNIX timestamp that indicates the expire date of the + * cookie. It has no effect if its value < 0. + * @param isSecure is the cookie secure? + * @param isCookiePersistent whether the cookie is persistent or not. + *the following code copy/past from Hadoop 3.0.0 copied to avoid compilation issue due to new signature, + * org.apache.hadoop.security.authentication.server.AuthenticationFilter#createAuthCookie + * ( + * javax.servlet.http.HttpServletResponse, + * java.lang.String, + * java.lang.String, + * java.lang.String, + * long, boolean, boolean) + */ + private static void tokenToAuthCookie( + HttpServletResponse resp, String token, + String domain, String path, long expires, + boolean isCookiePersistent, + boolean isSecure + ) + { + resp.addHeader("Set-Cookie", tokenToCookieString(token, domain, path, expires, isCookiePersistent, isSecure)); + } + + private static String tokenToCookieString( + String token, + String domain, String path, long expires, + boolean isCookiePersistent, + boolean isSecure + ) + { + StringBuilder sb = new StringBuilder(AuthenticatedURL.AUTH_COOKIE) + .append("="); + if (token != null && token.length() > 0) { + sb.append("\"").append(token).append("\""); + } + + if (path != null) { + sb.append("; Path=").append(path); + } + + if (domain != null) { + sb.append("; Domain=").append(domain); + } + + if (expires >= 0 && isCookiePersistent) { + Date date = new Date(expires); + SimpleDateFormat df = new SimpleDateFormat("EEE, dd-MMM-yyyy HH:mm:ss zzz", Locale.ENGLISH); + df.setTimeZone(TimeZone.getTimeZone("GMT")); + sb.append("; Expires=").append(df.format(date)); + } + + if (isSecure) { + sb.append("; Secure"); + } + + sb.append("; HttpOnly"); + return sb.toString(); + } } diff --git a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosEscalator.java b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosEscalator.java index 808e1169b64b..3b25f5e8af2e 100644 --- a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosEscalator.java +++ b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosEscalator.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; -import io.druid.java.util.http.client.HttpClient; import io.druid.java.util.common.logger.Logger; +import io.druid.java.util.http.client.HttpClient; import io.druid.server.security.AuthenticationResult; import io.druid.server.security.Escalator; @@ -57,6 +57,9 @@ public HttpClient createEscalatedClient(HttpClient baseClient) @Override public AuthenticationResult createEscalatedAuthenticationResult() { - return new AuthenticationResult(internalClientPrincipal, authorizerName, null); + // if you found your self asking why the authenticatedBy field is set to null please read this: + // https://github.com/druid-io/druid/pull/5706#discussion_r185940889 + return new AuthenticationResult(internalClientPrincipal, authorizerName, null, null); } + } diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/http/OverlordResourceTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/http/OverlordResourceTest.java index 78242e8f8603..22b9e512aaa6 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/http/OverlordResourceTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/http/OverlordResourceTest.java @@ -120,8 +120,8 @@ public Access authorize(AuthenticationResult authenticationResult, Resource reso public void expectAuthorizationTokenCheck() { - AuthenticationResult authenticationResult = new AuthenticationResult("druid", "druid", null); + AuthenticationResult authenticationResult = new AuthenticationResult("druid", "druid", null, null); EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).anyTimes(); EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)) .andReturn(authenticationResult) diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/http/OverlordTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/http/OverlordTest.java index 2cf3462aaa6a..9d95c65afd09 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/http/OverlordTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/http/OverlordTest.java @@ -131,7 +131,7 @@ public void setUp() throws Exception req = EasyMock.createMock(HttpServletRequest.class); EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).anyTimes(); EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( - new AuthenticationResult("druid", "druid", null) + new AuthenticationResult("druid", "druid", null, null) ).anyTimes(); req.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); EasyMock.expectLastCall().anyTimes(); diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorResourceTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorResourceTest.java index 3c0a2bd1585c..cb72a1922058 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorResourceTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/supervisor/SupervisorResourceTest.java @@ -116,7 +116,7 @@ public List getDataSources() EasyMock.expect(supervisorManager.createOrUpdateAndStartSupervisor(spec)).andReturn(true); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).atLeastOnce(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( - new AuthenticationResult("druid", "druid", null) + new AuthenticationResult("druid", "druid", null, null) ).atLeastOnce(); request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); EasyMock.expectLastCall().anyTimes(); @@ -165,7 +165,7 @@ public List getDataSources() EasyMock.expect(supervisorManager.getSupervisorSpec("id2")).andReturn(Optional.of(spec2)); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).atLeastOnce(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( - new AuthenticationResult("druid", "druid", null) + new AuthenticationResult("druid", "druid", null, null) ).atLeastOnce(); request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); EasyMock.expectLastCall().anyTimes(); @@ -355,7 +355,7 @@ public void testSpecGetAllHistory() throws Exception EasyMock.expect(supervisorManager.getSupervisorSpec("id2")).andReturn(Optional.of(spec2)).atLeastOnce(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).atLeastOnce(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( - new AuthenticationResult("druid", "druid", null) + new AuthenticationResult("druid", "druid", null, null) ).atLeastOnce(); request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); EasyMock.expectLastCall().anyTimes(); @@ -467,7 +467,7 @@ public void testSpecGetAllHistoryWithAuthFailureFiltering() throws Exception EasyMock.expect(supervisorManager.getSupervisorSpec("id2")).andReturn(Optional.of(spec2)).atLeastOnce(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).atLeastOnce(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( - new AuthenticationResult("wronguser", "druid", null) + new AuthenticationResult("wronguser", "druid", null, null) ).atLeastOnce(); request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); EasyMock.expectLastCall().anyTimes(); @@ -557,7 +557,7 @@ public void testSpecGetHistory() throws Exception EasyMock.expect(supervisorManager.getSupervisorHistory()).andReturn(history).times(3); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).atLeastOnce(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( - new AuthenticationResult("druid", "druid", null) + new AuthenticationResult("druid", "druid", null, null) ).atLeastOnce(); request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); EasyMock.expectLastCall().anyTimes(); @@ -654,7 +654,7 @@ public void testSpecGetHistoryWithAuthFailure() throws Exception EasyMock.expect(supervisorManager.getSupervisorHistory()).andReturn(history).times(4); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).atLeastOnce(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( - new AuthenticationResult("notdruid", "druid", null) + new AuthenticationResult("notdruid", "druid", null, null) ).atLeastOnce(); request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); EasyMock.expectLastCall().anyTimes(); diff --git a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java index 927d9a3a2146..23d60b82e985 100644 --- a/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java +++ b/server/src/main/java/io/druid/server/AsyncQueryForwardingServlet.java @@ -45,6 +45,9 @@ import io.druid.server.router.QueryHostFinder; import io.druid.server.router.Router; import io.druid.server.security.AuthConfig; +import io.druid.server.security.AuthenticationResult; +import io.druid.server.security.Authenticator; +import io.druid.server.security.AuthenticatorMapper; import org.apache.http.client.utils.URIBuilder; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.api.Request; @@ -112,6 +115,7 @@ private static void handleException(HttpServletResponse response, ObjectMapper o private final ServiceEmitter emitter; private final RequestLogger requestLogger; private final GenericQueryMetricsFactory queryMetricsFactory; + private final AuthenticatorMapper authenticatorMapper; private HttpClient broadcastClient; @@ -125,7 +129,8 @@ public AsyncQueryForwardingServlet( @Router DruidHttpClientConfig httpClientConfig, ServiceEmitter emitter, RequestLogger requestLogger, - GenericQueryMetricsFactory queryMetricsFactory + GenericQueryMetricsFactory queryMetricsFactory, + AuthenticatorMapper authenticatorMapper ) { this.warehouse = warehouse; @@ -137,6 +142,7 @@ public AsyncQueryForwardingServlet( this.emitter = emitter; this.requestLogger = requestLogger; this.queryMetricsFactory = queryMetricsFactory; + this.authenticatorMapper = authenticatorMapper; } @Override @@ -313,6 +319,22 @@ protected void sendProxyRequest( // will log that on the remote node. clientRequest.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); + // Check if there is an authentication result and use it to decorate the proxy request if needed. + AuthenticationResult authenticationResult = (AuthenticationResult) clientRequest.getAttribute( + AuthConfig.DRUID_AUTHENTICATION_RESULT); + if (authenticationResult != null && authenticationResult.getAuthenticatedBy() != null) { + Authenticator authenticator = authenticatorMapper.getAuthenticatorMap() + .get(authenticationResult.getAuthenticatedBy()); + if (authenticator != null) { + authenticator.decorateProxyRequest( + clientRequest, + proxyResponse, + proxyRequest + ); + } else { + log.error("Can not find Authenticator with Name [%s]", authenticationResult.getAuthenticatedBy()); + } + } super.sendProxyRequest( clientRequest, proxyResponse, diff --git a/server/src/main/java/io/druid/server/security/AllowAllAuthenticator.java b/server/src/main/java/io/druid/server/security/AllowAllAuthenticator.java index c87d67546ae2..1911a570267b 100644 --- a/server/src/main/java/io/druid/server/security/AllowAllAuthenticator.java +++ b/server/src/main/java/io/druid/server/security/AllowAllAuthenticator.java @@ -38,7 +38,7 @@ public class AllowAllAuthenticator implements Authenticator public static final AuthenticationResult ALLOW_ALL_RESULT = new AuthenticationResult( AuthConfig.ALLOW_ALL_NAME, AuthConfig.ALLOW_ALL_NAME, - null + AuthConfig.ALLOW_ALL_NAME, null ); @Override diff --git a/server/src/main/java/io/druid/server/security/AllowOptionsResourceFilter.java b/server/src/main/java/io/druid/server/security/AllowOptionsResourceFilter.java index 9ec87415593a..1e8f488902b4 100644 --- a/server/src/main/java/io/druid/server/security/AllowOptionsResourceFilter.java +++ b/server/src/main/java/io/druid/server/security/AllowOptionsResourceFilter.java @@ -63,7 +63,7 @@ public void doFilter( if (allowUnauthenticatedHttpOptions) { httpReq.setAttribute( AuthConfig.DRUID_AUTHENTICATION_RESULT, - new AuthenticationResult(AuthConfig.ALLOW_ALL_NAME, AuthConfig.ALLOW_ALL_NAME, null) + new AuthenticationResult(AuthConfig.ALLOW_ALL_NAME, AuthConfig.ALLOW_ALL_NAME, null, null) ); } else { ((HttpServletResponse) response).sendError(HttpServletResponse.SC_UNAUTHORIZED); diff --git a/server/src/main/java/io/druid/server/security/AuthenticationResult.java b/server/src/main/java/io/druid/server/security/AuthenticationResult.java index bb9effdf8e05..a1a7d5d966ec 100644 --- a/server/src/main/java/io/druid/server/security/AuthenticationResult.java +++ b/server/src/main/java/io/druid/server/security/AuthenticationResult.java @@ -37,6 +37,15 @@ public class AuthenticationResult */ private final String authorizerName; + + /** + * Name of authenticator whom created the results + * + * If you found your self asking why the authenticatedBy field can be null please read this + * https://github.com/druid-io/druid/pull/5706#discussion_r185940889 + */ + @Nullable + private final String authenticatedBy; /** * parameter containing additional context information from an Authenticator */ @@ -46,11 +55,13 @@ public class AuthenticationResult public AuthenticationResult( final String identity, final String authorizerName, + final String authenticatedBy, final Map context ) { this.identity = identity; this.authorizerName = authorizerName; + this.authenticatedBy = authenticatedBy; this.context = context; } @@ -68,4 +79,9 @@ public Map getContext() { return context; } + + public String getAuthenticatedBy() + { + return authenticatedBy; + } } diff --git a/server/src/main/java/io/druid/server/security/Authenticator.java b/server/src/main/java/io/druid/server/security/Authenticator.java index 969b4497f2a5..1b81dcfc4737 100644 --- a/server/src/main/java/io/druid/server/security/Authenticator.java +++ b/server/src/main/java/io/druid/server/security/Authenticator.java @@ -22,9 +22,12 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.druid.server.initialization.jetty.ServletFilterHolder; +import org.eclipse.jetty.client.api.Request; import javax.annotation.Nullable; import javax.servlet.Filter; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.util.Map; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @@ -93,4 +96,23 @@ public interface Authenticator extends ServletFilterHolder */ @Nullable AuthenticationResult authenticateJDBCContext(Map context); + + + /** + * This is used to add some Headers or Authentication token/results that can be used by down stream target host. + * Such token can be used to authenticate the user down stream, in cases where to original credenitals + * are not forwardable as is and therefore the need to attach some authentication tokens by the proxy. + * + * @param clientRequest original client request processed by the upstream chain of authenticator + * @param proxyResponse proxy Response + * @param proxyRequest actual proxy request targeted to a given broker + */ + default void decorateProxyRequest( + final HttpServletRequest clientRequest, + final HttpServletResponse proxyResponse, + final Request proxyRequest + ) + { + //noop + } } diff --git a/server/src/main/java/io/druid/server/security/Escalator.java b/server/src/main/java/io/druid/server/security/Escalator.java index b8fe920f98fe..31a77a74d07b 100644 --- a/server/src/main/java/io/druid/server/security/Escalator.java +++ b/server/src/main/java/io/druid/server/security/Escalator.java @@ -50,4 +50,5 @@ public interface Escalator * @return an AuthenticationResult representing the identity of the internal system user. */ AuthenticationResult createEscalatedAuthenticationResult(); + } diff --git a/server/src/main/java/io/druid/server/security/UnsecuredResourceFilter.java b/server/src/main/java/io/druid/server/security/UnsecuredResourceFilter.java index dda859c1af9b..df741dccccb1 100644 --- a/server/src/main/java/io/druid/server/security/UnsecuredResourceFilter.java +++ b/server/src/main/java/io/druid/server/security/UnsecuredResourceFilter.java @@ -48,7 +48,7 @@ public void doFilter( // but the value doesn't matter since we skip authorization checks for requests that go through this filter servletRequest.setAttribute( AuthConfig.DRUID_AUTHENTICATION_RESULT, - new AuthenticationResult(AuthConfig.ALLOW_ALL_NAME, AuthConfig.ALLOW_ALL_NAME, null) + new AuthenticationResult(AuthConfig.ALLOW_ALL_NAME, AuthConfig.ALLOW_ALL_NAME, AuthConfig.ALLOW_ALL_NAME, null) ); // This request will not go to an Authorizer, so we need to set this for PreResponseAuthorizationCheckFilter diff --git a/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java b/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java index c587ad8759ef..19fb7509e8b1 100644 --- a/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java +++ b/server/src/test/java/io/druid/server/AsyncQueryForwardingServletTest.java @@ -54,6 +54,7 @@ import io.druid.server.router.QueryHostFinder; import io.druid.server.router.RendezvousHashAvaticaConnectionBalancer; import io.druid.server.security.AllowAllAuthorizer; +import io.druid.server.security.AuthenticatorMapper; import io.druid.server.security.Authorizer; import io.druid.server.security.AuthorizerMapper; import org.easymock.EasyMock; @@ -252,7 +253,8 @@ public int read() null, new NoopServiceEmitter(), requestLogLine -> { /* noop */ }, - new DefaultGenericQueryMetricsFactory(jsonMapper) + new DefaultGenericQueryMetricsFactory(jsonMapper), + new AuthenticatorMapper(ImmutableMap.of()) ) { @Override @@ -343,7 +345,8 @@ public Collection getAllServers() injector.getInstance(DruidHttpClientConfig.class), new NoopServiceEmitter(), requestLogLine -> { /* noop */ }, - new DefaultGenericQueryMetricsFactory(jsonMapper) + new DefaultGenericQueryMetricsFactory(jsonMapper), + new AuthenticatorMapper(ImmutableMap.of()) ) { @Override diff --git a/server/src/test/java/io/druid/server/QueryResourceTest.java b/server/src/test/java/io/druid/server/QueryResourceTest.java index 945aa7ec39b7..2391a718a886 100644 --- a/server/src/test/java/io/druid/server/QueryResourceTest.java +++ b/server/src/test/java/io/druid/server/QueryResourceTest.java @@ -80,7 +80,7 @@ public class QueryResourceTest { private static final QueryToolChestWarehouse warehouse = new MapQueryToolChestWarehouse(ImmutableMap., QueryToolChest>of()); private static final ObjectMapper jsonMapper = new DefaultObjectMapper(); - private static final AuthenticationResult authenticationResult = new AuthenticationResult("druid", "druid", null); + private static final AuthenticationResult authenticationResult = new AuthenticationResult("druid", "druid", null, null); private final HttpServletRequest testServletRequest = EasyMock.createMock(HttpServletRequest.class); public static final QuerySegmentWalker testSegmentWalker = new QuerySegmentWalker() diff --git a/server/src/test/java/io/druid/server/http/DatasourcesResourceTest.java b/server/src/test/java/io/druid/server/http/DatasourcesResourceTest.java index 1b97a2807b49..d4b1462cff76 100644 --- a/server/src/test/java/io/druid/server/http/DatasourcesResourceTest.java +++ b/server/src/test/java/io/druid/server/http/DatasourcesResourceTest.java @@ -130,7 +130,7 @@ public void testGetFullQueryableDataSources() throws Exception ).once(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).once(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( - new AuthenticationResult("druid", "druid", null) + new AuthenticationResult("druid", "druid", null, null) ).atLeastOnce(); request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); EasyMock.expectLastCall().times(1); @@ -144,7 +144,7 @@ public void testGetFullQueryableDataSources() throws Exception ).once(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).once(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( - new AuthenticationResult("druid", "druid", null) + new AuthenticationResult("druid", "druid", null, null) ).once(); request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); EasyMock.expectLastCall().times(1); @@ -178,7 +178,7 @@ public void testGetFullQueryableDataSources() throws Exception @Test public void testSecuredGetFullQueryableDataSources() throws Exception { - AuthenticationResult authenticationResult = new AuthenticationResult("druid", "druid", null); + AuthenticationResult authenticationResult = new AuthenticationResult("druid", "druid", null, null); // first request EasyMock.expect(server.getDataSources()).andReturn( ImmutableList.of(listDataSources.get(0), listDataSources.get(1)) @@ -279,7 +279,7 @@ public void testGetSimpleQueryableDataSources() throws Exception ).atLeastOnce(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).once(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( - new AuthenticationResult("druid", "druid", null) + new AuthenticationResult("druid", "druid", null, null) ).atLeastOnce(); request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); EasyMock.expectLastCall().times(1); diff --git a/server/src/test/java/io/druid/server/http/IntervalsResourceTest.java b/server/src/test/java/io/druid/server/http/IntervalsResourceTest.java index 067bf869d1b6..530a2a10232a 100644 --- a/server/src/test/java/io/druid/server/http/IntervalsResourceTest.java +++ b/server/src/test/java/io/druid/server/http/IntervalsResourceTest.java @@ -110,7 +110,7 @@ public void testGetIntervals() ).atLeastOnce(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).once(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( - new AuthenticationResult("druid", "druid", null) + new AuthenticationResult("druid", "druid", null, null) ).once(); request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); EasyMock.expectLastCall().times(1); @@ -147,7 +147,7 @@ public void testSimpleGetSpecificIntervals() ).atLeastOnce(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).once(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( - new AuthenticationResult("druid", "druid", null) + new AuthenticationResult("druid", "druid", null, null) ).once(); request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); EasyMock.expectLastCall().times(1); @@ -178,7 +178,7 @@ public void testFullGetSpecificIntervals() ).atLeastOnce(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).once(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( - new AuthenticationResult("druid", "druid", null) + new AuthenticationResult("druid", "druid", null, null) ).once(); request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); EasyMock.expectLastCall().times(1); @@ -211,7 +211,7 @@ public void testGetSpecificIntervals() ).atLeastOnce(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).once(); EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn( - new AuthenticationResult("druid", "druid", null) + new AuthenticationResult("druid", "druid", null, null) ).once(); request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true); EasyMock.expectLastCall().times(1); diff --git a/server/src/test/java/io/druid/server/http/security/PreResponseAuthorizationCheckFilterTest.java b/server/src/test/java/io/druid/server/http/security/PreResponseAuthorizationCheckFilterTest.java index c027bde092c5..e4f7f1a6d7a0 100644 --- a/server/src/test/java/io/druid/server/http/security/PreResponseAuthorizationCheckFilterTest.java +++ b/server/src/test/java/io/druid/server/http/security/PreResponseAuthorizationCheckFilterTest.java @@ -50,7 +50,7 @@ public class PreResponseAuthorizationCheckFilterTest @Test public void testValidRequest() throws Exception { - AuthenticationResult authenticationResult = new AuthenticationResult("so-very-valid", "so-very-valid", null); + AuthenticationResult authenticationResult = new AuthenticationResult("so-very-valid", "so-very-valid", null, null); HttpServletRequest req = EasyMock.createStrictMock(HttpServletRequest.class); HttpServletResponse resp = EasyMock.createStrictMock(HttpServletResponse.class); @@ -103,7 +103,7 @@ public void testMissingAuthorizationCheck() throws Exception expectedException.expect(ISE.class); expectedException.expectMessage("Request did not have an authorization check performed."); - AuthenticationResult authenticationResult = new AuthenticationResult("so-very-valid", "so-very-valid", null); + AuthenticationResult authenticationResult = new AuthenticationResult("so-very-valid", "so-very-valid", null, null); HttpServletRequest req = EasyMock.createStrictMock(HttpServletRequest.class); HttpServletResponse resp = EasyMock.createStrictMock(HttpServletResponse.class); @@ -138,7 +138,7 @@ public void testMissingAuthorizationCheck() throws Exception public void testMissingAuthorizationCheckWithError() throws Exception { EmittingLogger.registerEmitter(EasyMock.createNiceMock(ServiceEmitter.class)); - AuthenticationResult authenticationResult = new AuthenticationResult("so-very-valid", "so-very-valid", null); + AuthenticationResult authenticationResult = new AuthenticationResult("so-very-valid", "so-very-valid", null, null); HttpServletRequest req = EasyMock.createStrictMock(HttpServletRequest.class); HttpServletResponse resp = EasyMock.createStrictMock(HttpServletResponse.class); diff --git a/server/src/test/java/io/druid/server/http/security/ResourceFilterTestHelper.java b/server/src/test/java/io/druid/server/http/security/ResourceFilterTestHelper.java index 4e66f72d5664..240785c19c2e 100644 --- a/server/src/test/java/io/druid/server/http/security/ResourceFilterTestHelper.java +++ b/server/src/test/java/io/druid/server/http/security/ResourceFilterTestHelper.java @@ -111,7 +111,7 @@ public MultivaluedMap getMatrixParameters() ).anyTimes(); EasyMock.expect(request.getMethod()).andReturn(requestMethod).anyTimes(); EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).anyTimes(); - AuthenticationResult authenticationResult = new AuthenticationResult("druid", "druid", null); + AuthenticationResult authenticationResult = new AuthenticationResult("druid", "druid", null, null); EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)) .andReturn(authenticationResult) .atLeastOnce(); diff --git a/server/src/test/java/io/druid/server/http/security/SecuritySanityCheckFilterTest.java b/server/src/test/java/io/druid/server/http/security/SecuritySanityCheckFilterTest.java index 891afc912343..30f88bad9ac4 100644 --- a/server/src/test/java/io/druid/server/http/security/SecuritySanityCheckFilterTest.java +++ b/server/src/test/java/io/druid/server/http/security/SecuritySanityCheckFilterTest.java @@ -58,7 +58,9 @@ public void testInvalidRequest() throws Exception FilterChain filterChain = EasyMock.createStrictMock(FilterChain.class); ServletOutputStream outputStream = EasyMock.createNiceMock(ServletOutputStream.class); - AuthenticationResult authenticationResult = new AuthenticationResult("does-not-belong", "does-not-belong", null); + AuthenticationResult authenticationResult = new AuthenticationResult("does-not-belong", "does-not-belong", + null, + null); EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(true).once(); EasyMock.expect(req.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(authenticationResult).once(); diff --git a/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java index bfff546e11d0..c22ab27c076e 100644 --- a/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java +++ b/sql/src/test/java/io/druid/sql/calcite/util/CalciteTests.java @@ -170,7 +170,7 @@ public Access authorize( @Override public AuthenticationResult authenticateJDBCContext(Map context) { - return new AuthenticationResult((String) context.get("user"), AuthConfig.ALLOW_ALL_NAME, null); + return new AuthenticationResult((String) context.get("user"), AuthConfig.ALLOW_ALL_NAME, null, null); } } ); @@ -191,13 +191,13 @@ public AuthenticationResult createEscalatedAuthenticationResult() public static final AuthenticationResult REGULAR_USER_AUTH_RESULT = new AuthenticationResult( AuthConfig.ALLOW_ALL_NAME, AuthConfig.ALLOW_ALL_NAME, - null + null, null ); public static final AuthenticationResult SUPER_USER_AUTH_RESULT = new AuthenticationResult( TEST_SUPERUSER_NAME, AuthConfig.ALLOW_ALL_NAME, - null + null, null ); private static final String TIMESTAMP_COLUMN = "t"; From af06da4d72b07c08be18ae883fbf1f91d866c1cc Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 9 May 2018 12:46:43 -0700 Subject: [PATCH 72/82] SegmentLoadDropHandler: Fix deadlock when segments have errors loading on startup. (#5735) (#5755) The "lock" object was used to synchronize start/stop as well as synchronize removals from segmentsToDelete (when a segment is done dropping). This could cause a deadlock if a segment-load throws an exception during loadLocalCache. loadLocalCache is run by start() while it holds the lock, but then it spawns loading threads, and those threads will try to acquire the "segmentsToDelete" lock if they want to drop a corrupt segments. I don't see any reason for these two locks to be the same lock, so I split them. --- .../coordination/SegmentLoadDropHandler.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java b/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java index 04de12d50970..5774c4bc8e61 100644 --- a/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java +++ b/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java @@ -73,7 +73,11 @@ public class SegmentLoadDropHandler implements DataSegmentChangeHandler { private static final EmittingLogger log = new EmittingLogger(SegmentLoadDropHandler.class); - private final Object lock = new Object(); + // Synchronizes removals from segmentsToDelete + private final Object segmentDeleteLock = new Object(); + + // Synchronizes start/stop of this object. + private final Object startStopLock = new Object(); private final ObjectMapper jsonMapper; private final SegmentLoaderConfig config; @@ -137,7 +141,7 @@ public SegmentLoadDropHandler( @LifecycleStart public void start() throws IOException { - synchronized (lock) { + synchronized (startStopLock) { if (started) { return; } @@ -159,7 +163,7 @@ public void start() throws IOException @LifecycleStop public void stop() { - synchronized (lock) { + synchronized (startStopLock) { if (!started) { return; } @@ -296,7 +300,7 @@ each time when addSegment() is called, it has to wait for the lock in order to m things slow. Given that in most cases segmentsToDelete.contains(segment) returns false, it will save a lot of cost of acquiring lock by doing the "contains" check outside the synchronized block. */ - synchronized (lock) { + synchronized (segmentDeleteLock) { segmentsToDelete.remove(segment); } } @@ -423,7 +427,7 @@ private void removeSegment( public void run() { try { - synchronized (lock) { + synchronized (segmentDeleteLock) { if (segmentsToDelete.remove(segment)) { segmentManager.dropSegment(segment); From e0e55eec287ee645af721a012a012d934f4c3ff1 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 10 May 2018 12:12:28 -0700 Subject: [PATCH 73/82] [Backport] Fix metrics for inserting segments (#5759) * [Backport] Fix metrics for inserting segments * fix build --- .../SegmentTransactionalInsertAction.java | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentTransactionalInsertAction.java b/indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentTransactionalInsertAction.java index a65498c6a277..acd6eb5a647c 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentTransactionalInsertAction.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/actions/SegmentTransactionalInsertAction.java @@ -23,12 +23,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.collect.ImmutableSet; -import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.indexing.common.task.Task; import io.druid.indexing.overlord.CriticalAction; import io.druid.indexing.overlord.DataSourceMetadata; import io.druid.indexing.overlord.SegmentPublishResult; import io.druid.java.util.common.logger.Logger; +import io.druid.java.util.emitter.service.ServiceMetricEvent; import io.druid.query.DruidMetrics; import io.druid.timeline.DataSegment; @@ -127,22 +127,21 @@ public SegmentPublishResult perform(Task task, TaskActionToolbox toolbox) throws throw new RuntimeException(e); } + // Emit metrics + final ServiceMetricEvent.Builder metricBuilder = new ServiceMetricEvent.Builder() + .setDimension(DruidMetrics.DATASOURCE, task.getDataSource()) + .setDimension(DruidMetrics.TASK_TYPE, task.getType()); + if (retVal.isSuccess()) { - // Emit metrics - final ServiceMetricEvent.Builder metricBuilder = new ServiceMetricEvent.Builder() - .setDimension(DruidMetrics.DATASOURCE, task.getDataSource()) - .setDimension(DruidMetrics.TASK_TYPE, task.getType()); - - if (retVal.isSuccess()) { - toolbox.getEmitter().emit(metricBuilder.build("segment/txn/success", 1)); - } else { - toolbox.getEmitter().emit(metricBuilder.build("segment/txn/failure", 1)); - } - - for (DataSegment segment : retVal.getSegments()) { - metricBuilder.setDimension(DruidMetrics.INTERVAL, segment.getInterval().toString()); - toolbox.getEmitter().emit(metricBuilder.build("segment/added/bytes", segment.getSize())); - } + toolbox.getEmitter().emit(metricBuilder.build("segment/txn/success", 1)); + } else { + toolbox.getEmitter().emit(metricBuilder.build("segment/txn/failure", 1)); + } + + // getSegments() should return an empty set if announceHistoricalSegments() failed + for (DataSegment segment : retVal.getSegments()) { + metricBuilder.setDimension(DruidMetrics.INTERVAL, segment.getInterval().toString()); + toolbox.getEmitter().emit(metricBuilder.build("segment/added/bytes", segment.getSize())); } return retVal; From a8d97a458daa042ab40d79bc84ad8f002cab5bfd Mon Sep 17 00:00:00 2001 From: Jonathan Wei Date: Fri, 11 May 2018 14:48:46 -0700 Subject: [PATCH 74/82] Fix KerberosAuthenticator serverPrincipal host replacement (#5766) (#5770) --- .../kerberos/KerberosAuthenticator.java | 44 +++++++++---------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java index 6b5d044f26a3..df799c09df56 100644 --- a/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java +++ b/extensions-core/druid-kerberos/src/main/java/io/druid/security/kerberos/KerberosAuthenticator.java @@ -127,13 +127,19 @@ public KerberosAuthenticator( ) { this.node = node; - this.serverPrincipal = serverPrincipal; this.serverKeytab = serverKeytab; this.authToLocal = authToLocal == null ? "DEFAULT" : authToLocal; this.excludedPaths = excludedPaths == null ? DEFAULT_EXCLUDED_PATHS : excludedPaths; this.cookieSignatureSecret = cookieSignatureSecret; this.authorizerName = authorizerName; this.name = Preconditions.checkNotNull(name); + + try { + this.serverPrincipal = SecurityUtil.getServerPrincipal(serverPrincipal, node.getHost()); + } + catch (Exception e) { + throw new RuntimeException(e); + } } @Override @@ -410,20 +416,12 @@ public Class getFilterClass() public Map getInitParameters() { Map params = new HashMap(); - try { - params.put( - "kerberos.principal", - SecurityUtil.getServerPrincipal(serverPrincipal, node.getHost()) - ); - params.put("kerberos.keytab", serverKeytab); - params.put(AuthenticationFilter.AUTH_TYPE, DruidKerberosAuthenticationHandler.class.getName()); - params.put("kerberos.name.rules", authToLocal); - if (cookieSignatureSecret != null) { - params.put("signature.secret", cookieSignatureSecret); - } - } - catch (IOException e) { - Throwables.propagate(e); + params.put("kerberos.principal", serverPrincipal); + params.put("kerberos.keytab", serverKeytab); + params.put(AuthenticationFilter.AUTH_TYPE, DruidKerberosAuthenticationHandler.class.getName()); + params.put("kerberos.name.rules", authToLocal); + if (cookieSignatureSecret != null) { + params.put("signature.secret", cookieSignatureSecret); } return params; } @@ -571,8 +569,8 @@ private String getPrincipalFromRequestNew(HttpServletRequest req) for (Object cred : serverCreds) { if (cred instanceof KeyTab) { KeyTab serverKeyTab = (KeyTab) cred; - KerberosPrincipal serverPrincipal = new KerberosPrincipal(this.serverPrincipal); - KerberosKey[] serverKeys = serverKeyTab.getKeys(serverPrincipal); + KerberosPrincipal kerberosPrincipal = new KerberosPrincipal(serverPrincipal); + KerberosKey[] serverKeys = serverKeyTab.getKeys(kerberosPrincipal); for (KerberosKey key : serverKeys) { if (key.getKeyType() == eType) { finalKey = new EncryptionKey(key.getKeyType(), key.getEncoded()); @@ -611,12 +609,10 @@ private boolean isValueAPReq(DerValue value) private void initializeKerberosLogin() throws ServletException { - String principal; String keytab; try { - principal = SecurityUtil.getServerPrincipal(serverPrincipal, node.getHost()); - if (principal == null || principal.trim().length() == 0) { + if (serverPrincipal == null || serverPrincipal.trim().length() == 0) { throw new ServletException("Principal not defined in configuration"); } keytab = serverKeytab; @@ -628,16 +624,16 @@ private void initializeKerberosLogin() throws ServletException } Set principals = new HashSet(); - principals.add(new KerberosPrincipal(principal)); + principals.add(new KerberosPrincipal(serverPrincipal)); Subject subject = new Subject(false, principals, new HashSet(), new HashSet()); - DruidKerberosConfiguration kerberosConfiguration = new DruidKerberosConfiguration(keytab, principal); + DruidKerberosConfiguration kerberosConfiguration = new DruidKerberosConfiguration(keytab, serverPrincipal); - log.info("Login using keytab " + keytab + ", for principal " + principal); + log.info("Login using keytab " + keytab + ", for principal " + serverPrincipal); loginContext = new LoginContext("", subject, null, kerberosConfiguration); loginContext.login(); - log.info("Initialized, principal %s from keytab %s", principal, keytab); + log.info("Initialized, principal %s from keytab %s", serverPrincipal, keytab); } catch (Exception ex) { throw new ServletException(ex); From 0a41d3766bbc23dde3f55415b99b598ffbaead43 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 15 May 2018 13:46:09 -0700 Subject: [PATCH 75/82] [maven-release-plugin] prepare release druid-0.12.1-rc2 --- api/pom.xml | 2 +- aws-common/pom.xml | 2 +- benchmarks/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- extendedset/pom.xml | 2 +- extensions-contrib/ambari-metrics-emitter/pom.xml | 2 +- extensions-contrib/azure-extensions/pom.xml | 2 +- extensions-contrib/cassandra-storage/pom.xml | 2 +- extensions-contrib/cloudfiles-extensions/pom.xml | 2 +- extensions-contrib/distinctcount/pom.xml | 2 +- extensions-contrib/druid-rocketmq/pom.xml | 2 +- extensions-contrib/google-extensions/pom.xml | 2 +- extensions-contrib/graphite-emitter/pom.xml | 2 +- extensions-contrib/kafka-eight-simpleConsumer/pom.xml | 2 +- extensions-contrib/kafka-emitter/pom.xml | 2 +- extensions-contrib/orc-extensions/pom.xml | 2 +- extensions-contrib/parquet-extensions/pom.xml | 2 +- extensions-contrib/rabbitmq/pom.xml | 2 +- extensions-contrib/redis-cache/pom.xml | 2 +- extensions-contrib/sqlserver-metadata-storage/pom.xml | 2 +- extensions-contrib/statsd-emitter/pom.xml | 2 +- extensions-contrib/thrift-extensions/pom.xml | 2 +- extensions-contrib/time-min-max/pom.xml | 2 +- extensions-contrib/virtual-columns/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 2 +- extensions-core/datasketches/pom.xml | 2 +- extensions-core/druid-basic-security/pom.xml | 2 +- extensions-core/druid-kerberos/pom.xml | 2 +- extensions-core/hdfs-storage/pom.xml | 2 +- extensions-core/histogram/pom.xml | 2 +- extensions-core/kafka-eight/pom.xml | 2 +- extensions-core/kafka-extraction-namespace/pom.xml | 2 +- extensions-core/kafka-indexing-service/pom.xml | 2 +- extensions-core/lookups-cached-global/pom.xml | 2 +- extensions-core/lookups-cached-single/pom.xml | 2 +- extensions-core/mysql-metadata-storage/pom.xml | 2 +- extensions-core/postgresql-metadata-storage/pom.xml | 2 +- extensions-core/protobuf-extensions/pom.xml | 2 +- extensions-core/s3-extensions/pom.xml | 2 +- extensions-core/simple-client-sslcontext/pom.xml | 2 +- extensions-core/stats/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- integration-tests/pom.xml | 2 +- java-util/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- sql/pom.xml | 2 +- 53 files changed, 54 insertions(+), 54 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index d262eeb898d9..d9ecd50af110 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 diff --git a/aws-common/pom.xml b/aws-common/pom.xml index 665e277d74f2..53457ad75887 100644 --- a/aws-common/pom.xml +++ b/aws-common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 8e62ab45f0d5..61fbd627cbfb 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 diff --git a/common/pom.xml b/common/pom.xml index 978aac817f06..8a453a950b13 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 diff --git a/distribution/pom.xml b/distribution/pom.xml index 04a145251485..15d9f898607b 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -28,7 +28,7 @@ druid io.druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 diff --git a/examples/pom.xml b/examples/pom.xml index 9136b1276ccc..03f7754828c9 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 diff --git a/extendedset/pom.xml b/extendedset/pom.xml index c0c2329a9f03..c2a37bc5f4a5 100755 --- a/extendedset/pom.xml +++ b/extendedset/pom.xml @@ -31,7 +31,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index 802c35af8aa7..23af2e5c7058 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-contrib/azure-extensions/pom.xml b/extensions-contrib/azure-extensions/pom.xml index 10e7d0b542a9..e8996d7d3fa9 100644 --- a/extensions-contrib/azure-extensions/pom.xml +++ b/extensions-contrib/azure-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-contrib/cassandra-storage/pom.xml b/extensions-contrib/cassandra-storage/pom.xml index e2d32197de5d..5100c57afa81 100644 --- a/extensions-contrib/cassandra-storage/pom.xml +++ b/extensions-contrib/cassandra-storage/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index 33902392cfd8..726e0bc916c9 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-contrib/distinctcount/pom.xml b/extensions-contrib/distinctcount/pom.xml index 417c16fbb464..54aefa784368 100644 --- a/extensions-contrib/distinctcount/pom.xml +++ b/extensions-contrib/distinctcount/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-contrib/druid-rocketmq/pom.xml b/extensions-contrib/druid-rocketmq/pom.xml index 6647eae9f95f..02f9519cc833 100644 --- a/extensions-contrib/druid-rocketmq/pom.xml +++ b/extensions-contrib/druid-rocketmq/pom.xml @@ -22,7 +22,7 @@ druid io.druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-contrib/google-extensions/pom.xml b/extensions-contrib/google-extensions/pom.xml index b7c21eff0530..dc50b943f776 100644 --- a/extensions-contrib/google-extensions/pom.xml +++ b/extensions-contrib/google-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index 9db9405886ef..8f14ee1a80fb 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml index 087323589c86..b69d30670693 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml +++ b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index d631f6d25595..652407e7bac7 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-contrib/orc-extensions/pom.xml b/extensions-contrib/orc-extensions/pom.xml index d1d62c6e71b4..f27ecded1005 100644 --- a/extensions-contrib/orc-extensions/pom.xml +++ b/extensions-contrib/orc-extensions/pom.xml @@ -24,7 +24,7 @@ druid io.druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/parquet-extensions/pom.xml b/extensions-contrib/parquet-extensions/pom.xml index 117b523ce3ec..d0915f75ab7d 100644 --- a/extensions-contrib/parquet-extensions/pom.xml +++ b/extensions-contrib/parquet-extensions/pom.xml @@ -8,7 +8,7 @@ druid io.druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/rabbitmq/pom.xml b/extensions-contrib/rabbitmq/pom.xml index f16a43044358..abb09485bcc6 100644 --- a/extensions-contrib/rabbitmq/pom.xml +++ b/extensions-contrib/rabbitmq/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-contrib/redis-cache/pom.xml b/extensions-contrib/redis-cache/pom.xml index e500311159b0..f49de22e3560 100644 --- a/extensions-contrib/redis-cache/pom.xml +++ b/extensions-contrib/redis-cache/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-contrib/sqlserver-metadata-storage/pom.xml b/extensions-contrib/sqlserver-metadata-storage/pom.xml index d38b04b66991..26eb5d99ef25 100644 --- a/extensions-contrib/sqlserver-metadata-storage/pom.xml +++ b/extensions-contrib/sqlserver-metadata-storage/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index 673fa9bca873..c3a256d4d946 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index f23d4c132780..3eca95a12512 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -9,7 +9,7 @@ druid io.druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/time-min-max/pom.xml b/extensions-contrib/time-min-max/pom.xml index 6ad82d3eddf8..9c33c8bc8d00 100644 --- a/extensions-contrib/time-min-max/pom.xml +++ b/extensions-contrib/time-min-max/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/virtual-columns/pom.xml b/extensions-contrib/virtual-columns/pom.xml index 269702e5476f..9a3a4337cb11 100644 --- a/extensions-contrib/virtual-columns/pom.xml +++ b/extensions-contrib/virtual-columns/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index c5c3f582ef97..ce2aeb080631 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index bf5f0982343c..2cd4e1fb5601 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-core/druid-basic-security/pom.xml b/extensions-core/druid-basic-security/pom.xml index fe0c45983c78..0883655f9c10 100644 --- a/extensions-core/druid-basic-security/pom.xml +++ b/extensions-core/druid-basic-security/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index 226f8d3b89e4..a8960f4411a3 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index c874c1d56ed9..9510c5e6d389 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-core/histogram/pom.xml b/extensions-core/histogram/pom.xml index bfb960160a05..d31dfef639b5 100644 --- a/extensions-core/histogram/pom.xml +++ b/extensions-core/histogram/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-core/kafka-eight/pom.xml b/extensions-core/kafka-eight/pom.xml index 511bba3f7d15..4a3bd108c3ef 100644 --- a/extensions-core/kafka-eight/pom.xml +++ b/extensions-core/kafka-eight/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index 8d0b7094f636..0a9cd6e3fdb2 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index 11a806c78f6a..414f951d90d2 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-core/lookups-cached-global/pom.xml b/extensions-core/lookups-cached-global/pom.xml index 7bdc3cb77ff6..fe8908564e79 100644 --- a/extensions-core/lookups-cached-global/pom.xml +++ b/extensions-core/lookups-cached-global/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-core/lookups-cached-single/pom.xml b/extensions-core/lookups-cached-single/pom.xml index e6fc760e14df..a2c335d53fb8 100644 --- a/extensions-core/lookups-cached-single/pom.xml +++ b/extensions-core/lookups-cached-single/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index 3c33d71c7fa6..1187cf0d6069 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-core/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml index 36529ed36196..f62f0c582370 100644 --- a/extensions-core/postgresql-metadata-storage/pom.xml +++ b/extensions-core/postgresql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-core/protobuf-extensions/pom.xml b/extensions-core/protobuf-extensions/pom.xml index 053058eb53b8..0d9c8da4c194 100644 --- a/extensions-core/protobuf-extensions/pom.xml +++ b/extensions-core/protobuf-extensions/pom.xml @@ -27,7 +27,7 @@ druid io.druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index 744a00589443..8e16183f57a8 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/extensions-core/simple-client-sslcontext/pom.xml b/extensions-core/simple-client-sslcontext/pom.xml index 26a6f093fc3f..f03946338e67 100644 --- a/extensions-core/simple-client-sslcontext/pom.xml +++ b/extensions-core/simple-client-sslcontext/pom.xml @@ -3,7 +3,7 @@ druid io.druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml 4.0.0 diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml index 82f9b592f43f..098ba0889143 100644 --- a/extensions-core/stats/pom.xml +++ b/extensions-core/stats/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 ../../pom.xml diff --git a/hll/pom.xml b/hll/pom.xml index 77abf7cf0e9a..f2952b1a304b 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 druid-hll diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 79c722fc8b89..aafe8d19166a 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 994dd25b3d1e..a058fcf2f757 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 6496b9d244ea..7b90aa63ac67 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 diff --git a/java-util/pom.xml b/java-util/pom.xml index 0dad4e22d814..0337eaa16291 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 java-util diff --git a/pom.xml b/pom.xml index 70f2e7a96655..3f18de8934ec 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 pom ${project.groupId}:${project.artifactId} @@ -53,7 +53,7 @@ scm:git:ssh://git@github.com/druid-io/druid.git scm:git:ssh://git@github.com/druid-io/druid.git https://github.com/druid-io/druid.git - 0.12.0-SNAPSHOT + druid-0.12.1-rc2 diff --git a/processing/pom.xml b/processing/pom.xml index f8745662af84..bc0b9a3ab41c 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 diff --git a/server/pom.xml b/server/pom.xml index feeb71dd0cd9..1b72cd89a164 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 diff --git a/services/pom.xml b/services/pom.xml index 1b2185343fb0..bdde3b5b1514 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -25,7 +25,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 diff --git a/sql/pom.xml b/sql/pom.xml index 85be9ec6b139..12e226e83d64 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc2-SNAPSHOT + 0.12.1-rc2 From dffc819584dc1bc9b97110fdcbeb59ea34a8197c Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 15 May 2018 13:46:18 -0700 Subject: [PATCH 76/82] [maven-release-plugin] prepare for next development iteration --- api/pom.xml | 2 +- aws-common/pom.xml | 2 +- benchmarks/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- extendedset/pom.xml | 2 +- extensions-contrib/ambari-metrics-emitter/pom.xml | 2 +- extensions-contrib/azure-extensions/pom.xml | 2 +- extensions-contrib/cassandra-storage/pom.xml | 2 +- extensions-contrib/cloudfiles-extensions/pom.xml | 2 +- extensions-contrib/distinctcount/pom.xml | 2 +- extensions-contrib/druid-rocketmq/pom.xml | 2 +- extensions-contrib/google-extensions/pom.xml | 2 +- extensions-contrib/graphite-emitter/pom.xml | 2 +- extensions-contrib/kafka-eight-simpleConsumer/pom.xml | 2 +- extensions-contrib/kafka-emitter/pom.xml | 2 +- extensions-contrib/orc-extensions/pom.xml | 2 +- extensions-contrib/parquet-extensions/pom.xml | 2 +- extensions-contrib/rabbitmq/pom.xml | 2 +- extensions-contrib/redis-cache/pom.xml | 2 +- extensions-contrib/sqlserver-metadata-storage/pom.xml | 2 +- extensions-contrib/statsd-emitter/pom.xml | 2 +- extensions-contrib/thrift-extensions/pom.xml | 2 +- extensions-contrib/time-min-max/pom.xml | 2 +- extensions-contrib/virtual-columns/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 2 +- extensions-core/datasketches/pom.xml | 2 +- extensions-core/druid-basic-security/pom.xml | 2 +- extensions-core/druid-kerberos/pom.xml | 2 +- extensions-core/hdfs-storage/pom.xml | 2 +- extensions-core/histogram/pom.xml | 2 +- extensions-core/kafka-eight/pom.xml | 2 +- extensions-core/kafka-extraction-namespace/pom.xml | 2 +- extensions-core/kafka-indexing-service/pom.xml | 2 +- extensions-core/lookups-cached-global/pom.xml | 2 +- extensions-core/lookups-cached-single/pom.xml | 2 +- extensions-core/mysql-metadata-storage/pom.xml | 2 +- extensions-core/postgresql-metadata-storage/pom.xml | 2 +- extensions-core/protobuf-extensions/pom.xml | 2 +- extensions-core/s3-extensions/pom.xml | 2 +- extensions-core/simple-client-sslcontext/pom.xml | 2 +- extensions-core/stats/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- integration-tests/pom.xml | 2 +- java-util/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- sql/pom.xml | 2 +- 53 files changed, 54 insertions(+), 54 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index d9ecd50af110..f9625884d958 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT diff --git a/aws-common/pom.xml b/aws-common/pom.xml index 53457ad75887..ef2a915064fd 100644 --- a/aws-common/pom.xml +++ b/aws-common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 61fbd627cbfb..6840f1fcd84c 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 8a453a950b13..d03771c14f43 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT diff --git a/distribution/pom.xml b/distribution/pom.xml index 15d9f898607b..3fc696c5e436 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -28,7 +28,7 @@ druid io.druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 03f7754828c9..686ea498584d 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT diff --git a/extendedset/pom.xml b/extendedset/pom.xml index c2a37bc5f4a5..0a9959ae0489 100755 --- a/extendedset/pom.xml +++ b/extendedset/pom.xml @@ -31,7 +31,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index 23af2e5c7058..7304a0e2c2f5 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/azure-extensions/pom.xml b/extensions-contrib/azure-extensions/pom.xml index e8996d7d3fa9..b7d24ef24b95 100644 --- a/extensions-contrib/azure-extensions/pom.xml +++ b/extensions-contrib/azure-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/cassandra-storage/pom.xml b/extensions-contrib/cassandra-storage/pom.xml index 5100c57afa81..ef08e6a1cabd 100644 --- a/extensions-contrib/cassandra-storage/pom.xml +++ b/extensions-contrib/cassandra-storage/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index 726e0bc916c9..b239e384e791 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/distinctcount/pom.xml b/extensions-contrib/distinctcount/pom.xml index 54aefa784368..393243e0f37e 100644 --- a/extensions-contrib/distinctcount/pom.xml +++ b/extensions-contrib/distinctcount/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/druid-rocketmq/pom.xml b/extensions-contrib/druid-rocketmq/pom.xml index 02f9519cc833..7b9a26488e1a 100644 --- a/extensions-contrib/druid-rocketmq/pom.xml +++ b/extensions-contrib/druid-rocketmq/pom.xml @@ -22,7 +22,7 @@ druid io.druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/google-extensions/pom.xml b/extensions-contrib/google-extensions/pom.xml index dc50b943f776..85f8101c4aa3 100644 --- a/extensions-contrib/google-extensions/pom.xml +++ b/extensions-contrib/google-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index 8f14ee1a80fb..c52549512f92 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml index b69d30670693..c8ba4b3488e1 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml +++ b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index 652407e7bac7..a1dde47379cc 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/orc-extensions/pom.xml b/extensions-contrib/orc-extensions/pom.xml index f27ecded1005..941c95c25742 100644 --- a/extensions-contrib/orc-extensions/pom.xml +++ b/extensions-contrib/orc-extensions/pom.xml @@ -24,7 +24,7 @@ druid io.druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/parquet-extensions/pom.xml b/extensions-contrib/parquet-extensions/pom.xml index d0915f75ab7d..6f210c7edf58 100644 --- a/extensions-contrib/parquet-extensions/pom.xml +++ b/extensions-contrib/parquet-extensions/pom.xml @@ -8,7 +8,7 @@ druid io.druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/rabbitmq/pom.xml b/extensions-contrib/rabbitmq/pom.xml index abb09485bcc6..23748ef8b437 100644 --- a/extensions-contrib/rabbitmq/pom.xml +++ b/extensions-contrib/rabbitmq/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/redis-cache/pom.xml b/extensions-contrib/redis-cache/pom.xml index f49de22e3560..d713b6f49b6f 100644 --- a/extensions-contrib/redis-cache/pom.xml +++ b/extensions-contrib/redis-cache/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/sqlserver-metadata-storage/pom.xml b/extensions-contrib/sqlserver-metadata-storage/pom.xml index 26eb5d99ef25..5cb5e2aaa0f5 100644 --- a/extensions-contrib/sqlserver-metadata-storage/pom.xml +++ b/extensions-contrib/sqlserver-metadata-storage/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index c3a256d4d946..749d44850ba0 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index 3eca95a12512..f3e111ca067a 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -9,7 +9,7 @@ druid io.druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/time-min-max/pom.xml b/extensions-contrib/time-min-max/pom.xml index 9c33c8bc8d00..20e29b8a9719 100644 --- a/extensions-contrib/time-min-max/pom.xml +++ b/extensions-contrib/time-min-max/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/virtual-columns/pom.xml b/extensions-contrib/virtual-columns/pom.xml index 9a3a4337cb11..a7443172acb4 100644 --- a/extensions-contrib/virtual-columns/pom.xml +++ b/extensions-contrib/virtual-columns/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index ce2aeb080631..97efb4bbbf2b 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index 2cd4e1fb5601..75d7142c2c4c 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-basic-security/pom.xml b/extensions-core/druid-basic-security/pom.xml index 0883655f9c10..801d6cc26c73 100644 --- a/extensions-core/druid-basic-security/pom.xml +++ b/extensions-core/druid-basic-security/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index a8960f4411a3..f3f406423d57 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index 9510c5e6d389..99b64b606c5c 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/histogram/pom.xml b/extensions-core/histogram/pom.xml index d31dfef639b5..094722120d9b 100644 --- a/extensions-core/histogram/pom.xml +++ b/extensions-core/histogram/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-eight/pom.xml b/extensions-core/kafka-eight/pom.xml index 4a3bd108c3ef..42d0c59d5856 100644 --- a/extensions-core/kafka-eight/pom.xml +++ b/extensions-core/kafka-eight/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index 0a9cd6e3fdb2..af7db61dc172 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index 414f951d90d2..09fabd038e04 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/lookups-cached-global/pom.xml b/extensions-core/lookups-cached-global/pom.xml index fe8908564e79..4ec0e370512e 100644 --- a/extensions-core/lookups-cached-global/pom.xml +++ b/extensions-core/lookups-cached-global/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/lookups-cached-single/pom.xml b/extensions-core/lookups-cached-single/pom.xml index a2c335d53fb8..9169afd6179b 100644 --- a/extensions-core/lookups-cached-single/pom.xml +++ b/extensions-core/lookups-cached-single/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index 1187cf0d6069..d4d8931e4017 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml index f62f0c582370..fcbd4e5463ae 100644 --- a/extensions-core/postgresql-metadata-storage/pom.xml +++ b/extensions-core/postgresql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/protobuf-extensions/pom.xml b/extensions-core/protobuf-extensions/pom.xml index 0d9c8da4c194..5c105bb3fde7 100644 --- a/extensions-core/protobuf-extensions/pom.xml +++ b/extensions-core/protobuf-extensions/pom.xml @@ -27,7 +27,7 @@ druid io.druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index 8e16183f57a8..488d223cf7e5 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/extensions-core/simple-client-sslcontext/pom.xml b/extensions-core/simple-client-sslcontext/pom.xml index f03946338e67..aa61182c5ebf 100644 --- a/extensions-core/simple-client-sslcontext/pom.xml +++ b/extensions-core/simple-client-sslcontext/pom.xml @@ -3,7 +3,7 @@ druid io.druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml index 098ba0889143..f7b6d24a1f13 100644 --- a/extensions-core/stats/pom.xml +++ b/extensions-core/stats/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT ../../pom.xml diff --git a/hll/pom.xml b/hll/pom.xml index f2952b1a304b..8cd83f87d116 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT druid-hll diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index aafe8d19166a..7892babaad0c 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index a058fcf2f757..66e1108ce095 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 7b90aa63ac67..267112fdbe76 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT diff --git a/java-util/pom.xml b/java-util/pom.xml index 0337eaa16291..095415aa548b 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT java-util diff --git a/pom.xml b/pom.xml index 3f18de8934ec..f6ed4fa9a2ec 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT pom ${project.groupId}:${project.artifactId} @@ -53,7 +53,7 @@ scm:git:ssh://git@github.com/druid-io/druid.git scm:git:ssh://git@github.com/druid-io/druid.git https://github.com/druid-io/druid.git - druid-0.12.1-rc2 + 0.12.0-SNAPSHOT diff --git a/processing/pom.xml b/processing/pom.xml index bc0b9a3ab41c..f6ededd8820b 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index 1b72cd89a164..7f21a88c908e 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index bdde3b5b1514..b5a0e3196063 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -25,7 +25,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT diff --git a/sql/pom.xml b/sql/pom.xml index 12e226e83d64..3ff1891bcec0 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc2 + 0.12.1-rc3-SNAPSHOT From 1b615593d62382cf98b7cb934c12e7888f31517f Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 5 Jun 2018 13:25:02 -0700 Subject: [PATCH 77/82] [maven-release-plugin] prepare release druid-0.12.1 --- api/pom.xml | 2 +- aws-common/pom.xml | 2 +- benchmarks/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- extendedset/pom.xml | 2 +- extensions-contrib/ambari-metrics-emitter/pom.xml | 2 +- extensions-contrib/azure-extensions/pom.xml | 2 +- extensions-contrib/cassandra-storage/pom.xml | 2 +- extensions-contrib/cloudfiles-extensions/pom.xml | 2 +- extensions-contrib/distinctcount/pom.xml | 2 +- extensions-contrib/druid-rocketmq/pom.xml | 2 +- extensions-contrib/google-extensions/pom.xml | 2 +- extensions-contrib/graphite-emitter/pom.xml | 2 +- extensions-contrib/kafka-eight-simpleConsumer/pom.xml | 2 +- extensions-contrib/kafka-emitter/pom.xml | 2 +- extensions-contrib/orc-extensions/pom.xml | 2 +- extensions-contrib/parquet-extensions/pom.xml | 2 +- extensions-contrib/rabbitmq/pom.xml | 2 +- extensions-contrib/redis-cache/pom.xml | 2 +- extensions-contrib/sqlserver-metadata-storage/pom.xml | 2 +- extensions-contrib/statsd-emitter/pom.xml | 2 +- extensions-contrib/thrift-extensions/pom.xml | 2 +- extensions-contrib/time-min-max/pom.xml | 2 +- extensions-contrib/virtual-columns/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 2 +- extensions-core/datasketches/pom.xml | 2 +- extensions-core/druid-basic-security/pom.xml | 2 +- extensions-core/druid-kerberos/pom.xml | 2 +- extensions-core/hdfs-storage/pom.xml | 2 +- extensions-core/histogram/pom.xml | 2 +- extensions-core/kafka-eight/pom.xml | 2 +- extensions-core/kafka-extraction-namespace/pom.xml | 2 +- extensions-core/kafka-indexing-service/pom.xml | 2 +- extensions-core/lookups-cached-global/pom.xml | 2 +- extensions-core/lookups-cached-single/pom.xml | 2 +- extensions-core/mysql-metadata-storage/pom.xml | 2 +- extensions-core/postgresql-metadata-storage/pom.xml | 2 +- extensions-core/protobuf-extensions/pom.xml | 2 +- extensions-core/s3-extensions/pom.xml | 2 +- extensions-core/simple-client-sslcontext/pom.xml | 2 +- extensions-core/stats/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- integration-tests/pom.xml | 2 +- java-util/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- sql/pom.xml | 2 +- 53 files changed, 54 insertions(+), 54 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index f9625884d958..85fff8a5ca9d 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 diff --git a/aws-common/pom.xml b/aws-common/pom.xml index ef2a915064fd..c2aa627c836a 100644 --- a/aws-common/pom.xml +++ b/aws-common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 6840f1fcd84c..069e48142105 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 diff --git a/common/pom.xml b/common/pom.xml index d03771c14f43..1485f89f90f9 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 diff --git a/distribution/pom.xml b/distribution/pom.xml index 3fc696c5e436..eaf65bec41c4 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -28,7 +28,7 @@ druid io.druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 diff --git a/examples/pom.xml b/examples/pom.xml index 686ea498584d..4a25126707ae 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 diff --git a/extendedset/pom.xml b/extendedset/pom.xml index 0a9959ae0489..03bc33982afe 100755 --- a/extendedset/pom.xml +++ b/extendedset/pom.xml @@ -31,7 +31,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index 7304a0e2c2f5..1c4d9d81731c 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-contrib/azure-extensions/pom.xml b/extensions-contrib/azure-extensions/pom.xml index b7d24ef24b95..43c9bb3495d5 100644 --- a/extensions-contrib/azure-extensions/pom.xml +++ b/extensions-contrib/azure-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-contrib/cassandra-storage/pom.xml b/extensions-contrib/cassandra-storage/pom.xml index ef08e6a1cabd..90e930bef9ed 100644 --- a/extensions-contrib/cassandra-storage/pom.xml +++ b/extensions-contrib/cassandra-storage/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index b239e384e791..7c19b9471f67 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-contrib/distinctcount/pom.xml b/extensions-contrib/distinctcount/pom.xml index 393243e0f37e..9a59da024af9 100644 --- a/extensions-contrib/distinctcount/pom.xml +++ b/extensions-contrib/distinctcount/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-contrib/druid-rocketmq/pom.xml b/extensions-contrib/druid-rocketmq/pom.xml index 7b9a26488e1a..8ea9acba9d98 100644 --- a/extensions-contrib/druid-rocketmq/pom.xml +++ b/extensions-contrib/druid-rocketmq/pom.xml @@ -22,7 +22,7 @@ druid io.druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-contrib/google-extensions/pom.xml b/extensions-contrib/google-extensions/pom.xml index 85f8101c4aa3..19938e56c6e9 100644 --- a/extensions-contrib/google-extensions/pom.xml +++ b/extensions-contrib/google-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index c52549512f92..55ec2e38b212 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml index c8ba4b3488e1..96648494387d 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml +++ b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index a1dde47379cc..0fc4feda4a50 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-contrib/orc-extensions/pom.xml b/extensions-contrib/orc-extensions/pom.xml index 941c95c25742..9a0c2c7f38ff 100644 --- a/extensions-contrib/orc-extensions/pom.xml +++ b/extensions-contrib/orc-extensions/pom.xml @@ -24,7 +24,7 @@ druid io.druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/parquet-extensions/pom.xml b/extensions-contrib/parquet-extensions/pom.xml index 6f210c7edf58..629bbce22826 100644 --- a/extensions-contrib/parquet-extensions/pom.xml +++ b/extensions-contrib/parquet-extensions/pom.xml @@ -8,7 +8,7 @@ druid io.druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/rabbitmq/pom.xml b/extensions-contrib/rabbitmq/pom.xml index 23748ef8b437..684c7d46bf26 100644 --- a/extensions-contrib/rabbitmq/pom.xml +++ b/extensions-contrib/rabbitmq/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-contrib/redis-cache/pom.xml b/extensions-contrib/redis-cache/pom.xml index d713b6f49b6f..4f058c18602b 100644 --- a/extensions-contrib/redis-cache/pom.xml +++ b/extensions-contrib/redis-cache/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-contrib/sqlserver-metadata-storage/pom.xml b/extensions-contrib/sqlserver-metadata-storage/pom.xml index 5cb5e2aaa0f5..6230cbc90e52 100644 --- a/extensions-contrib/sqlserver-metadata-storage/pom.xml +++ b/extensions-contrib/sqlserver-metadata-storage/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index 749d44850ba0..7169ceec72db 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index f3e111ca067a..71b7ff6c1b92 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -9,7 +9,7 @@ druid io.druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/time-min-max/pom.xml b/extensions-contrib/time-min-max/pom.xml index 20e29b8a9719..7a6356bd4386 100644 --- a/extensions-contrib/time-min-max/pom.xml +++ b/extensions-contrib/time-min-max/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml 4.0.0 diff --git a/extensions-contrib/virtual-columns/pom.xml b/extensions-contrib/virtual-columns/pom.xml index a7443172acb4..4bde3fdda29b 100644 --- a/extensions-contrib/virtual-columns/pom.xml +++ b/extensions-contrib/virtual-columns/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index 97efb4bbbf2b..580a61a9acee 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index 75d7142c2c4c..c338e83305ad 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-core/druid-basic-security/pom.xml b/extensions-core/druid-basic-security/pom.xml index 801d6cc26c73..be77392e9cac 100644 --- a/extensions-core/druid-basic-security/pom.xml +++ b/extensions-core/druid-basic-security/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index f3f406423d57..a4f3b1ec4b81 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index 99b64b606c5c..30ad52881302 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-core/histogram/pom.xml b/extensions-core/histogram/pom.xml index 094722120d9b..19fc71a442b2 100644 --- a/extensions-core/histogram/pom.xml +++ b/extensions-core/histogram/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-core/kafka-eight/pom.xml b/extensions-core/kafka-eight/pom.xml index 42d0c59d5856..45b45bbc9087 100644 --- a/extensions-core/kafka-eight/pom.xml +++ b/extensions-core/kafka-eight/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index af7db61dc172..ca51ccca5fd7 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index 09fabd038e04..6bdb5b19bb36 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-core/lookups-cached-global/pom.xml b/extensions-core/lookups-cached-global/pom.xml index 4ec0e370512e..3d8250ac8a0c 100644 --- a/extensions-core/lookups-cached-global/pom.xml +++ b/extensions-core/lookups-cached-global/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-core/lookups-cached-single/pom.xml b/extensions-core/lookups-cached-single/pom.xml index 9169afd6179b..a9e6639ad8d3 100644 --- a/extensions-core/lookups-cached-single/pom.xml +++ b/extensions-core/lookups-cached-single/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index d4d8931e4017..f522cd91ccd0 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-core/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml index fcbd4e5463ae..a6017c17fe59 100644 --- a/extensions-core/postgresql-metadata-storage/pom.xml +++ b/extensions-core/postgresql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-core/protobuf-extensions/pom.xml b/extensions-core/protobuf-extensions/pom.xml index 5c105bb3fde7..3eb5afa76171 100644 --- a/extensions-core/protobuf-extensions/pom.xml +++ b/extensions-core/protobuf-extensions/pom.xml @@ -27,7 +27,7 @@ druid io.druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index 488d223cf7e5..8970046a9245 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/extensions-core/simple-client-sslcontext/pom.xml b/extensions-core/simple-client-sslcontext/pom.xml index aa61182c5ebf..fb7b5538174e 100644 --- a/extensions-core/simple-client-sslcontext/pom.xml +++ b/extensions-core/simple-client-sslcontext/pom.xml @@ -3,7 +3,7 @@ druid io.druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml 4.0.0 diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml index f7b6d24a1f13..07419c79671a 100644 --- a/extensions-core/stats/pom.xml +++ b/extensions-core/stats/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 ../../pom.xml diff --git a/hll/pom.xml b/hll/pom.xml index 8cd83f87d116..5ca248955ab1 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 druid-hll diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 7892babaad0c..94ff59d72bf2 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 66e1108ce095..6c2b1d0ec2be 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 267112fdbe76..2c000b187c64 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 diff --git a/java-util/pom.xml b/java-util/pom.xml index 095415aa548b..3370f4934fee 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 java-util diff --git a/pom.xml b/pom.xml index f6ed4fa9a2ec..73d8fdbc2dde 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 pom ${project.groupId}:${project.artifactId} @@ -53,7 +53,7 @@ scm:git:ssh://git@github.com/druid-io/druid.git scm:git:ssh://git@github.com/druid-io/druid.git https://github.com/druid-io/druid.git - 0.12.0-SNAPSHOT + druid-0.12.1 diff --git a/processing/pom.xml b/processing/pom.xml index f6ededd8820b..2a5f8793dca6 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 diff --git a/server/pom.xml b/server/pom.xml index 7f21a88c908e..d528a0467855 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 diff --git a/services/pom.xml b/services/pom.xml index b5a0e3196063..df85cb23fdb1 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -25,7 +25,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 diff --git a/sql/pom.xml b/sql/pom.xml index 3ff1891bcec0..c141ee719d44 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1-rc3-SNAPSHOT + 0.12.1 From b07b3af6f6b9a9271d6cc73f69be8499de3c9b63 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 5 Jun 2018 13:25:14 -0700 Subject: [PATCH 78/82] [maven-release-plugin] prepare for next development iteration --- api/pom.xml | 2 +- aws-common/pom.xml | 2 +- benchmarks/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- examples/pom.xml | 2 +- extendedset/pom.xml | 2 +- extensions-contrib/ambari-metrics-emitter/pom.xml | 2 +- extensions-contrib/azure-extensions/pom.xml | 2 +- extensions-contrib/cassandra-storage/pom.xml | 2 +- extensions-contrib/cloudfiles-extensions/pom.xml | 2 +- extensions-contrib/distinctcount/pom.xml | 2 +- extensions-contrib/druid-rocketmq/pom.xml | 2 +- extensions-contrib/google-extensions/pom.xml | 2 +- extensions-contrib/graphite-emitter/pom.xml | 2 +- extensions-contrib/kafka-eight-simpleConsumer/pom.xml | 2 +- extensions-contrib/kafka-emitter/pom.xml | 2 +- extensions-contrib/orc-extensions/pom.xml | 2 +- extensions-contrib/parquet-extensions/pom.xml | 2 +- extensions-contrib/rabbitmq/pom.xml | 2 +- extensions-contrib/redis-cache/pom.xml | 2 +- extensions-contrib/sqlserver-metadata-storage/pom.xml | 2 +- extensions-contrib/statsd-emitter/pom.xml | 2 +- extensions-contrib/thrift-extensions/pom.xml | 2 +- extensions-contrib/time-min-max/pom.xml | 2 +- extensions-contrib/virtual-columns/pom.xml | 2 +- extensions-core/avro-extensions/pom.xml | 2 +- extensions-core/datasketches/pom.xml | 2 +- extensions-core/druid-basic-security/pom.xml | 2 +- extensions-core/druid-kerberos/pom.xml | 2 +- extensions-core/hdfs-storage/pom.xml | 2 +- extensions-core/histogram/pom.xml | 2 +- extensions-core/kafka-eight/pom.xml | 2 +- extensions-core/kafka-extraction-namespace/pom.xml | 2 +- extensions-core/kafka-indexing-service/pom.xml | 2 +- extensions-core/lookups-cached-global/pom.xml | 2 +- extensions-core/lookups-cached-single/pom.xml | 2 +- extensions-core/mysql-metadata-storage/pom.xml | 2 +- extensions-core/postgresql-metadata-storage/pom.xml | 2 +- extensions-core/protobuf-extensions/pom.xml | 2 +- extensions-core/s3-extensions/pom.xml | 2 +- extensions-core/simple-client-sslcontext/pom.xml | 2 +- extensions-core/stats/pom.xml | 2 +- hll/pom.xml | 2 +- indexing-hadoop/pom.xml | 2 +- indexing-service/pom.xml | 2 +- integration-tests/pom.xml | 2 +- java-util/pom.xml | 2 +- pom.xml | 4 ++-- processing/pom.xml | 2 +- server/pom.xml | 2 +- services/pom.xml | 2 +- sql/pom.xml | 2 +- 53 files changed, 54 insertions(+), 54 deletions(-) diff --git a/api/pom.xml b/api/pom.xml index 85fff8a5ca9d..5bc9bbcbb50f 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT diff --git a/aws-common/pom.xml b/aws-common/pom.xml index c2aa627c836a..0335148957d4 100644 --- a/aws-common/pom.xml +++ b/aws-common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 069e48142105..c13de92f5d5f 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 1485f89f90f9..050f714e163a 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT diff --git a/distribution/pom.xml b/distribution/pom.xml index eaf65bec41c4..217e46ad2809 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -28,7 +28,7 @@ druid io.druid - 0.12.1 + 0.12.2-SNAPSHOT diff --git a/examples/pom.xml b/examples/pom.xml index 4a25126707ae..0f1f5edefc4c 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT diff --git a/extendedset/pom.xml b/extendedset/pom.xml index 03bc33982afe..f9afc99259bd 100755 --- a/extendedset/pom.xml +++ b/extendedset/pom.xml @@ -31,7 +31,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT diff --git a/extensions-contrib/ambari-metrics-emitter/pom.xml b/extensions-contrib/ambari-metrics-emitter/pom.xml index 1c4d9d81731c..bed987c60915 100644 --- a/extensions-contrib/ambari-metrics-emitter/pom.xml +++ b/extensions-contrib/ambari-metrics-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/azure-extensions/pom.xml b/extensions-contrib/azure-extensions/pom.xml index 43c9bb3495d5..6f9edc549a65 100644 --- a/extensions-contrib/azure-extensions/pom.xml +++ b/extensions-contrib/azure-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/cassandra-storage/pom.xml b/extensions-contrib/cassandra-storage/pom.xml index 90e930bef9ed..1068cf8530dd 100644 --- a/extensions-contrib/cassandra-storage/pom.xml +++ b/extensions-contrib/cassandra-storage/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/cloudfiles-extensions/pom.xml b/extensions-contrib/cloudfiles-extensions/pom.xml index 7c19b9471f67..7ed7874b534b 100644 --- a/extensions-contrib/cloudfiles-extensions/pom.xml +++ b/extensions-contrib/cloudfiles-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/distinctcount/pom.xml b/extensions-contrib/distinctcount/pom.xml index 9a59da024af9..7dbb7fcc17c6 100644 --- a/extensions-contrib/distinctcount/pom.xml +++ b/extensions-contrib/distinctcount/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/druid-rocketmq/pom.xml b/extensions-contrib/druid-rocketmq/pom.xml index 8ea9acba9d98..3cba1a17663a 100644 --- a/extensions-contrib/druid-rocketmq/pom.xml +++ b/extensions-contrib/druid-rocketmq/pom.xml @@ -22,7 +22,7 @@ druid io.druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/google-extensions/pom.xml b/extensions-contrib/google-extensions/pom.xml index 19938e56c6e9..5b51c53a997f 100644 --- a/extensions-contrib/google-extensions/pom.xml +++ b/extensions-contrib/google-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/graphite-emitter/pom.xml b/extensions-contrib/graphite-emitter/pom.xml index 55ec2e38b212..f585362a4b8b 100644 --- a/extensions-contrib/graphite-emitter/pom.xml +++ b/extensions-contrib/graphite-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml index 96648494387d..9572a71dbcdb 100644 --- a/extensions-contrib/kafka-eight-simpleConsumer/pom.xml +++ b/extensions-contrib/kafka-eight-simpleConsumer/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index 0fc4feda4a50..3c087b1c0e03 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/orc-extensions/pom.xml b/extensions-contrib/orc-extensions/pom.xml index 9a0c2c7f38ff..e2e6beca8440 100644 --- a/extensions-contrib/orc-extensions/pom.xml +++ b/extensions-contrib/orc-extensions/pom.xml @@ -24,7 +24,7 @@ druid io.druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/parquet-extensions/pom.xml b/extensions-contrib/parquet-extensions/pom.xml index 629bbce22826..9242bb05879c 100644 --- a/extensions-contrib/parquet-extensions/pom.xml +++ b/extensions-contrib/parquet-extensions/pom.xml @@ -8,7 +8,7 @@ druid io.druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/rabbitmq/pom.xml b/extensions-contrib/rabbitmq/pom.xml index 684c7d46bf26..bbda4243759b 100644 --- a/extensions-contrib/rabbitmq/pom.xml +++ b/extensions-contrib/rabbitmq/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/redis-cache/pom.xml b/extensions-contrib/redis-cache/pom.xml index 4f058c18602b..bfa439100c78 100644 --- a/extensions-contrib/redis-cache/pom.xml +++ b/extensions-contrib/redis-cache/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/sqlserver-metadata-storage/pom.xml b/extensions-contrib/sqlserver-metadata-storage/pom.xml index 6230cbc90e52..5451d8ebdeb6 100644 --- a/extensions-contrib/sqlserver-metadata-storage/pom.xml +++ b/extensions-contrib/sqlserver-metadata-storage/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/statsd-emitter/pom.xml b/extensions-contrib/statsd-emitter/pom.xml index 7169ceec72db..e8226dc3e5c7 100644 --- a/extensions-contrib/statsd-emitter/pom.xml +++ b/extensions-contrib/statsd-emitter/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml index 71b7ff6c1b92..8c118c6cdbaa 100644 --- a/extensions-contrib/thrift-extensions/pom.xml +++ b/extensions-contrib/thrift-extensions/pom.xml @@ -9,7 +9,7 @@ druid io.druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/time-min-max/pom.xml b/extensions-contrib/time-min-max/pom.xml index 7a6356bd4386..028d42fbb573 100644 --- a/extensions-contrib/time-min-max/pom.xml +++ b/extensions-contrib/time-min-max/pom.xml @@ -21,7 +21,7 @@ druid io.druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-contrib/virtual-columns/pom.xml b/extensions-contrib/virtual-columns/pom.xml index 4bde3fdda29b..48af929aa085 100644 --- a/extensions-contrib/virtual-columns/pom.xml +++ b/extensions-contrib/virtual-columns/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml index 580a61a9acee..00695d4446f5 100644 --- a/extensions-core/avro-extensions/pom.xml +++ b/extensions-core/avro-extensions/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/datasketches/pom.xml b/extensions-core/datasketches/pom.xml index c338e83305ad..d9e528dd7a56 100644 --- a/extensions-core/datasketches/pom.xml +++ b/extensions-core/datasketches/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-basic-security/pom.xml b/extensions-core/druid-basic-security/pom.xml index be77392e9cac..353975f2b698 100644 --- a/extensions-core/druid-basic-security/pom.xml +++ b/extensions-core/druid-basic-security/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/druid-kerberos/pom.xml b/extensions-core/druid-kerberos/pom.xml index a4f3b1ec4b81..e533c5ed3071 100644 --- a/extensions-core/druid-kerberos/pom.xml +++ b/extensions-core/druid-kerberos/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/hdfs-storage/pom.xml b/extensions-core/hdfs-storage/pom.xml index 30ad52881302..eb1bb4cad97e 100644 --- a/extensions-core/hdfs-storage/pom.xml +++ b/extensions-core/hdfs-storage/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/histogram/pom.xml b/extensions-core/histogram/pom.xml index 19fc71a442b2..dc6728bac2c2 100644 --- a/extensions-core/histogram/pom.xml +++ b/extensions-core/histogram/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-eight/pom.xml b/extensions-core/kafka-eight/pom.xml index 45b45bbc9087..ab235b7e290d 100644 --- a/extensions-core/kafka-eight/pom.xml +++ b/extensions-core/kafka-eight/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-extraction-namespace/pom.xml b/extensions-core/kafka-extraction-namespace/pom.xml index ca51ccca5fd7..ff6d6e749b5d 100644 --- a/extensions-core/kafka-extraction-namespace/pom.xml +++ b/extensions-core/kafka-extraction-namespace/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/kafka-indexing-service/pom.xml b/extensions-core/kafka-indexing-service/pom.xml index 6bdb5b19bb36..f5d17bbe2199 100644 --- a/extensions-core/kafka-indexing-service/pom.xml +++ b/extensions-core/kafka-indexing-service/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/lookups-cached-global/pom.xml b/extensions-core/lookups-cached-global/pom.xml index 3d8250ac8a0c..629705204ddc 100644 --- a/extensions-core/lookups-cached-global/pom.xml +++ b/extensions-core/lookups-cached-global/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/lookups-cached-single/pom.xml b/extensions-core/lookups-cached-single/pom.xml index a9e6639ad8d3..6d22a81bf977 100644 --- a/extensions-core/lookups-cached-single/pom.xml +++ b/extensions-core/lookups-cached-single/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/mysql-metadata-storage/pom.xml b/extensions-core/mysql-metadata-storage/pom.xml index f522cd91ccd0..1a5d4b3943ae 100644 --- a/extensions-core/mysql-metadata-storage/pom.xml +++ b/extensions-core/mysql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/postgresql-metadata-storage/pom.xml b/extensions-core/postgresql-metadata-storage/pom.xml index a6017c17fe59..c34c9d0db5f4 100644 --- a/extensions-core/postgresql-metadata-storage/pom.xml +++ b/extensions-core/postgresql-metadata-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/protobuf-extensions/pom.xml b/extensions-core/protobuf-extensions/pom.xml index 3eb5afa76171..72994ffc5dc3 100644 --- a/extensions-core/protobuf-extensions/pom.xml +++ b/extensions-core/protobuf-extensions/pom.xml @@ -27,7 +27,7 @@ druid io.druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index 8970046a9245..23a59366ff74 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/extensions-core/simple-client-sslcontext/pom.xml b/extensions-core/simple-client-sslcontext/pom.xml index fb7b5538174e..2e0647dc420e 100644 --- a/extensions-core/simple-client-sslcontext/pom.xml +++ b/extensions-core/simple-client-sslcontext/pom.xml @@ -3,7 +3,7 @@ druid io.druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/extensions-core/stats/pom.xml b/extensions-core/stats/pom.xml index 07419c79671a..7a968ad8bda7 100644 --- a/extensions-core/stats/pom.xml +++ b/extensions-core/stats/pom.xml @@ -29,7 +29,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT ../../pom.xml diff --git a/hll/pom.xml b/hll/pom.xml index 5ca248955ab1..fc9dc8468253 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT druid-hll diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 94ff59d72bf2..1bdd540102e4 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 6c2b1d0ec2be..5900bae2e5f4 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 2c000b187c64..80729cc3e5bc 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT diff --git a/java-util/pom.xml b/java-util/pom.xml index 3370f4934fee..bf348d92878a 100644 --- a/java-util/pom.xml +++ b/java-util/pom.xml @@ -24,7 +24,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT java-util diff --git a/pom.xml b/pom.xml index 73d8fdbc2dde..69cfacaa1991 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ druid - 0.12.1 + 0.12.2-SNAPSHOT pom ${project.groupId}:${project.artifactId} @@ -53,7 +53,7 @@ scm:git:ssh://git@github.com/druid-io/druid.git scm:git:ssh://git@github.com/druid-io/druid.git https://github.com/druid-io/druid.git - druid-0.12.1 + 0.12.0-SNAPSHOT diff --git a/processing/pom.xml b/processing/pom.xml index 2a5f8793dca6..b70814e3d07d 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index d528a0467855..c3fd86e0d153 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -26,7 +26,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT diff --git a/services/pom.xml b/services/pom.xml index df85cb23fdb1..4e2dee82e49c 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -25,7 +25,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT diff --git a/sql/pom.xml b/sql/pom.xml index c141ee719d44..d393d68e5d63 100644 --- a/sql/pom.xml +++ b/sql/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.12.1 + 0.12.2-SNAPSHOT From f974bb017da325bc5e14872ec20b6e2f16e77740 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 5 Jul 2018 16:32:18 -0700 Subject: [PATCH 79/82] Fix defaultQueryTimeout (#5807) (#5944) * Fix defaultQueryTimeout - set default timeout in query context before query fail time is evaluated Remove unused import * Address failing checks * Addressing code review comments * Removed line that was no longer used --- .../java/io/druid/client/DirectDruidClient.java | 11 +++-------- .../main/java/io/druid/server/QueryLifecycle.java | 7 ++----- .../server/SetAndVerifyContextQueryRunner.java | 9 +++++++-- .../java/io/druid/client/DirectDruidClientTest.java | 13 ++++++++----- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/io/druid/client/DirectDruidClient.java b/server/src/main/java/io/druid/client/DirectDruidClient.java index d34efee089de..f18345cc8a64 100644 --- a/server/src/main/java/io/druid/client/DirectDruidClient.java +++ b/server/src/main/java/io/druid/client/DirectDruidClient.java @@ -116,21 +116,16 @@ public class DirectDruidClient implements QueryRunner private final boolean isSmile; /** - * Removes the magical fields added by {@link #makeResponseContextForQuery(Query, long)}. + * Removes the magical fields added by {@link #makeResponseContextForQuery()}. */ public static void removeMagicResponseContextFields(Map responseContext) { - responseContext.remove(DirectDruidClient.QUERY_FAIL_TIME); responseContext.remove(DirectDruidClient.QUERY_TOTAL_BYTES_GATHERED); } - public static Map makeResponseContextForQuery(Query query, long startTimeMillis) + public static Map makeResponseContextForQuery() { final Map responseContext = new ConcurrentHashMap<>(); - responseContext.put( - DirectDruidClient.QUERY_FAIL_TIME, - startTimeMillis + QueryContexts.getTimeout(query) - ); responseContext.put( DirectDruidClient.QUERY_TOTAL_BYTES_GATHERED, new AtomicLong() @@ -199,7 +194,7 @@ public Sequence run(final QueryPlus queryPlus, final Map c final long requestStartTimeNs = System.nanoTime(); - long timeoutAt = ((Long) context.get(QUERY_FAIL_TIME)).longValue(); + long timeoutAt = query.getContextValue(QUERY_FAIL_TIME); long maxScatterGatherBytes = QueryContexts.getMaxScatterGatherBytes(query); AtomicLong totalBytesGathered = (AtomicLong) context.get(QUERY_TOTAL_BYTES_GATHERED); diff --git a/server/src/main/java/io/druid/server/QueryLifecycle.java b/server/src/main/java/io/druid/server/QueryLifecycle.java index 70486b9b7bb9..3c03314cd595 100644 --- a/server/src/main/java/io/druid/server/QueryLifecycle.java +++ b/server/src/main/java/io/druid/server/QueryLifecycle.java @@ -21,7 +21,6 @@ import com.google.common.base.Strings; import com.google.common.collect.Iterables; -import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.client.DirectDruidClient; import io.druid.java.util.common.DateTimes; import io.druid.java.util.common.ISE; @@ -29,6 +28,7 @@ import io.druid.java.util.common.guava.SequenceWrapper; import io.druid.java.util.common.guava.Sequences; import io.druid.java.util.common.logger.Logger; +import io.druid.java.util.emitter.service.ServiceEmitter; import io.druid.query.DruidMetrics; import io.druid.query.GenericQueryMetricsFactory; import io.druid.query.Query; @@ -249,10 +249,7 @@ public QueryResponse execute() { transition(State.AUTHORIZED, State.EXECUTING); - final Map responseContext = DirectDruidClient.makeResponseContextForQuery( - queryPlus.getQuery(), - System.currentTimeMillis() - ); + final Map responseContext = DirectDruidClient.makeResponseContextForQuery(); final Sequence res = queryPlus.run(texasRanger, responseContext); diff --git a/server/src/main/java/io/druid/server/SetAndVerifyContextQueryRunner.java b/server/src/main/java/io/druid/server/SetAndVerifyContextQueryRunner.java index 637b9dd14fb0..7901e3674b09 100644 --- a/server/src/main/java/io/druid/server/SetAndVerifyContextQueryRunner.java +++ b/server/src/main/java/io/druid/server/SetAndVerifyContextQueryRunner.java @@ -19,6 +19,8 @@ package io.druid.server; +import com.google.common.collect.ImmutableMap; +import io.druid.client.DirectDruidClient; import io.druid.java.util.common.guava.Sequence; import io.druid.query.Query; import io.druid.query.QueryContexts; @@ -35,11 +37,13 @@ public class SetAndVerifyContextQueryRunner implements QueryRunner { private final ServerConfig serverConfig; private final QueryRunner baseRunner; + private final long startTimeMillis; public SetAndVerifyContextQueryRunner(ServerConfig serverConfig, QueryRunner baseRunner) { this.serverConfig = serverConfig; this.baseRunner = baseRunner; + this.startTimeMillis = System.currentTimeMillis(); } @Override @@ -54,12 +58,12 @@ public Sequence run(QueryPlus queryPlus, Map responseContext) ); } - public static > QueryType withTimeoutAndMaxScatterGatherBytes( + public > QueryType withTimeoutAndMaxScatterGatherBytes( final QueryType query, ServerConfig serverConfig ) { - return (QueryType) QueryContexts.verifyMaxQueryTimeout( + Query newQuery = QueryContexts.verifyMaxQueryTimeout( QueryContexts.withMaxScatterGatherBytes( QueryContexts.withDefaultTimeout( (Query) query, @@ -69,5 +73,6 @@ public static > QueryType withTimeoutAndMaxScatter ), serverConfig.getMaxQueryTimeout() ); + return (QueryType) newQuery.withOverriddenContext(ImmutableMap.of(DirectDruidClient.QUERY_FAIL_TIME, this.startTimeMillis + QueryContexts.getTimeout(newQuery))); } } diff --git a/server/src/test/java/io/druid/client/DirectDruidClientTest.java b/server/src/test/java/io/druid/client/DirectDruidClientTest.java index 3347d7d29bc5..de9d1eda80ac 100644 --- a/server/src/test/java/io/druid/client/DirectDruidClientTest.java +++ b/server/src/test/java/io/druid/client/DirectDruidClientTest.java @@ -20,15 +20,12 @@ package io.druid.client; import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; -import io.druid.java.util.http.client.HttpClient; -import io.druid.java.util.http.client.Request; -import io.druid.java.util.http.client.response.HttpResponseHandler; -import io.druid.java.util.http.client.response.StatusResponseHolder; import io.druid.client.selector.ConnectionCountServerSelectorStrategy; import io.druid.client.selector.HighestPriorityTierSelectorStrategy; import io.druid.client.selector.QueryableDruidServer; @@ -39,6 +36,10 @@ import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; +import io.druid.java.util.http.client.HttpClient; +import io.druid.java.util.http.client.Request; +import io.druid.java.util.http.client.response.HttpResponseHandler; +import io.druid.java.util.http.client.response.StatusResponseHolder; import io.druid.query.Druids; import io.druid.query.QueryInterruptedException; import io.druid.query.QueryPlus; @@ -165,7 +166,7 @@ public void testRun() throws Exception serverSelector.addServerAndUpdateSegment(queryableDruidServer2, serverSelector.getSegment()); TimeBoundaryQuery query = Druids.newTimeBoundaryQueryBuilder().dataSource("test").build(); - + query = query.withOverriddenContext(ImmutableMap.of(DirectDruidClient.QUERY_FAIL_TIME, Long.MAX_VALUE)); Sequence s1 = client1.run(QueryPlus.wrap(query), defaultContext); Assert.assertTrue(capturedRequest.hasCaptured()); Assert.assertEquals(url, capturedRequest.getValue().getUrl()); @@ -269,6 +270,7 @@ public void testCancel() throws Exception serverSelector.addServerAndUpdateSegment(queryableDruidServer1, serverSelector.getSegment()); TimeBoundaryQuery query = Druids.newTimeBoundaryQueryBuilder().dataSource("test").build(); + query = query.withOverriddenContext(ImmutableMap.of(DirectDruidClient.QUERY_FAIL_TIME, Long.MAX_VALUE)); cancellationFuture.set(new StatusResponseHolder(HttpResponseStatus.OK, new StringBuilder("cancelled"))); Sequence results = client1.run(QueryPlus.wrap(query), defaultContext); Assert.assertEquals(HttpMethod.DELETE, capturedRequest.getValue().getMethod()); @@ -340,6 +342,7 @@ public void testQueryInterruptionExceptionLogMessage() throws JsonProcessingExce serverSelector.addServerAndUpdateSegment(queryableDruidServer, dataSegment); TimeBoundaryQuery query = Druids.newTimeBoundaryQueryBuilder().dataSource("test").build(); + query = query.withOverriddenContext(ImmutableMap.of(DirectDruidClient.QUERY_FAIL_TIME, Long.MAX_VALUE)); interruptionFuture.set( new ByteArrayInputStream( StringUtils.toUtf8("{\"error\":\"testing1\",\"errorMessage\":\"testing2\"}") From 15f99ec7afe67d492f462079dee6eb466643e651 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 5 Jul 2018 16:32:30 -0700 Subject: [PATCH 80/82] Fix for when Hadoop dataSource inputSpec is specified multiple times. (#5790) (#5942) This feature was introduced in #5717 but it didn't work in production because this magical rewriter code wasn't also modified. Now, it is. --- .../io/druid/indexer/HadoopIngestionSpec.java | 10 ++-- ...cUpdateDatasourcePathSpecSegmentsTest.java | 60 +++++++++++++++++-- 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopIngestionSpec.java b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopIngestionSpec.java index aea58796b654..b227d6fe4df0 100644 --- a/indexing-hadoop/src/main/java/io/druid/indexer/HadoopIngestionSpec.java +++ b/indexing-hadoop/src/main/java/io/druid/indexer/HadoopIngestionSpec.java @@ -37,6 +37,7 @@ import org.joda.time.Interval; import java.io.IOException; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -151,20 +152,19 @@ public static HadoopIngestionSpec updateSegmentListIfDatasourcePathSpecIsUsed( String ingestionSpec = "ingestionSpec"; Map pathSpec = spec.getIOConfig().getPathSpec(); - Map datasourcePathSpec = null; + List> datasourcePathSpecs = new ArrayList<>(); if (pathSpec.get(type).equals(dataSource)) { - datasourcePathSpec = pathSpec; + datasourcePathSpecs.add(pathSpec); } else if (pathSpec.get(type).equals(multi)) { List> childPathSpecs = (List>) pathSpec.get(children); for (Map childPathSpec : childPathSpecs) { if (childPathSpec.get(type).equals(dataSource)) { - datasourcePathSpec = childPathSpec; - break; + datasourcePathSpecs.add(childPathSpec); } } } - if (datasourcePathSpec != null) { + for (Map datasourcePathSpec : datasourcePathSpecs) { Map ingestionSpecMap = (Map) datasourcePathSpec.get(ingestionSpec); DatasourceIngestionSpec ingestionSpecObj = jsonMapper.convertValue( ingestionSpecMap, diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest.java index 50f51752cbec..0351a114e9c6 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest.java @@ -51,9 +51,12 @@ */ public class HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest { - private final String testDatasource = "test"; - private final Interval testDatasourceInterval = Intervals.of("1970/3000"); - private final Interval testDatasourceIntervalPartial = Intervals.of("2050/3000"); + private static final String testDatasource = "test"; + private static final String testDatasource2 = "test2"; + private static final Interval testDatasourceInterval = Intervals.of("1970/3000"); + private static final Interval testDatasourceInterval2 = Intervals.of("2000/2001"); + private static final Interval testDatasourceIntervalPartial = Intervals.of("2050/3000"); + private final ObjectMapper jsonMapper; public HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest() @@ -67,7 +70,7 @@ public HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest() } private static final DataSegment SEGMENT = new DataSegment( - "test1", + testDatasource, Intervals.of("2000/3000"), "ver", ImmutableMap.of( @@ -81,6 +84,21 @@ public HadoopIngestionSpecUpdateDatasourcePathSpecSegmentsTest() 2 ); + private static final DataSegment SEGMENT2 = new DataSegment( + testDatasource2, + Intervals.of("2000/3000"), + "ver2", + ImmutableMap.of( + "type", "local", + "path", "/tmp/index2.zip" + ), + ImmutableList.of("host2"), + ImmutableList.of("visited_sum", "unique_hosts"), + NoneShardSpec.instance(), + 9, + 2 + ); + @Test public void testUpdateSegmentListIfDatasourcePathSpecIsUsedWithNoDatasourcePathSpec() throws Exception { @@ -213,6 +231,22 @@ public void testUpdateSegmentListIfDatasourcePathSpecIsUsedWithMultiplePathSpec( null ), null + ), + new DatasourcePathSpec( + jsonMapper, + null, + new DatasourceIngestionSpec( + testDatasource2, + testDatasourceInterval2, + null, + null, + null, + null, + null, + false, + null + ), + null ) ) ); @@ -224,6 +258,10 @@ public void testUpdateSegmentListIfDatasourcePathSpecIsUsedWithMultiplePathSpec( ImmutableList.of(WindowedDataSegment.of(SEGMENT)), ((DatasourcePathSpec) ((MultiplePathSpec) config.getPathSpec()).getChildren().get(1)).getSegments() ); + Assert.assertEquals( + ImmutableList.of(new WindowedDataSegment(SEGMENT2, testDatasourceInterval2)), + ((DatasourcePathSpec) ((MultiplePathSpec) config.getPathSpec()).getChildren().get(2)).getSegments() + ); } private HadoopDruidIndexerConfig testRunUpdateSegmentListIfDatasourcePathSpecIsUsed( @@ -259,9 +297,21 @@ private HadoopDruidIndexerConfig testRunUpdateSegmentListIfDatasourcePathSpecIsU ); UsedSegmentLister segmentLister = EasyMock.createMock(UsedSegmentLister.class); + EasyMock.expect( - segmentLister.getUsedSegmentsForIntervals(testDatasource, Lists.newArrayList(jobInterval)) + segmentLister.getUsedSegmentsForIntervals( + testDatasource, + Lists.newArrayList(jobInterval != null ? jobInterval.overlap(testDatasourceInterval) : null) + ) ).andReturn(ImmutableList.of(SEGMENT)); + + EasyMock.expect( + segmentLister.getUsedSegmentsForIntervals( + testDatasource2, + Lists.newArrayList(jobInterval != null ? jobInterval.overlap(testDatasourceInterval2) : null) + ) + ).andReturn(ImmutableList.of(SEGMENT2)); + EasyMock.replay(segmentLister); spec = HadoopIngestionSpec.updateSegmentListIfDatasourcePathSpecIsUsed(spec, jsonMapper, segmentLister); From cf0306708dd66f6bc67e5f123c181cb44589d3fa Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 5 Jul 2018 16:33:06 -0700 Subject: [PATCH 81/82] [Backport] Allow reordered segment allocation in kafka indexing service (#5943) * Allow reordered segment allocation in kafka indexing service (#5805) * Allow reordered segment allocation in kafka indexing service * address comments * fix a bug * commit remaining changes --- .../indexing/common/task/IndexTaskTest.java | 4 +- .../java/io/druid/java/util/common/Pair.java | 9 +- .../IndexerSQLMetadataStorageCoordinator.java | 557 +++++++++++------- .../appenderator/BaseAppenderatorDriver.java | 158 ++++- .../appenderator/BatchAppenderatorDriver.java | 61 +- .../StreamAppenderatorDriver.java | 58 +- .../BatchAppenderatorDriverTest.java | 2 +- 7 files changed, 575 insertions(+), 274 deletions(-) diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java index 7113efa6aa50..d7022ad91517 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/IndexTaskTest.java @@ -397,12 +397,12 @@ public void testAppendToExisting() throws Exception Assert.assertEquals("test", segments.get(0).getDataSource()); Assert.assertEquals(Intervals.of("2014/P1D"), segments.get(0).getInterval()); - Assert.assertTrue(segments.get(0).getShardSpec().getClass().equals(NumberedShardSpec.class)); + Assert.assertEquals(NumberedShardSpec.class, segments.get(0).getShardSpec().getClass()); Assert.assertEquals(0, segments.get(0).getShardSpec().getPartitionNum()); Assert.assertEquals("test", segments.get(1).getDataSource()); Assert.assertEquals(Intervals.of("2014/P1D"), segments.get(1).getInterval()); - Assert.assertTrue(segments.get(1).getShardSpec().getClass().equals(NumberedShardSpec.class)); + Assert.assertEquals(NumberedShardSpec.class, segments.get(1).getShardSpec().getClass()); Assert.assertEquals(1, segments.get(1).getShardSpec().getPartitionNum()); } diff --git a/java-util/src/main/java/io/druid/java/util/common/Pair.java b/java-util/src/main/java/io/druid/java/util/common/Pair.java index 62142622c70c..8bdd62172bdf 100644 --- a/java-util/src/main/java/io/druid/java/util/common/Pair.java +++ b/java-util/src/main/java/io/druid/java/util/common/Pair.java @@ -21,6 +21,7 @@ import com.google.common.base.Function; +import javax.annotation.Nullable; import java.util.Comparator; /** @@ -28,18 +29,20 @@ public class Pair { - public static Pair of(T1 lhs, T2 rhs) + public static Pair of(@Nullable T1 lhs, @Nullable T2 rhs) { return new Pair<>(lhs, rhs); } + @Nullable public final T1 lhs; + @Nullable public final T2 rhs; public Pair( - T1 lhs, - T2 rhs + @Nullable T1 lhs, + @Nullable T2 rhs ) { this.lhs = lhs; diff --git a/server/src/main/java/io/druid/metadata/IndexerSQLMetadataStorageCoordinator.java b/server/src/main/java/io/druid/metadata/IndexerSQLMetadataStorageCoordinator.java index 52067ab960c4..746e359127ef 100644 --- a/server/src/main/java/io/druid/metadata/IndexerSQLMetadataStorageCoordinator.java +++ b/server/src/main/java/io/druid/metadata/IndexerSQLMetadataStorageCoordinator.java @@ -19,6 +19,7 @@ package io.druid.metadata; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.base.Preconditions; @@ -39,6 +40,7 @@ import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; import io.druid.java.util.common.Intervals; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.StringUtils; import io.druid.java.util.common.lifecycle.LifecycleStart; import io.druid.java.util.common.logger.Logger; @@ -64,6 +66,7 @@ import org.skife.jdbi.v2.util.ByteArrayMapper; import org.skife.jdbi.v2.util.StringMapper; +import javax.annotation.Nullable; import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; @@ -383,7 +386,7 @@ public SegmentPublishResult inTransaction( public SegmentIdentifier allocatePendingSegment( final String dataSource, final String sequenceName, - final String previousSegmentId, + @Nullable final String previousSegmentId, final Interval interval, final String maxVersion, final boolean skipSegmentLineageCheck @@ -394,224 +397,376 @@ public SegmentIdentifier allocatePendingSegment( Preconditions.checkNotNull(interval, "interval"); Preconditions.checkNotNull(maxVersion, "maxVersion"); - final String previousSegmentIdNotNull = previousSegmentId == null ? "" : previousSegmentId; - return connector.retryTransaction( new TransactionCallback() { @Override public SegmentIdentifier inTransaction(Handle handle, TransactionStatus transactionStatus) throws Exception { - final List existingBytes; - if (!skipSegmentLineageCheck) { - existingBytes = handle - .createQuery( - StringUtils.format( - "SELECT payload FROM %s WHERE " - + "dataSource = :dataSource AND " - + "sequence_name = :sequence_name AND " - + "sequence_prev_id = :sequence_prev_id", - dbTables.getPendingSegmentsTable() - ) - ).bind("dataSource", dataSource) - .bind("sequence_name", sequenceName) - .bind("sequence_prev_id", previousSegmentIdNotNull) - .map(ByteArrayMapper.FIRST) - .list(); - } else { - existingBytes = handle - .createQuery( - StringUtils.format( - "SELECT payload FROM %s WHERE " - + "dataSource = :dataSource AND " - + "sequence_name = :sequence_name AND " - + "start = :start AND " - + "%2$send%2$s = :end", - dbTables.getPendingSegmentsTable(), connector.getQuoteString() - ) - ).bind("dataSource", dataSource) - .bind("sequence_name", sequenceName) - .bind("start", interval.getStart().toString()) - .bind("end", interval.getEnd().toString()) - .map(ByteArrayMapper.FIRST) - .list(); - } + return skipSegmentLineageCheck ? + allocatePendingSegment(handle, dataSource, sequenceName, interval, maxVersion) : + allocatePendingSegmentWithSegmentLineageCheck( + handle, + dataSource, + sequenceName, + previousSegmentId, + interval, + maxVersion + ); + } + }, + ALLOCATE_SEGMENT_QUIET_TRIES, + SQLMetadataConnector.DEFAULT_MAX_TRIES + ); + } - if (!existingBytes.isEmpty()) { - final SegmentIdentifier existingIdentifier = jsonMapper.readValue( - Iterables.getOnlyElement(existingBytes), - SegmentIdentifier.class - ); - - if (existingIdentifier.getInterval().getStartMillis() == interval.getStartMillis() - && existingIdentifier.getInterval().getEndMillis() == interval.getEndMillis()) { - log.info( - "Found existing pending segment [%s] for sequence[%s] (previous = [%s]) in DB", - existingIdentifier.getIdentifierAsString(), - sequenceName, - previousSegmentIdNotNull - ); + @Nullable + private SegmentIdentifier allocatePendingSegmentWithSegmentLineageCheck( + final Handle handle, + final String dataSource, + final String sequenceName, + @Nullable final String previousSegmentId, + final Interval interval, + final String maxVersion + ) throws IOException + { + final String previousSegmentIdNotNull = previousSegmentId == null ? "" : previousSegmentId; + final CheckExistingSegmentIdResult result = checkAndGetExistingSegmentId( + handle.createQuery( + StringUtils.format( + "SELECT payload FROM %s WHERE " + + "dataSource = :dataSource AND " + + "sequence_name = :sequence_name AND " + + "sequence_prev_id = :sequence_prev_id", + dbTables.getPendingSegmentsTable() + ) + ), + interval, + sequenceName, + previousSegmentIdNotNull, + Pair.of("dataSource", dataSource), + Pair.of("sequence_name", sequenceName), + Pair.of("sequence_prev_id", previousSegmentIdNotNull) + ); - return existingIdentifier; - } else { - log.warn( - "Cannot use existing pending segment [%s] for sequence[%s] (previous = [%s]) in DB, " - + "does not match requested interval[%s]", - existingIdentifier.getIdentifierAsString(), - sequenceName, - previousSegmentIdNotNull, - interval - ); + if (result.found) { + // The found existing segment identifier can be null if its interval doesn't match with the given interval + return result.segmentIdentifier; + } - return null; - } - } + final SegmentIdentifier newIdentifier = createNewSegment(handle, dataSource, interval, maxVersion); + if (newIdentifier == null) { + return null; + } - // Make up a pending segment based on existing segments and pending segments in the DB. This works - // assuming that all tasks inserting segments at a particular point in time are going through the - // allocatePendingSegment flow. This should be assured through some other mechanism (like task locks). + // SELECT -> INSERT can fail due to races; callers must be prepared to retry. + // Avoiding ON DUPLICATE KEY since it's not portable. + // Avoiding try/catch since it may cause inadvertent transaction-splitting. + + // UNIQUE key for the row, ensuring sequences do not fork in two directions. + // Using a single column instead of (sequence_name, sequence_prev_id) as some MySQL storage engines + // have difficulty with large unique keys (see https://github.com/druid-io/druid/issues/2319) + final String sequenceNamePrevIdSha1 = BaseEncoding.base16().encode( + Hashing.sha1() + .newHasher() + .putBytes(StringUtils.toUtf8(sequenceName)) + .putByte((byte) 0xff) + .putBytes(StringUtils.toUtf8(previousSegmentIdNotNull)) + .hash() + .asBytes() + ); - final SegmentIdentifier newIdentifier; + insertToMetastore( + handle, + newIdentifier, + dataSource, + interval, + previousSegmentIdNotNull, + sequenceName, + sequenceNamePrevIdSha1 + ); + return newIdentifier; + } - final List> existingChunks = getTimelineForIntervalsWithHandle( - handle, - dataSource, - ImmutableList.of(interval) - ).lookup(interval); - - if (existingChunks.size() > 1) { - // Not possible to expand more than one chunk with a single segment. - log.warn( - "Cannot allocate new segment for dataSource[%s], interval[%s], maxVersion[%s]: already have [%,d] chunks.", - dataSource, - interval, - maxVersion, - existingChunks.size() - ); - return null; - } else { - SegmentIdentifier max = null; - - if (!existingChunks.isEmpty()) { - TimelineObjectHolder existingHolder = Iterables.getOnlyElement(existingChunks); - for (PartitionChunk existing : existingHolder.getObject()) { - if (max == null || max.getShardSpec().getPartitionNum() < existing.getObject() - .getShardSpec() - .getPartitionNum()) { - max = SegmentIdentifier.fromDataSegment(existing.getObject()); - } - } - } + @Nullable + private SegmentIdentifier allocatePendingSegment( + final Handle handle, + final String dataSource, + final String sequenceName, + final Interval interval, + final String maxVersion + ) throws IOException + { + final CheckExistingSegmentIdResult result = checkAndGetExistingSegmentId( + handle.createQuery( + StringUtils.format( + "SELECT payload FROM %s WHERE " + + "dataSource = :dataSource AND " + + "sequence_name = :sequence_name AND " + + "start = :start AND " + + "%2$send%2$s = :end", + dbTables.getPendingSegmentsTable(), + connector.getQuoteString() + ) + ), + interval, + sequenceName, + null, + Pair.of("dataSource", dataSource), + Pair.of("sequence_name", sequenceName), + Pair.of("start", interval.getStart().toString()), + Pair.of("end", interval.getEnd().toString()) + ); - final List pendings = getPendingSegmentsForIntervalWithHandle( - handle, - dataSource, - interval - ); - - for (SegmentIdentifier pending : pendings) { - if (max == null || - pending.getVersion().compareTo(max.getVersion()) > 0 || - (pending.getVersion().equals(max.getVersion()) - && pending.getShardSpec().getPartitionNum() > max.getShardSpec().getPartitionNum())) { - max = pending; - } - } + if (result.found) { + // The found existing segment identifier can be null if its interval doesn't match with the given interval + return result.segmentIdentifier; + } - if (max == null) { - newIdentifier = new SegmentIdentifier( - dataSource, - interval, - maxVersion, - new NumberedShardSpec(0, 0) - ); - } else if (!max.getInterval().equals(interval) || max.getVersion().compareTo(maxVersion) > 0) { - log.warn( - "Cannot allocate new segment for dataSource[%s], interval[%s], maxVersion[%s]: conflicting segment[%s].", - dataSource, - interval, - maxVersion, - max.getIdentifierAsString() - ); - return null; - } else if (max.getShardSpec() instanceof LinearShardSpec) { - newIdentifier = new SegmentIdentifier( - dataSource, - max.getInterval(), - max.getVersion(), - new LinearShardSpec(max.getShardSpec().getPartitionNum() + 1) - ); - } else if (max.getShardSpec() instanceof NumberedShardSpec) { - newIdentifier = new SegmentIdentifier( - dataSource, - max.getInterval(), - max.getVersion(), - new NumberedShardSpec( - max.getShardSpec().getPartitionNum() + 1, - ((NumberedShardSpec) max.getShardSpec()).getPartitions() - ) - ); - } else { - log.warn( - "Cannot allocate new segment for dataSource[%s], interval[%s], maxVersion[%s]: ShardSpec class[%s] used by [%s].", - dataSource, - interval, - maxVersion, - max.getShardSpec().getClass(), - max.getIdentifierAsString() - ); - return null; - } - } + final SegmentIdentifier newIdentifier = createNewSegment(handle, dataSource, interval, maxVersion); + if (newIdentifier == null) { + return null; + } - // SELECT -> INSERT can fail due to races; callers must be prepared to retry. - // Avoiding ON DUPLICATE KEY since it's not portable. - // Avoiding try/catch since it may cause inadvertent transaction-splitting. - - // UNIQUE key for the row, ensuring sequences do not fork in two directions. - // Using a single column instead of (sequence_name, sequence_prev_id) as some MySQL storage engines - // have difficulty with large unique keys (see https://github.com/druid-io/druid/issues/2319) - final String sequenceNamePrevIdSha1 = BaseEncoding.base16().encode( - Hashing.sha1() - .newHasher() - .putBytes(StringUtils.toUtf8(sequenceName)) - .putByte((byte) 0xff) - .putBytes(StringUtils.toUtf8(previousSegmentIdNotNull)) - .hash() - .asBytes() - ); + // SELECT -> INSERT can fail due to races; callers must be prepared to retry. + // Avoiding ON DUPLICATE KEY since it's not portable. + // Avoiding try/catch since it may cause inadvertent transaction-splitting. + + // UNIQUE key for the row, ensuring we don't have more than one segment per sequence per interval. + // Using a single column instead of (sequence_name, sequence_prev_id) as some MySQL storage engines + // have difficulty with large unique keys (see https://github.com/druid-io/druid/issues/2319) + final String sequenceNamePrevIdSha1 = BaseEncoding.base16().encode( + Hashing.sha1() + .newHasher() + .putBytes(StringUtils.toUtf8(sequenceName)) + .putByte((byte) 0xff) + .putLong(interval.getStartMillis()) + .putLong(interval.getEndMillis()) + .hash() + .asBytes() + ); - handle.createStatement( - StringUtils.format( - "INSERT INTO %1$s (id, dataSource, created_date, start, %2$send%2$s, sequence_name, sequence_prev_id, sequence_name_prev_id_sha1, payload) " - + "VALUES (:id, :dataSource, :created_date, :start, :end, :sequence_name, :sequence_prev_id, :sequence_name_prev_id_sha1, :payload)", - dbTables.getPendingSegmentsTable(), connector.getQuoteString() - ) - ) - .bind("id", newIdentifier.getIdentifierAsString()) - .bind("dataSource", dataSource) - .bind("created_date", DateTimes.nowUtc().toString()) - .bind("start", interval.getStart().toString()) - .bind("end", interval.getEnd().toString()) - .bind("sequence_name", sequenceName) - .bind("sequence_prev_id", previousSegmentIdNotNull) - .bind("sequence_name_prev_id_sha1", sequenceNamePrevIdSha1) - .bind("payload", jsonMapper.writeValueAsBytes(newIdentifier)) - .execute(); - - log.info( - "Allocated pending segment [%s] for sequence[%s] (previous = [%s]) in DB", - newIdentifier.getIdentifierAsString(), - sequenceName, - previousSegmentIdNotNull - ); + // always insert empty previous sequence id + insertToMetastore(handle, newIdentifier, dataSource, interval, "", sequenceName, sequenceNamePrevIdSha1); - return newIdentifier; - } - }, - ALLOCATE_SEGMENT_QUIET_TRIES, - SQLMetadataConnector.DEFAULT_MAX_TRIES + log.info( + "Allocated pending segment [%s] for sequence[%s] in DB", + newIdentifier.getIdentifierAsString(), + sequenceName ); + + return newIdentifier; + } + + private CheckExistingSegmentIdResult checkAndGetExistingSegmentId( + final Query> query, + final Interval interval, + final String sequenceName, + final @Nullable String previousSegmentId, + final Pair... queryVars + ) throws IOException + { + Query> boundQuery = query; + for (Pair var : queryVars) { + boundQuery = boundQuery.bind(var.lhs, var.rhs); + } + final List existingBytes = boundQuery.map(ByteArrayMapper.FIRST).list(); + + if (!existingBytes.isEmpty()) { + final SegmentIdentifier existingIdentifier = jsonMapper.readValue( + Iterables.getOnlyElement(existingBytes), + SegmentIdentifier.class + ); + + if (existingIdentifier.getInterval().getStartMillis() == interval.getStartMillis() + && existingIdentifier.getInterval().getEndMillis() == interval.getEndMillis()) { + if (previousSegmentId == null) { + log.info( + "Found existing pending segment [%s] for sequence[%s] in DB", + existingIdentifier.getIdentifierAsString(), + sequenceName + ); + } else { + log.info( + "Found existing pending segment [%s] for sequence[%s] (previous = [%s]) in DB", + existingIdentifier.getIdentifierAsString(), + sequenceName, + previousSegmentId + ); + } + + return new CheckExistingSegmentIdResult(true, existingIdentifier); + } else { + if (previousSegmentId == null) { + log.warn( + "Cannot use existing pending segment [%s] for sequence[%s] in DB, " + + "does not match requested interval[%s]", + existingIdentifier.getIdentifierAsString(), + sequenceName, + interval + ); + } else { + log.warn( + "Cannot use existing pending segment [%s] for sequence[%s] (previous = [%s]) in DB, " + + "does not match requested interval[%s]", + existingIdentifier.getIdentifierAsString(), + sequenceName, + previousSegmentId, + interval + ); + } + + return new CheckExistingSegmentIdResult(true, null); + } + } + return new CheckExistingSegmentIdResult(false, null); + } + + private static class CheckExistingSegmentIdResult + { + private final boolean found; + @Nullable + private final SegmentIdentifier segmentIdentifier; + + CheckExistingSegmentIdResult(boolean found, @Nullable SegmentIdentifier segmentIdentifier) + { + this.found = found; + this.segmentIdentifier = segmentIdentifier; + } + } + + private void insertToMetastore( + Handle handle, + SegmentIdentifier newIdentifier, + String dataSource, + Interval interval, + String previousSegmentId, + String sequenceName, + String sequenceNamePrevIdSha1 + ) throws JsonProcessingException + { + handle.createStatement( + StringUtils.format( + "INSERT INTO %1$s (id, dataSource, created_date, start, %2$send%2$s, sequence_name, sequence_prev_id, sequence_name_prev_id_sha1, payload) " + + "VALUES (:id, :dataSource, :created_date, :start, :end, :sequence_name, :sequence_prev_id, :sequence_name_prev_id_sha1, :payload)", + dbTables.getPendingSegmentsTable(), + connector.getQuoteString() + ) + ) + .bind("id", newIdentifier.getIdentifierAsString()) + .bind("dataSource", dataSource) + .bind("created_date", DateTimes.nowUtc().toString()) + .bind("start", interval.getStart().toString()) + .bind("end", interval.getEnd().toString()) + .bind("sequence_name", sequenceName) + .bind("sequence_prev_id", previousSegmentId) + .bind("sequence_name_prev_id_sha1", sequenceNamePrevIdSha1) + .bind("payload", jsonMapper.writeValueAsBytes(newIdentifier)) + .execute(); + } + + @Nullable + private SegmentIdentifier createNewSegment( + final Handle handle, + final String dataSource, + final Interval interval, + final String maxVersion + ) throws IOException + { + // Make up a pending segment based on existing segments and pending segments in the DB. This works + // assuming that all tasks inserting segments at a particular point in time are going through the + // allocatePendingSegment flow. This should be assured through some other mechanism (like task locks). + + final List> existingChunks = getTimelineForIntervalsWithHandle( + handle, + dataSource, + ImmutableList.of(interval) + ).lookup(interval); + + if (existingChunks.size() > 1) { + // Not possible to expand more than one chunk with a single segment. + log.warn( + "Cannot allocate new segment for dataSource[%s], interval[%s], maxVersion[%s]: already have [%,d] chunks.", + dataSource, + interval, + maxVersion, + existingChunks.size() + ); + return null; + } else { + SegmentIdentifier max = null; + + if (!existingChunks.isEmpty()) { + TimelineObjectHolder existingHolder = Iterables.getOnlyElement(existingChunks); + for (PartitionChunk existing : existingHolder.getObject()) { + if (max == null || max.getShardSpec().getPartitionNum() < existing.getObject() + .getShardSpec() + .getPartitionNum()) { + max = SegmentIdentifier.fromDataSegment(existing.getObject()); + } + } + } + + final List pendings = getPendingSegmentsForIntervalWithHandle( + handle, + dataSource, + interval + ); + + for (SegmentIdentifier pending : pendings) { + if (max == null || + pending.getVersion().compareTo(max.getVersion()) > 0 || + (pending.getVersion().equals(max.getVersion()) + && pending.getShardSpec().getPartitionNum() > max.getShardSpec().getPartitionNum())) { + max = pending; + } + } + + if (max == null) { + return new SegmentIdentifier( + dataSource, + interval, + maxVersion, + new NumberedShardSpec(0, 0) + ); + } else if (!max.getInterval().equals(interval) || max.getVersion().compareTo(maxVersion) > 0) { + log.warn( + "Cannot allocate new segment for dataSource[%s], interval[%s], maxVersion[%s]: conflicting segment[%s].", + dataSource, + interval, + maxVersion, + max.getIdentifierAsString() + ); + return null; + } else if (max.getShardSpec() instanceof LinearShardSpec) { + return new SegmentIdentifier( + dataSource, + max.getInterval(), + max.getVersion(), + new LinearShardSpec(max.getShardSpec().getPartitionNum() + 1) + ); + } else if (max.getShardSpec() instanceof NumberedShardSpec) { + return new SegmentIdentifier( + dataSource, + max.getInterval(), + max.getVersion(), + new NumberedShardSpec( + max.getShardSpec().getPartitionNum() + 1, + ((NumberedShardSpec) max.getShardSpec()).getPartitions() + ) + ); + } else { + log.warn( + "Cannot allocate new segment for dataSource[%s], interval[%s], maxVersion[%s]: ShardSpec class[%s] used by [%s].", + dataSource, + interval, + maxVersion, + max.getShardSpec().getClass(), + max.getIdentifierAsString() + ); + return null; + } + } } @Override diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/BaseAppenderatorDriver.java b/server/src/main/java/io/druid/segment/realtime/appenderator/BaseAppenderatorDriver.java index c578a65d6cd6..1f04236c41c7 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/BaseAppenderatorDriver.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/BaseAppenderatorDriver.java @@ -42,12 +42,13 @@ import io.druid.segment.loading.DataSegmentKiller; import io.druid.segment.realtime.appenderator.SegmentWithState.SegmentState; import org.joda.time.DateTime; +import org.joda.time.Interval; import javax.annotation.Nullable; import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -55,6 +56,7 @@ import java.util.Objects; import java.util.Set; import java.util.TreeMap; +import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -75,6 +77,94 @@ */ public abstract class BaseAppenderatorDriver implements Closeable { + /** + * Segments allocated for an intervval. + * There should be at most a single active (appending) segment at any time. + */ + static class SegmentsOfInterval + { + private final Interval interval; + private final List appendFinishedSegments = new ArrayList<>(); + + @Nullable + private SegmentWithState appendingSegment; + + SegmentsOfInterval(Interval interval) + { + this.interval = interval; + } + + SegmentsOfInterval( + Interval interval, + @Nullable SegmentWithState appendingSegment, + List appendFinishedSegments + ) + { + this.interval = interval; + this.appendingSegment = appendingSegment; + this.appendFinishedSegments.addAll(appendFinishedSegments); + + if (appendingSegment != null) { + Preconditions.checkArgument( + appendingSegment.getState() == SegmentState.APPENDING, + "appendingSegment[%s] is not in the APPENDING state", + appendingSegment.getSegmentIdentifier() + ); + } + if (appendFinishedSegments + .stream() + .anyMatch(segmentWithState -> segmentWithState.getState() == SegmentState.APPENDING)) { + throw new ISE("Some appendFinishedSegments[%s] is in the APPENDING state", appendFinishedSegments); + } + } + + void setAppendingSegment(SegmentWithState appendingSegment) + { + Preconditions.checkArgument( + appendingSegment.getState() == SegmentState.APPENDING, + "segment[%s] is not in the APPENDING state", + appendingSegment.getSegmentIdentifier() + ); + // There should be only one appending segment at any time + Preconditions.checkState( + this.appendingSegment == null, + "WTF?! Current appendingSegment[%s] is not null. " + + "Its state must be changed before setting a new appendingSegment[%s]", + this.appendingSegment, + appendingSegment + ); + this.appendingSegment = appendingSegment; + } + + void finishAppendingToCurrentActiveSegment(Consumer stateTransitionFn) + { + Preconditions.checkNotNull(appendingSegment, "appendingSegment"); + stateTransitionFn.accept(appendingSegment); + appendFinishedSegments.add(appendingSegment); + appendingSegment = null; + } + + Interval getInterval() + { + return interval; + } + + SegmentWithState getAppendingSegment() + { + return appendingSegment; + } + + List getAllSegments() + { + final List allSegments = new ArrayList<>(appendFinishedSegments.size() + 1); + if (appendingSegment != null) { + allSegments.add(appendingSegment); + } + allSegments.addAll(appendFinishedSegments); + return allSegments; + } + } + /** * Allocated segments for a sequence */ @@ -83,7 +173,7 @@ static class SegmentsForSequence // Interval Start millis -> List of Segments for this interval // there might be multiple segments for a start interval, for example one segment // can be in APPENDING state and others might be in PUBLISHING state - private final NavigableMap> intervalToSegmentStates; + private final NavigableMap intervalToSegmentStates; // most recently allocated segment private String lastSegmentId; @@ -94,7 +184,7 @@ static class SegmentsForSequence } SegmentsForSequence( - NavigableMap> intervalToSegmentStates, + NavigableMap intervalToSegmentStates, String lastSegmentId ) { @@ -104,24 +194,34 @@ static class SegmentsForSequence void add(SegmentIdentifier identifier) { - intervalToSegmentStates.computeIfAbsent(identifier.getInterval().getStartMillis(), k -> new LinkedList<>()) - .addFirst(SegmentWithState.newSegment(identifier)); + intervalToSegmentStates.computeIfAbsent( + identifier.getInterval().getStartMillis(), + k -> new SegmentsOfInterval(identifier.getInterval()) + ).setAppendingSegment(SegmentWithState.newSegment(identifier)); lastSegmentId = identifier.getIdentifierAsString(); } - Entry> floor(long timestamp) + Entry floor(long timestamp) { return intervalToSegmentStates.floorEntry(timestamp); } - LinkedList get(long timestamp) + SegmentsOfInterval get(long timestamp) { return intervalToSegmentStates.get(timestamp); } - Stream segmentStateStream() + Stream allSegmentStateStream() + { + return intervalToSegmentStates + .values() + .stream() + .flatMap(segmentsOfInterval -> segmentsOfInterval.getAllSegments().stream()); + } + + Stream getAllSegmentsOfInterval() { - return intervalToSegmentStates.values().stream().flatMap(Collection::stream); + return intervalToSegmentStates.values().stream(); } } @@ -183,13 +283,19 @@ private SegmentIdentifier getAppendableSegment(final DateTime timestamp, final S return null; } - final Map.Entry> candidateEntry = segmentsForSequence.floor( + final Map.Entry candidateEntry = segmentsForSequence.floor( timestamp.getMillis() ); - if (candidateEntry != null - && candidateEntry.getValue().getFirst().getSegmentIdentifier().getInterval().contains(timestamp) - && candidateEntry.getValue().getFirst().getState() == SegmentState.APPENDING) { - return candidateEntry.getValue().getFirst().getSegmentIdentifier(); + + if (candidateEntry != null) { + final SegmentsOfInterval segmentsOfInterval = candidateEntry.getValue(); + if (segmentsOfInterval.interval.contains(timestamp)) { + return segmentsOfInterval.appendingSegment == null ? + null : + segmentsOfInterval.appendingSegment.getSegmentIdentifier(); + } else { + return null; + } } else { return null; } @@ -326,7 +432,20 @@ Stream getSegmentWithStates(Collection sequenceNames) .map(segments::get) .filter(Objects::nonNull) .flatMap(segmentsForSequence -> segmentsForSequence.intervalToSegmentStates.values().stream()) - .flatMap(Collection::stream); + .flatMap(segmentsOfInterval -> segmentsOfInterval.getAllSegments().stream()); + } + } + + Stream getAppendingSegments(Collection sequenceNames) + { + synchronized (segments) { + return sequenceNames + .stream() + .map(segments::get) + .filter(Objects::nonNull) + .flatMap(segmentsForSequence -> segmentsForSequence.intervalToSegmentStates.values().stream()) + .map(segmentsOfInterval -> segmentsOfInterval.appendingSegment) + .filter(Objects::nonNull); } } @@ -537,10 +656,11 @@ WrappedCommitter wrapCommitter(final Committer committer) Maps.transformValues( snapshot, (Function>) input -> ImmutableList.copyOf( - input.intervalToSegmentStates.values() - .stream() - .flatMap(Collection::stream) - .collect(Collectors.toList()) + input.intervalToSegmentStates + .values() + .stream() + .flatMap(segmentsOfInterval -> segmentsOfInterval.getAllSegments().stream()) + .collect(Collectors.toList()) ) ) ), diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriver.java b/server/src/main/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriver.java index 4ceb5264097f..7d6363796435 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriver.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriver.java @@ -33,11 +33,11 @@ import javax.annotation.Nullable; import java.io.IOException; import java.util.Collection; -import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -129,13 +129,11 @@ private SegmentsAndMetadata pushAndClear( long pushAndClearTimeoutMs ) throws InterruptedException, ExecutionException, TimeoutException { - final List segmentIdentifierList = getSegmentWithStates(sequenceNames) - .filter(segmentWithState -> segmentWithState.getState() == SegmentState.APPENDING) - .map(SegmentWithState::getSegmentIdentifier) - .collect(Collectors.toList()); + final Map requestedSegmentIdsForSequences = getAppendingSegments(sequenceNames) + .collect(Collectors.toMap(SegmentWithState::getSegmentIdentifier, Function.identity())); final ListenableFuture future = Futures.transform( - pushInBackground(null, segmentIdentifierList, false), + pushInBackground(null, requestedSegmentIdsForSequences.keySet(), false), this::dropInBackground ); @@ -147,17 +145,7 @@ private SegmentsAndMetadata pushAndClear( final Map pushedSegmentIdToSegmentMap = segmentsAndMetadata .getSegments() .stream() - .collect(Collectors.toMap( - SegmentIdentifier::fromDataSegment, - dataSegment -> dataSegment - )); - - final Map requestedSegmentIdsForSequences = getSegmentWithStates(sequenceNames) - .filter(segmentWithState -> segmentWithState.getState() == SegmentState.APPENDING) - .collect(Collectors.toMap( - SegmentWithState::getSegmentIdentifier, - segmentWithState -> segmentWithState - )); + .collect(Collectors.toMap(SegmentIdentifier::fromDataSegment, Function.identity())); if (!pushedSegmentIdToSegmentMap.keySet().equals(requestedSegmentIdsForSequences.keySet())) { throw new ISE( @@ -167,12 +155,28 @@ private SegmentsAndMetadata pushAndClear( ); } - // State transition - requestedSegmentIdsForSequences.forEach( - (segmentId, segmentWithState) -> { - segmentWithState.pushAndDrop(pushedSegmentIdToSegmentMap.get(segmentId)); + synchronized (segments) { + for (String sequenceName : sequenceNames) { + final SegmentsForSequence segmentsForSequence = segments.get(sequenceName); + if (segmentsForSequence == null) { + throw new ISE("Can't find segmentsForSequence for sequence[%s]", sequenceName); } - ); + + segmentsForSequence.getAllSegmentsOfInterval().forEach(segmentsOfInterval -> { + final SegmentWithState appendingSegment = segmentsOfInterval.getAppendingSegment(); + if (appendingSegment != null) { + final DataSegment pushedSegment = pushedSegmentIdToSegmentMap.get(appendingSegment.getSegmentIdentifier()); + if (pushedSegment == null) { + throw new ISE("Can't find pushedSegments for segment[%s]", appendingSegment.getSegmentIdentifier()); + } + + segmentsOfInterval.finishAppendingToCurrentActiveSegment( + segmentWithState -> segmentWithState.pushAndDrop(pushedSegment) + ); + } + }); + } + } return segmentsAndMetadata; } @@ -196,12 +200,13 @@ public ListenableFuture publishAll(final TransactionalSegme snapshot .values() .stream() - .flatMap(SegmentsForSequence::segmentStateStream) - .map(segmentWithState -> Preconditions.checkNotNull( - segmentWithState.getDataSegment(), - "dataSegment for segmentId[%s]", - segmentWithState.getSegmentIdentifier() - ) + .flatMap(SegmentsForSequence::allSegmentStateStream) + .map(segmentWithState -> Preconditions + .checkNotNull( + segmentWithState.getDataSegment(), + "dataSegment for segmentId[%s]", + segmentWithState.getSegmentIdentifier() + ) ) .collect(Collectors.toList()), null diff --git a/server/src/main/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriver.java b/server/src/main/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriver.java index 4b0c171ef8a9..6d1109202eaa 100644 --- a/server/src/main/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriver.java +++ b/server/src/main/java/io/druid/segment/realtime/appenderator/StreamAppenderatorDriver.java @@ -33,6 +33,8 @@ import io.druid.data.input.Committer; import io.druid.data.input.InputRow; import io.druid.java.util.common.ISE; +import io.druid.java.util.common.Pair; +import io.druid.java.util.common.guava.Comparators; import io.druid.java.util.common.logger.Logger; import io.druid.query.SegmentDescriptor; import io.druid.segment.loading.DataSegmentKiller; @@ -43,8 +45,9 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; -import java.util.LinkedList; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -185,18 +188,13 @@ public void moveSegmentOut(final String sequenceName, final List { - if (segmentWithState.getSegmentIdentifier().equals(identifier)) { - segmentWithState.finishAppending(); - return true; - } else { - return false; - } - } - )) { + final SegmentsOfInterval segmentsOfInterval = activeSegmentsForSequence.get(key); + if (segmentsOfInterval == null || + segmentsOfInterval.getAppendingSegment() == null || + !segmentsOfInterval.getAppendingSegment().getSegmentIdentifier().equals(identifier)) { throw new ISE("WTF?! Asked to remove segment[%s] that didn't exist...", identifier); } + segmentsOfInterval.finishAppendingToCurrentActiveSegment(SegmentWithState::finishAppending); } } } @@ -388,33 +386,53 @@ public void close() private static class SegmentsForSequenceBuilder { - private final NavigableMap> intervalToSegmentStates; + // segmentId -> (appendingSegment, appendFinishedSegments) + private final NavigableMap>> intervalToSegments = + new TreeMap<>(Comparator.comparing(SegmentIdentifier::getInterval, Comparators.intervalsByStartThenEnd())); private final String lastSegmentId; SegmentsForSequenceBuilder(String lastSegmentId) { - this.intervalToSegmentStates = new TreeMap<>(); this.lastSegmentId = lastSegmentId; } void add(SegmentWithState segmentWithState) { final SegmentIdentifier identifier = segmentWithState.getSegmentIdentifier(); - final LinkedList segmentsInInterval = intervalToSegmentStates.computeIfAbsent( - identifier.getInterval().getStartMillis(), - k -> new LinkedList<>() - ); + final Pair> pair = intervalToSegments.get(identifier); + final List appendFinishedSegments = pair == null || pair.rhs == null ? + new ArrayList<>() : + pair.rhs; + // always keep APPENDING segments for an interval start millis in the front if (segmentWithState.getState() == SegmentState.APPENDING) { - segmentsInInterval.addFirst(segmentWithState); + if (pair != null && pair.lhs != null) { + throw new ISE( + "WTF?! there was already an appendingSegment[%s] before adding an appendingSegment[%s]", + pair.lhs, + segmentWithState + ); + } + + intervalToSegments.put(identifier, Pair.of(segmentWithState, appendFinishedSegments)); } else { - segmentsInInterval.addLast(segmentWithState); + final SegmentWithState appendingSegment = pair == null ? null : pair.lhs; + appendFinishedSegments.add(segmentWithState); + intervalToSegments.put(identifier, Pair.of(appendingSegment, appendFinishedSegments)); } } SegmentsForSequence build() { - return new SegmentsForSequence(intervalToSegmentStates, lastSegmentId); + final NavigableMap map = new TreeMap<>(); + for (Entry>> entry : + intervalToSegments.entrySet()) { + map.put( + entry.getKey().getInterval().getStartMillis(), + new SegmentsOfInterval(entry.getKey().getInterval(), entry.getValue().lhs, entry.getValue().rhs) + ); + } + return new SegmentsForSequence(map, lastSegmentId); } } } diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriverTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriverTest.java index 5ca72100a945..2fd0f087d38a 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriverTest.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/BatchAppenderatorDriverTest.java @@ -185,7 +185,7 @@ private void checkSegmentStates(int expectedNumSegmentsInState, SegmentState exp final SegmentsForSequence segmentsForSequence = driver.getSegments().get("dummy"); Assert.assertNotNull(segmentsForSequence); final List segmentWithStates = segmentsForSequence - .segmentStateStream() + .allSegmentStateStream() .filter(segmentWithState -> segmentWithState.getState() == expectedState) .collect(Collectors.toList()); From df7e822f93388ba42310a52e5029d9678d297dd5 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Mon, 2 Apr 2018 09:40:20 -0700 Subject: [PATCH 82/82] Coordinator primary segment assignment fix (#5532) * fix issue where assign primary assigns segments to all historical servers in cluster * fix test * add test to ensure primary assignment will not assign to another server while loading is in progress --- .../coordinator/SegmentReplicantLookup.java | 43 +++++- .../server/coordinator/rules/LoadRule.java | 8 +- .../DruidCoordinatorRuleRunnerTest.java | 3 + .../coordinator/rules/LoadRuleTest.java | 139 ++++++++++++++++++ 4 files changed, 187 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/io/druid/server/coordinator/SegmentReplicantLookup.java b/server/src/main/java/io/druid/server/coordinator/SegmentReplicantLookup.java index a713c8d181c1..357eeb99be7c 100644 --- a/server/src/main/java/io/druid/server/coordinator/SegmentReplicantLookup.java +++ b/server/src/main/java/io/druid/server/coordinator/SegmentReplicantLookup.java @@ -36,6 +36,7 @@ public class SegmentReplicantLookup public static SegmentReplicantLookup make(DruidCluster cluster) { final Table segmentsInCluster = HashBasedTable.create(); + final Table loadingSegments = HashBasedTable.create(); for (SortedSet serversByType : cluster.getSortedHistoricalsByTier()) { for (ServerHolder serverHolder : serversByType) { @@ -48,17 +49,29 @@ public static SegmentReplicantLookup make(DruidCluster cluster) } segmentsInCluster.put(segment.getIdentifier(), server.getTier(), ++numReplicants); } + + // Also account for queued segments + for (DataSegment segment : serverHolder.getPeon().getSegmentsToLoad()) { + Integer numReplicants = loadingSegments.get(segment.getIdentifier(), server.getTier()); + if (numReplicants == null) { + numReplicants = 0; + } + loadingSegments.put(segment.getIdentifier(), server.getTier(), ++numReplicants); + } } } - return new SegmentReplicantLookup(segmentsInCluster); + return new SegmentReplicantLookup(segmentsInCluster, loadingSegments); } private final Table segmentsInCluster; - private SegmentReplicantLookup(Table segmentsInCluster) + private final Table loadingSegments; + + private SegmentReplicantLookup(Table segmentsInCluster, Table loadingSegments) { this.segmentsInCluster = segmentsInCluster; + this.loadingSegments = loadingSegments; } public Map getClusterTiers(String segmentId) @@ -82,4 +95,30 @@ public int getLoadedReplicants(String segmentId, String tier) Integer retVal = segmentsInCluster.get(segmentId, tier); return (retVal == null) ? 0 : retVal; } + + public int getLoadingReplicants(String segmentId, String tier) + { + Integer retVal = loadingSegments.get(segmentId, tier); + return (retVal == null) ? 0 : retVal; + } + + public int getLoadingReplicants(String segmentId) + { + Map allTiers = loadingSegments.row(segmentId); + int retVal = 0; + for (Integer replicants : allTiers.values()) { + retVal += replicants; + } + return retVal; + } + + public int getTotalReplicants(String segmentId) + { + return getLoadedReplicants(segmentId) + getLoadingReplicants(segmentId); + } + + public int getTotalReplicants(String segmentId, String tier) + { + return getLoadedReplicants(segmentId, tier) + getLoadingReplicants(segmentId, tier); + } } diff --git a/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java b/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java index 5af4d822c4b3..dc615f800518 100644 --- a/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java +++ b/server/src/main/java/io/druid/server/coordinator/rules/LoadRule.java @@ -95,8 +95,9 @@ private void assign( final CoordinatorStats stats ) { - // if primary replica already exists - if (!currentReplicants.isEmpty()) { + // if primary replica already exists or is loading + final int loading = params.getSegmentReplicantLookup().getTotalReplicants(segment.getIdentifier()); + if (!currentReplicants.isEmpty() || loading > 0) { assignReplicas(params, segment, stats, null); } else { final ServerHolder primaryHolderToLoad = assignPrimary(params, segment); @@ -169,7 +170,6 @@ private ServerHolder assignPrimary( if (targetReplicantsInTier <= 0) { continue; } - final String tier = entry.getKey(); final List holders = getFilteredHolders( @@ -228,7 +228,7 @@ private void assignReplicas( final int numAssigned = assignReplicasForTier( tier, entry.getIntValue(), - currentReplicants.getOrDefault(tier, 0), + params.getSegmentReplicantLookup().getTotalReplicants(segment.getIdentifier(), tier), params, createLoadQueueSizeLimitingPredicate(params), segment diff --git a/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorRuleRunnerTest.java b/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorRuleRunnerTest.java index 9c17bab60c38..9fd6d0bdbf86 100644 --- a/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorRuleRunnerTest.java +++ b/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorRuleRunnerTest.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import io.druid.java.util.emitter.EmittingLogger; @@ -942,6 +943,8 @@ public void testDropServerActuallyServesSegment() throws Exception LoadQueuePeon anotherMockPeon = EasyMock.createMock(LoadQueuePeon.class); EasyMock.expect(anotherMockPeon.getLoadQueueSize()).andReturn(10L).atLeastOnce(); + EasyMock.expect(anotherMockPeon.getSegmentsToLoad()).andReturn(Sets.newHashSet()).anyTimes(); + EasyMock.replay(anotherMockPeon); DruidCluster druidCluster = new DruidCluster( diff --git a/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java b/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java index 0388318b7262..c7a5e4f930b2 100644 --- a/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java +++ b/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java @@ -20,7 +20,9 @@ package io.druid.server.coordinator.rules; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -60,7 +62,9 @@ import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeSet; import java.util.concurrent.Executors; import java.util.stream.Collectors; @@ -189,6 +193,127 @@ public void testLoad() throws Exception EasyMock.verify(throttler, mockPeon, mockBalancerStrategy); } + @Test + public void testLoadPrimaryAssignDoesNotOverAssign() + { + EasyMock.expect(throttler.canCreateReplicant(EasyMock.anyString())).andReturn(true).anyTimes(); + + final LoadQueuePeon mockPeon = createEmptyPeon(); + mockPeon.loadSegment(EasyMock.anyObject(), EasyMock.anyObject()); + EasyMock.expectLastCall().atLeastOnce(); + + LoadRule rule = createLoadRule(ImmutableMap.of( + "hot", 1 + )); + + final DataSegment segment = createDataSegment("foo"); + + EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(EasyMock.anyObject(), EasyMock.anyObject())) + .andDelegateTo(balancerStrategy) + .anyTimes(); + + EasyMock.replay(throttler, mockPeon, mockBalancerStrategy); + + DruidCluster druidCluster = new DruidCluster( + null, + ImmutableMap.of( + "hot", + Stream.of( + new ServerHolder( + new DruidServer( + "serverHot", + "hostHot", + null, + 1000, + ServerType.HISTORICAL, + "hot", + 1 + ).toImmutableDruidServer(), + mockPeon + ), new ServerHolder( + new DruidServer( + "serverHot2", + "hostHot2", + null, + 1000, + ServerType.HISTORICAL, + "hot", + 1 + ).toImmutableDruidServer(), + mockPeon + ) + ).collect(Collectors.toCollection(() -> new TreeSet<>(Collections.reverseOrder()))) + ) + ); + + CoordinatorStats stats = rule.run( + null, + DruidCoordinatorRuntimeParams.newBuilder() + .withDruidCluster(druidCluster) + .withSegmentReplicantLookup(SegmentReplicantLookup.make(druidCluster)) + .withReplicationManager(throttler) + .withBalancerStrategy(mockBalancerStrategy) + .withBalancerReferenceTimestamp(DateTimes.of("2013-01-01")) + .withAvailableSegments(Arrays.asList(segment)).build(), + segment + ); + + + Assert.assertEquals(1L, stats.getTieredStat(LoadRule.ASSIGNED_COUNT, "hot")); + + // ensure multiple runs don't assign primary segment again if at replication count + final LoadQueuePeon loadingPeon = createLoadingPeon(ImmutableList.of(segment)); + EasyMock.replay(loadingPeon); + + DruidCluster afterLoad = new DruidCluster( + null, + ImmutableMap.of( + "hot", + Stream.of( + new ServerHolder( + new DruidServer( + "serverHot", + "hostHot", + null, + 1000, + ServerType.HISTORICAL, + "hot", + 1 + ).toImmutableDruidServer(), + loadingPeon + ), new ServerHolder( + new DruidServer( + "serverHot2", + "hostHot2", + null, + 1000, + ServerType.HISTORICAL, + "hot", + 1 + ).toImmutableDruidServer(), + mockPeon + ) + ).collect(Collectors.toCollection(() -> new TreeSet<>(Collections.reverseOrder()))) + ) + ); + CoordinatorStats statsAfterLoadPrimary = rule.run( + null, + DruidCoordinatorRuntimeParams.newBuilder() + .withDruidCluster(afterLoad) + .withSegmentReplicantLookup(SegmentReplicantLookup.make(afterLoad)) + .withReplicationManager(throttler) + .withBalancerStrategy(mockBalancerStrategy) + .withBalancerReferenceTimestamp(DateTimes.of("2013-01-01")) + .withAvailableSegments(Arrays.asList(segment)).build(), + segment + ); + + + Assert.assertEquals(0, statsAfterLoadPrimary.getTieredStat(LoadRule.ASSIGNED_COUNT, "hot")); + + EasyMock.verify(throttler, mockPeon, mockBalancerStrategy); + } + @Test public void testLoadPriority() throws Exception { @@ -618,4 +743,18 @@ private static LoadQueuePeon createEmptyPeon() return mockPeon; } + + private static LoadQueuePeon createLoadingPeon(List segments) + { + final Set segs = ImmutableSet.copyOf(segments); + final long loadingSize = segs.stream().mapToLong(DataSegment::getSize).sum(); + + final LoadQueuePeon mockPeon = EasyMock.createMock(LoadQueuePeon.class); + EasyMock.expect(mockPeon.getSegmentsToLoad()).andReturn(segs).anyTimes(); + EasyMock.expect(mockPeon.getSegmentsMarkedToDrop()).andReturn(Sets.newHashSet()).anyTimes(); + EasyMock.expect(mockPeon.getLoadQueueSize()).andReturn(loadingSize).anyTimes(); + EasyMock.expect(mockPeon.getNumberOfSegmentsInQueue()).andReturn(segs.size()).anyTimes(); + + return mockPeon; + } }