From 222f732449529ed5e2f39b93f9d6feb65bb79f6d Mon Sep 17 00:00:00 2001 From: leventov Date: Fri, 21 Sep 2018 14:54:43 +0400 Subject: [PATCH 01/27] Introduce SegmentId class --- .idea/inspectionProfiles/Druid.xml | 9 +- .../org/apache/druid/data/input/Rows.java | 7 +- .../segment/loading/DataSegmentFinder.java | 9 +- .../apache/druid/timeline/DataSegment.java | 99 ++--- .../druid/timeline/DataSegmentUtils.java | 205 --------- .../org/apache/druid/timeline/SegmentId.java | 391 ++++++++++++++++++ .../druid/timeline/DataSegmentTest.java | 8 +- .../druid/timeline/DataSegmentUtilsTest.java | 123 ------ .../apache/druid/timeline/SegmentIdTest.java | 179 ++++++++ .../FilteredAggregatorBenchmark.java | 11 +- .../GroupByTypeInterfaceBenchmark.java | 101 ++--- .../druid/benchmark/LoadStatusBenchmark.java | 11 +- .../benchmark/TopNTypeInterfaceBenchmark.java | 87 ++-- .../benchmark/datagen/SegmentGenerator.java | 13 +- .../benchmark/query/GroupByBenchmark.java | 13 +- .../benchmark/query/QueryBenchmarkUtil.java | 10 +- .../benchmark/query/SearchBenchmark.java | 13 +- .../benchmark/query/SelectBenchmark.java | 14 +- .../benchmark/query/TimeseriesBenchmark.java | 19 +- .../druid/benchmark/query/TopNBenchmark.java | 15 +- .../timecompare/TimeCompareBenchmark.java | 13 +- .../storage/azure/AzureDataSegmentKiller.java | 6 +- .../CloudFilesDataSegmentPusher.java | 2 +- .../google/GoogleDataSegmentFinder.java | 2 +- .../google/GoogleDataSegmentKiller.java | 2 +- .../MaterializedViewSupervisor.java | 4 +- .../druid/segment/MapVirtualColumnTest.java | 9 +- .../storage/hdfs/HdfsDataSegmentFinder.java | 5 +- .../storage/hdfs/HdfsDataSegmentKiller.java | 2 +- .../storage/hdfs/HdfsDataSegmentPusher.java | 2 +- .../loading/HdfsDataSegmentFinderTest.java | 12 +- ...ementalPublishingKafkaIndexTaskRunner.java | 22 +- .../kafka/LegacyKafkaIndexTaskRunner.java | 19 +- .../druid/storage/s3/S3DataSegmentFinder.java | 5 +- .../druid/storage/s3/S3DataSegmentKiller.java | 2 +- .../druid/storage/s3/S3DataSegmentMover.java | 2 +- .../druid/storage/s3/S3DataSegmentPusher.java | 2 +- .../storage/s3/S3DataSegmentFinderTest.java | 12 +- .../indexer/HadoopDruidIndexerConfig.java | 3 +- .../druid/indexer/IndexGeneratorJob.java | 2 +- .../indexer/hadoop/DatasourceInputFormat.java | 2 +- .../hadoop/DatasourceRecordReader.java | 2 +- .../updater/HadoopConverterJobTest.java | 4 +- .../ActionBasedSegmentAllocator.java | 4 +- .../ActionBasedUsedSegmentChecker.java | 16 +- .../SegmentAllocateActionGenerator.java | 8 +- .../common/actions/SegmentAllocateAction.java | 26 +- .../common/index/YeOldePlumberSchool.java | 8 +- .../indexing/common/task/AppendTask.java | 2 +- .../indexing/common/task/ArchiveTask.java | 4 +- .../indexing/common/task/CompactionTask.java | 8 +- .../common/task/ConvertSegmentTask.java | 4 +- .../druid/indexing/common/task/IndexTask.java | 20 +- .../druid/indexing/common/task/KillTask.java | 4 +- .../indexing/common/task/MergeTaskBase.java | 47 +-- .../druid/indexing/common/task/MoveTask.java | 4 +- .../indexing/common/task/RestoreTask.java | 4 +- .../parallel/ParallelIndexSupervisorTask.java | 10 +- .../parallel/ParallelIndexTaskClient.java | 6 +- .../SinglePhaseParallelIndexTaskRunner.java | 6 +- .../IngestSegmentFirehoseFactory.java | 2 +- .../indexing/overlord/hrtr/WorkerHolder.java | 3 +- .../ActionBasedUsedSegmentCheckerTest.java | 8 +- .../actions/SegmentAllocateActionTest.java | 116 +++--- .../actions/SegmentListActionsTest.java | 4 +- .../indexing/common/task/IndexTaskTest.java | 4 +- ...stractParallelIndexSupervisorTaskTest.java | 4 +- ...estSegmentFirehoseFactoryTimelineTest.java | 7 +- ...TestIndexerMetadataStorageCoordinator.java | 4 +- .../druid/java/util/common/DateTimes.java | 11 + .../java/util/common/guava/Sequence.java | 4 +- .../druid/java/util/common/DateTimesTest.java | 41 ++ .../druid/query/BySegmentQueryRunner.java | 17 +- .../query/BySegmentResultValueClass.java | 9 + .../query/FinalizeResultsQueryRunner.java | 2 +- .../java/org/apache/druid/query/Result.java | 5 +- .../SegmentMetadataQueryRunnerFactory.java | 2 +- .../metadata/metadata/SegmentAnalysis.java | 8 + .../druid/query/scan/ScanQueryEngine.java | 5 +- .../druid/query/scan/ScanResultValue.java | 8 + .../druid/query/select/SelectQueryEngine.java | 11 +- .../select/SelectQueryQueryToolChest.java | 60 ++- .../druid/query/topn/TopNResultValue.java | 4 +- .../segment/IncrementalIndexSegment.java | 14 +- .../druid/segment/QueryableIndexSegment.java | 13 +- .../segment/ReferenceCountingSegment.java | 7 +- .../org/apache/druid/segment/Segment.java | 6 +- .../MMappedQueryableSegmentizerFactory.java | 2 +- .../collections/bitmap/BitmapBenchmark.java | 6 +- .../apache/druid/query/DoubleStorageTest.java | 9 +- .../druid/query/MultiValuedDimensionTest.java | 15 +- .../druid/query/QueryRunnerTestHelper.java | 20 +- .../druid/query/SchemaEvolutionTest.java | 11 +- .../aggregation/AggregationTestHelper.java | 3 +- .../DataSourceMetadataQueryTest.java | 7 +- ...ByLimitPushDownInsufficientBufferTest.java | 18 +- ...roupByLimitPushDownMultiNodeMergeTest.java | 14 +- .../groupby/GroupByMultiSegmentTest.java | 7 +- .../GroupByQueryRunnerFactoryTest.java | 3 +- .../query/metadata/SegmentAnalyzerTest.java | 5 +- .../metadata/SegmentMetadataQueryTest.java | 15 +- .../SegmentMetadataUnionQueryTest.java | 18 +- .../query/scan/MultiSegmentScanQueryTest.java | 18 +- .../druid/query/scan/ScanQueryRunnerTest.java | 8 +- .../query/search/SearchQueryRunnerTest.java | 5 +- .../search/SearchQueryRunnerWithCaseTest.java | 17 +- .../select/MultiSegmentSelectQueryTest.java | 29 +- .../query/select/SelectQueryRunnerTest.java | 79 ++-- .../TimeBoundaryQueryRunnerTest.java | 25 +- .../TimeseriesQueryRunnerBonusTest.java | 16 +- .../topn/TopNQueryQueryToolChestTest.java | 3 +- .../query/topn/TopNQueryRunnerBenchmark.java | 7 +- .../druid/query/topn/TopNQueryRunnerTest.java | 15 +- .../org/apache/druid/segment/AppendTest.java | 19 +- .../IndexMergerV9WithSpatialIndexTest.java | 8 +- .../segment/ReferenceCountingSegmentTest.java | 5 +- .../druid/segment/SchemalessTestFullTest.java | 14 +- .../segment/SchemalessTestSimpleTest.java | 4 +- .../filter/SpatialFilterBonusTest.java | 4 +- .../segment/filter/SpatialFilterTest.java | 4 +- .../AbstractCuratorServerInventoryView.java | 65 +-- .../client/BatchServerInventoryView.java | 4 +- .../apache/druid/client/BrokerServerView.java | 12 +- .../org/apache/druid/client/CacheUtil.java | 4 +- .../druid/client/CachingClusteredClient.java | 18 +- .../druid/client/CachingQueryRunner.java | 8 +- .../druid/client/CoordinatorServerView.java | 14 +- .../apache/druid/client/DruidDataSource.java | 59 ++- .../org/apache/druid/client/DruidServer.java | 215 +++++----- .../druid/client/HttpServerInventoryView.java | 27 +- .../client/ImmutableDruidDataSource.java | 48 ++- .../druid/client/ImmutableDruidServer.java | 40 +- .../apache/druid/client/InventoryView.java | 4 +- .../inventory/CuratorInventoryManager.java | 24 +- .../CuratorInventoryManagerStrategy.java | 4 + .../druid/discovery/DiscoveryDruidNode.java | 14 + .../SQLMetadataStorageUpdaterJobHandler.java | 4 +- .../IndexerMetadataStorageCoordinator.java | 4 +- .../IndexerSQLMetadataStorageCoordinator.java | 146 +++---- .../metadata/MetadataSegmentManager.java | 27 +- .../metadata/SQLMetadataRuleManager.java | 2 +- .../metadata/SQLMetadataSegmentManager.java | 348 ++++++++-------- .../metadata/SQLMetadataSegmentPublisher.java | 10 +- .../query/lookup/LookupReferencesManager.java | 3 +- .../loading/LocalDataSegmentFinder.java | 9 +- .../loading/LocalDataSegmentKiller.java | 2 +- .../loading/LocalDataSegmentPusher.java | 4 +- .../SegmentLoaderLocalCacheManager.java | 4 +- .../segment/loading/StorageLocation.java | 4 +- .../druid/segment/realtime/FireHydrant.java | 19 +- .../realtime/appenderator/Appenderator.java | 24 +- .../AppenderatorDriverAddResult.java | 8 +- .../AppenderatorDriverMetadata.java | 4 +- .../appenderator/AppenderatorImpl.java | 85 ++-- .../appenderator/AppenderatorPlumber.java | 35 +- .../appenderator/BaseAppenderatorDriver.java | 33 +- .../appenderator/BatchAppenderatorDriver.java | 6 +- .../realtime/appenderator/Committed.java | 6 +- .../appenderator/SegmentAllocator.java | 2 +- ...ifier.java => SegmentIdWithShardSpec.java} | 61 ++- .../appenderator/SegmentWithState.java | 13 +- .../appenderator/SinkQuerySegmentWalker.java | 18 +- .../StreamAppenderatorDriver.java | 23 +- .../appenderator/UsedSegmentChecker.java | 2 +- .../realtime/plumber/FlushingPlumber.java | 4 +- .../realtime/plumber/RealtimePlumber.java | 40 +- .../druid/segment/realtime/plumber/Sink.java | 15 +- .../druid/server/ClientInfoResource.java | 72 ++-- .../apache/druid/server/SegmentManager.java | 4 +- .../BatchDataSegmentAnnouncer.java | 14 +- .../coordination/ChangeRequestHistory.java | 55 +-- .../coordination/ChangeRequestHttpSyncer.java | 112 ++--- .../coordination/ChangeRequestsSnapshot.java | 33 +- .../coordination/DruidServerMetadata.java | 5 +- .../SegmentChangeRequestDrop.java | 2 +- .../SegmentChangeRequestLoad.java | 2 +- .../coordination/SegmentLoadDropHandler.java | 24 +- .../server/coordination/ServerManager.java | 22 +- .../server/coordination/package-info.java | 23 ++ .../coordinator/CostBalancerStrategy.java | 12 +- .../coordinator/CuratorLoadQueuePeon.java | 34 +- .../server/coordinator/DruidCoordinator.java | 61 ++- .../server/coordinator/HttpLoadQueuePeon.java | 12 +- .../coordinator/ReplicationThrottler.java | 57 ++- .../coordinator/ReservoirSegmentSampler.java | 2 +- .../coordinator/SegmentReplicantLookup.java | 39 +- .../server/coordinator/ServerHolder.java | 10 +- .../helper/DruidCoordinatorBalancer.java | 23 +- .../DruidCoordinatorCleanupUnneeded.java | 2 +- .../helper/DruidCoordinatorLogger.java | 10 +- .../helper/DruidCoordinatorRuleRunner.java | 5 +- .../helper/DruidCoordinatorSegmentKiller.java | 2 +- .../helper/DruidCoordinatorSegmentMerger.java | 22 +- .../DruidCoordinatorVersionConverter.java | 2 +- .../rules/BroadcastDistributionRule.java | 2 +- .../server/coordinator/rules/LoadRule.java | 27 +- .../server/http/CoordinatorResource.java | 34 +- ...Resource.java => DataSourcesResource.java} | 208 ++++------ .../druid/server/http/InventoryViewUtils.java | 7 +- .../druid/server/http/MetadataResource.java | 32 +- .../druid/server/http/ServersResource.java | 165 ++++---- .../druid/server/http/TiersResource.java | 108 ++--- .../partition/HashBasedNumberedShardSpec.java | 7 +- .../timeline/partition/LinearShardSpec.java | 29 +- .../client/CachingClusteredClientTest.java | 123 +++--- .../druid/client/CachingQueryRunnerTest.java | 12 +- .../client/CoordinatorServerViewTest.java | 10 +- .../client/HttpServerInventoryViewTest.java | 40 +- .../client/ImmutableDruidDataSourceTest.java | 2 +- .../client/BatchServerInventoryViewTest.java | 21 +- .../apache/druid/curator/CuratorTestBase.java | 20 +- ...exerSQLMetadataStorageCoordinatorTest.java | 68 +-- .../SQLMetadataSegmentManagerTest.java | 16 +- .../loading/CacheTestSegmentLoader.java | 5 +- .../loading/LocalDataSegmentFinderTest.java | 10 +- .../appenderator/AppenderatorPlumberTest.java | 2 +- .../appenderator/AppenderatorTest.java | 20 +- .../BatchAppenderatorDriverTest.java | 14 +- .../realtime/appenderator/CommittedTest.java | 12 +- ...DefaultOfflineAppenderatorFactoryTest.java | 2 +- ...t.java => SegmentIdWithShardSpecTest.java} | 14 +- .../StreamAppenderatorDriverFailTest.java | 14 +- .../StreamAppenderatorDriverTest.java | 20 +- .../appenderator/TestUsedSegmentChecker.java | 6 +- .../druid/server/SegmentManagerTest.java | 10 +- .../SegmentChangeRequestDropTest.java | 3 +- .../SegmentLoadDropHandlerTest.java | 42 +- .../coordination/ServerManagerTest.java | 5 +- .../coordination/ZkCoordinatorTest.java | 6 +- .../coordinator/CostBalancerStrategyTest.java | 47 ++- .../CuratorDruidCoordinatorTest.java | 19 +- ...iskNormalizedCostBalancerStrategyTest.java | 27 +- .../server/coordinator/DruidClusterTest.java | 55 +-- .../DruidCoordinatorBalancerProfiler.java | 20 +- .../DruidCoordinatorBalancerTest.java | 37 +- .../DruidCoordinatorBalancerTester.java | 19 +- .../coordinator/DruidCoordinatorTest.java | 27 +- .../coordinator/HttpLoadQueuePeonTest.java | 53 +-- .../server/coordinator/LoadQueuePeonTest.java | 15 +- .../ReservoirSegmentSamplerTest.java | 40 +- .../server/coordinator/ServerHolderTest.java | 105 +---- .../cost/CachingCostBalancerStrategyTest.java | 4 +- .../coordinator/rules/LoadRuleTest.java | 2 +- ...Test.java => DataSourcesResourceTest.java} | 107 ++--- .../server/http/IntervalsResourceTest.java | 6 +- .../security/SecurityResourceFilterTest.java | 4 +- .../druid/timeline/DataSegmentTest.java | 14 +- .../org/apache/druid/cli/CliCoordinator.java | 6 +- .../apache/druid/cli/CliRealtimeExample.java | 2 +- .../org/apache/druid/cli/DumpSegment.java | 12 +- .../org/apache/druid/cli/InsertSegment.java | 4 +- .../sql/calcite/planner/DruidPlanner.java | 4 +- .../sql/calcite/rel/DruidOuterQueryRel.java | 4 +- .../druid/sql/calcite/rel/DruidQueryRel.java | 2 +- .../druid/sql/calcite/rel/DruidRel.java | 2 +- .../druid/sql/calcite/rel/DruidSemiJoin.java | 6 +- .../druid/sql/calcite/rel/DruidUnionRel.java | 4 +- .../druid/sql/calcite/schema/DruidSchema.java | 30 +- .../SpecificSegmentsQuerySegmentWalker.java | 4 +- 259 files changed, 3226 insertions(+), 3223 deletions(-) delete mode 100644 api/src/main/java/org/apache/druid/timeline/DataSegmentUtils.java create mode 100644 api/src/main/java/org/apache/druid/timeline/SegmentId.java delete mode 100644 api/src/test/java/org/apache/druid/timeline/DataSegmentUtilsTest.java create mode 100644 api/src/test/java/org/apache/druid/timeline/SegmentIdTest.java create mode 100644 java-util/src/test/java/org/apache/druid/java/util/common/DateTimesTest.java rename server/src/main/java/org/apache/druid/segment/realtime/appenderator/{SegmentIdentifier.java => SegmentIdWithShardSpec.java} (68%) create mode 100644 server/src/main/java/org/apache/druid/server/coordination/package-info.java rename server/src/main/java/org/apache/druid/server/http/{DatasourcesResource.java => DataSourcesResource.java} (77%) rename server/src/test/java/org/apache/druid/segment/realtime/appenderator/{SegmentIdentifierTest.java => SegmentIdWithShardSpecTest.java} (83%) rename server/src/test/java/org/apache/druid/server/http/{DatasourcesResourceTest.java => DataSourcesResourceTest.java} (88%) diff --git a/.idea/inspectionProfiles/Druid.xml b/.idea/inspectionProfiles/Druid.xml index 38d6033a81a1..6d6fd3ea4a5b 100644 --- a/.idea/inspectionProfiles/Druid.xml +++ b/.idea/inspectionProfiles/Druid.xml @@ -63,7 +63,6 @@ -