From 0a6eb308f5f3c01099f6cf540dd67eaacc4f85ed Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 5 Jan 2021 11:45:26 +0800 Subject: [PATCH 01/29] ready to test --- .idea/misc.xml | 4 +-- .../druid/common/utils/CommonCallback.java | 26 +++++++++++++++++ ...bstractMultiPhaseParallelIndexingTest.java | 3 +- .../org/apache/druid/segment/IndexIO.java | 23 +++++++++------ .../MMappedQueryableSegmentizerFactory.java | 5 ++-- .../segment/loading/SegmentizerFactory.java | 3 +- .../druid/segment/loading/SegmentLoader.java | 3 +- .../SegmentLoaderLocalCacheManager.java | 5 ++-- .../apache/druid/server/SegmentManager.java | 9 +++--- .../coordination/SegmentLoadDropHandler.java | 5 +++- .../loading/CacheTestSegmentLoader.java | 3 +- .../druid/server/SegmentManagerTest.java | 29 ++++++++++--------- .../SegmentManagerThreadSafetyTest.java | 7 +++-- .../coordination/ServerManagerTest.java | 6 ++-- 14 files changed, 88 insertions(+), 43 deletions(-) create mode 100644 core/src/main/java/org/apache/druid/common/utils/CommonCallback.java diff --git a/.idea/misc.xml b/.idea/misc.xml index bf2061d7392d..fe39ed623c9c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - + \ No newline at end of file diff --git a/core/src/main/java/org/apache/druid/common/utils/CommonCallback.java b/core/src/main/java/org/apache/druid/common/utils/CommonCallback.java new file mode 100644 index 000000000000..0b2ce09164df --- /dev/null +++ b/core/src/main/java/org/apache/druid/common/utils/CommonCallback.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.common.utils; + +public interface CommonCallback +{ + void execute(); + CommonCallback NOOP = () -> {}; +} diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java index b57cedeebbd4..e9f46fa6c1ae 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java @@ -20,6 +20,7 @@ package org.apache.druid.indexing.common.task.batch.parallel; import com.google.common.base.Preconditions; +import org.apache.druid.common.utils.CommonCallback; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.LocalInputSource; @@ -282,7 +283,7 @@ private Segment loadSegment(DataSegment dataSegment, File tempSegmentDir) final SegmentLoader loader = new SegmentLoaderFactory(getIndexIO(), getObjectMapper()) .manufacturate(tempSegmentDir); try { - return loader.getSegment(dataSegment, false); + return loader.getSegment(dataSegment, false, CommonCallback.NOOP); } catch (SegmentLoadingException e) { throw new RuntimeException(e); diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index 655807275737..2532f839de50 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -38,6 +38,7 @@ import org.apache.druid.collections.bitmap.ImmutableBitmap; import org.apache.druid.collections.spatial.ImmutableRTree; import org.apache.druid.common.utils.SerializerUtils; +import org.apache.druid.coordination.CommonCallback; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.IOE; import org.apache.druid.java.util.common.ISE; @@ -182,16 +183,16 @@ public void validateTwoSegments(final IndexableAdapter adapter1, final Indexable public QueryableIndex loadIndex(File inDir) throws IOException { - return loadIndex(inDir, false); + return loadIndex(inDir, false, CommonCallback.NOOP); } - public QueryableIndex loadIndex(File inDir, boolean lazy) throws IOException + public QueryableIndex loadIndex(File inDir, boolean lazy, CommonCallback loadFailed) throws IOException { final int version = SegmentUtils.getVersionFromDir(inDir); final IndexLoader loader = indexLoaders.get(version); if (loader != null) { - return loader.load(inDir, mapper, lazy); + return loader.load(inDir, mapper, lazy, loadFailed); } else { throw new ISE("Unknown index version[%s]", version); } @@ -412,7 +413,7 @@ public MMappedIndex mapDir(File inDir) throws IOException interface IndexLoader { - QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy) throws IOException; + QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, CommonCallback loadFailed) throws IOException; } static class LegacyIndexLoader implements IndexLoader @@ -427,7 +428,7 @@ static class LegacyIndexLoader implements IndexLoader } @Override - public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy) throws IOException + public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, CommonCallback loadFailed) throws IOException { MMappedIndex index = legacyHandler.mapDir(inDir); @@ -522,7 +523,7 @@ static class V9IndexLoader implements IndexLoader } @Override - public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy) throws IOException + public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, CommonCallback loadFailed) throws IOException { log.debug("Mapping v9 index[%s]", inDir); long startTime = System.currentTimeMillis(); @@ -598,7 +599,9 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy) throws try { return deserializeColumn(mapper, colBuffer, smooshedFiles); } - catch (IOException e) { + catch (IOException | RuntimeException e) { + log.warn(e,"Exception when deserialize Column " + columnName); + loadFailed.execute(); throw Throwables.propagate(e); } } @@ -618,7 +621,9 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy) throws try { return deserializeColumn(mapper, timeBuffer, smooshedFiles); } - catch (IOException e) { + catch (IOException | RuntimeException e) { + log.warn(e,"Exception when deserialize Column " + ColumnHolder.TIME_COLUMN_NAME); + loadFailed.execute(); throw Throwables.propagate(e); } } @@ -644,7 +649,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy) throws } private ColumnHolder deserializeColumn(ObjectMapper mapper, ByteBuffer byteBuffer, SmooshedFileMapper smooshedFiles) - throws IOException + throws IOException, RuntimeException { ColumnDescriptor serde = mapper.readValue( SERIALIZER_UTILS.readString(byteBuffer), ColumnDescriptor.class diff --git a/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java b/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java index ea5290e204c0..0789ada12656 100644 --- a/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.google.common.base.Preconditions; +import org.apache.druid.common.utils.CommonCallback; import org.apache.druid.segment.IndexIO; import org.apache.druid.segment.QueryableIndexSegment; import org.apache.druid.segment.Segment; @@ -42,10 +43,10 @@ public MMappedQueryableSegmentizerFactory(@JacksonInject IndexIO indexIO) } @Override - public Segment factorize(DataSegment dataSegment, File parentDir, boolean lazy) throws SegmentLoadingException + public Segment factorize(DataSegment dataSegment, File parentDir, boolean lazy, CommonCallback loadFailed) throws SegmentLoadingException { try { - return new QueryableIndexSegment(indexIO.loadIndex(parentDir, lazy), dataSegment.getId()); + return new QueryableIndexSegment(indexIO.loadIndex(parentDir, lazy, loadFailed), dataSegment.getId()); } catch (IOException e) { throw new SegmentLoadingException(e, "%s", e.getMessage()); diff --git a/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java b/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java index 09bc048448b8..4a8ea5a2481a 100644 --- a/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java @@ -20,6 +20,7 @@ package org.apache.druid.segment.loading; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.apache.druid.common.utils.CommonCallback; import org.apache.druid.segment.Segment; import org.apache.druid.timeline.DataSegment; @@ -31,5 +32,5 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = MMappedQueryableSegmentizerFactory.class) public interface SegmentizerFactory { - Segment factorize(DataSegment segment, File parentDir, boolean lazy) throws SegmentLoadingException; + Segment factorize(DataSegment segment, File parentDir, boolean lazy, CommonCallback loadFailed) throws SegmentLoadingException; } diff --git a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java index f63024a58cc6..715733747eeb 100644 --- a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java +++ b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java @@ -19,6 +19,7 @@ package org.apache.druid.segment.loading; +import org.apache.druid.common.utils.CommonCallback; import org.apache.druid.segment.Segment; import org.apache.druid.timeline.DataSegment; @@ -31,7 +32,7 @@ public interface SegmentLoader { boolean isSegmentLoaded(DataSegment segment); - Segment getSegment(DataSegment segment, boolean lazy) throws SegmentLoadingException; + Segment getSegment(DataSegment segment, boolean lazy, CommonCallback loadFailed) throws SegmentLoadingException; File getSegmentFiles(DataSegment segment) throws SegmentLoadingException; void cleanup(DataSegment segment); } diff --git a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java index de9bff151693..927c5bf66e54 100644 --- a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java +++ b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; +import org.apache.druid.common.utils.CommonCallback; import org.apache.druid.guice.annotations.Json; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.ISE; @@ -172,7 +173,7 @@ private boolean checkSegmentFilesIntactWithStartMarker(File localStorageDir) } @Override - public Segment getSegment(DataSegment segment, boolean lazy) throws SegmentLoadingException + public Segment getSegment(DataSegment segment, boolean lazy, CommonCallback loadFailed) throws SegmentLoadingException { final ReferenceCountingLock lock = createOrGetLock(segment); final File segmentFiles; @@ -198,7 +199,7 @@ public Segment getSegment(DataSegment segment, boolean lazy) throws SegmentLoadi factory = new MMappedQueryableSegmentizerFactory(indexIO); } - return factory.factorize(segment, segmentFiles, lazy); + return factory.factorize(segment, segmentFiles, lazy, loadFailed); } /** diff --git a/server/src/main/java/org/apache/druid/server/SegmentManager.java b/server/src/main/java/org/apache/druid/server/SegmentManager.java index 3b3d891596bb..dd7dc3edd2b6 100644 --- a/server/src/main/java/org/apache/druid/server/SegmentManager.java +++ b/server/src/main/java/org/apache/druid/server/SegmentManager.java @@ -23,6 +23,7 @@ import com.google.common.collect.Ordering; import com.google.inject.Inject; import org.apache.druid.common.guava.SettableSupplier; +import org.apache.druid.common.utils.CommonCallback; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.io.Closer; @@ -219,9 +220,9 @@ private TableDataSource getTableDataSource(DataSourceAnalysis analysis) * * @throws SegmentLoadingException if the segment cannot be loaded */ - public boolean loadSegment(final DataSegment segment, boolean lazy) throws SegmentLoadingException + public boolean loadSegment(final DataSegment segment, boolean lazy, CommonCallback loadFailed) throws SegmentLoadingException { - final Segment adapter = getAdapter(segment, lazy); + final Segment adapter = getAdapter(segment, lazy, loadFailed); final SettableSupplier resultSupplier = new SettableSupplier<>(); @@ -271,11 +272,11 @@ public boolean loadSegment(final DataSegment segment, boolean lazy) throws Segme return resultSupplier.get(); } - private Segment getAdapter(final DataSegment segment, boolean lazy) throws SegmentLoadingException + private Segment getAdapter(final DataSegment segment, boolean lazy, CommonCallback loadFailed) throws SegmentLoadingException { final Segment adapter; try { - adapter = segmentLoader.getSegment(segment, lazy); + adapter = segmentLoader.getSegment(segment, lazy, loadFailed); } catch (SegmentLoadingException e) { segmentLoader.cleanup(segment); diff --git a/server/src/main/java/org/apache/druid/server/coordination/SegmentLoadDropHandler.java b/server/src/main/java/org/apache/druid/server/coordination/SegmentLoadDropHandler.java index 0cb0ec0218b8..e96c4a54efaa 100644 --- a/server/src/main/java/org/apache/druid/server/coordination/SegmentLoadDropHandler.java +++ b/server/src/main/java/org/apache/druid/server/coordination/SegmentLoadDropHandler.java @@ -269,7 +269,10 @@ private void loadSegment(DataSegment segment, DataSegmentChangeCallback callback { final boolean loaded; try { - loaded = segmentManager.loadSegment(segment, lazy); + loaded = segmentManager.loadSegment(segment, + lazy, + () -> this.removeSegment(segment, DataSegmentChangeCallback.NOOP, false) + ); } catch (Exception e) { removeSegment(segment, callback, false); diff --git a/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java b/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java index a42c553bad5a..daaed6034b80 100644 --- a/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java +++ b/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java @@ -19,6 +19,7 @@ package org.apache.druid.segment.loading; +import org.apache.druid.coordination.CommonCallback; import org.apache.druid.java.util.common.MapUtils; import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.Segment; @@ -46,7 +47,7 @@ public boolean isSegmentLoaded(DataSegment segment) } @Override - public Segment getSegment(final DataSegment segment, boolean lazy) + public Segment getSegment(final DataSegment segment, boolean lazy, CommonCallback commonCallback) { return new Segment() { diff --git a/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java b/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java index a91411b8db73..1ac5035c92f2 100644 --- a/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; +import org.apache.druid.coordination.CommonCallback; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.MapUtils; import org.apache.druid.query.TableDataSource; @@ -69,7 +70,7 @@ public boolean isSegmentLoaded(DataSegment segment) } @Override - public Segment getSegment(final DataSegment segment, boolean lazy) + public Segment getSegment(final DataSegment segment, boolean lazy, CommonCallback loadFailed) { return new SegmentForTesting( MapUtils.getString(segment.getLoadSpec(), "version"), @@ -222,7 +223,7 @@ public void testLoadSegment() throws ExecutionException, InterruptedException, S final List> futures = SEGMENTS.stream() .map( segment -> executor.submit( - () -> segmentManager.loadSegment(segment, false) + () -> segmentManager.loadSegment(segment, false, CommonCallback.NOOP) ) ) .collect(Collectors.toList()); @@ -238,7 +239,7 @@ public void testLoadSegment() throws ExecutionException, InterruptedException, S public void testDropSegment() throws SegmentLoadingException, ExecutionException, InterruptedException { for (DataSegment eachSegment : SEGMENTS) { - Assert.assertTrue(segmentManager.loadSegment(eachSegment, false)); + Assert.assertTrue(segmentManager.loadSegment(eachSegment, false, CommonCallback.NOOP)); } final List> futures = ImmutableList.of(SEGMENTS.get(0), SEGMENTS.get(2)).stream() @@ -264,14 +265,14 @@ public void testDropSegment() throws SegmentLoadingException, ExecutionException @Test public void testLoadDropSegment() throws SegmentLoadingException, ExecutionException, InterruptedException { - Assert.assertTrue(segmentManager.loadSegment(SEGMENTS.get(0), false)); - Assert.assertTrue(segmentManager.loadSegment(SEGMENTS.get(2), false)); + Assert.assertTrue(segmentManager.loadSegment(SEGMENTS.get(0), false, CommonCallback.NOOP)); + Assert.assertTrue(segmentManager.loadSegment(SEGMENTS.get(2), false, CommonCallback.NOOP)); final List> loadFutures = ImmutableList.of(SEGMENTS.get(1), SEGMENTS.get(3), SEGMENTS.get(4)) .stream() .map( segment -> executor.submit( - () -> segmentManager.loadSegment(segment, false) + () -> segmentManager.loadSegment(segment, false, CommonCallback.NOOP) ) ) .collect(Collectors.toList()); @@ -302,10 +303,10 @@ public void testLoadDropSegment() throws SegmentLoadingException, ExecutionExcep public void testLoadDuplicatedSegmentsSequentially() throws SegmentLoadingException { for (DataSegment segment : SEGMENTS) { - Assert.assertTrue(segmentManager.loadSegment(segment, false)); + Assert.assertTrue(segmentManager.loadSegment(segment, false, CommonCallback.NOOP)); } // try to load an existing segment - Assert.assertFalse(segmentManager.loadSegment(SEGMENTS.get(0), false)); + Assert.assertFalse(segmentManager.loadSegment(SEGMENTS.get(0), false, CommonCallback.NOOP)); assertResult(SEGMENTS); } @@ -318,7 +319,7 @@ public void testLoadDuplicatedSegmentsInParallel() .stream() .map( segment -> executor.submit( - () -> segmentManager.loadSegment(segment, false) + () -> segmentManager.loadSegment(segment, false, CommonCallback.NOOP) ) ) .collect(Collectors.toList()); @@ -339,7 +340,7 @@ public void testLoadDuplicatedSegmentsInParallel() @Test public void testNonExistingSegmentsSequentially() throws SegmentLoadingException { - Assert.assertTrue(segmentManager.loadSegment(SEGMENTS.get(0), false)); + Assert.assertTrue(segmentManager.loadSegment(SEGMENTS.get(0), false, CommonCallback.NOOP)); // try to drop a non-existing segment of different data source segmentManager.dropSegment(SEGMENTS.get(2)); @@ -352,7 +353,7 @@ public void testNonExistingSegmentsSequentially() throws SegmentLoadingException public void testNonExistingSegmentsInParallel() throws SegmentLoadingException, ExecutionException, InterruptedException { - segmentManager.loadSegment(SEGMENTS.get(0), false); + segmentManager.loadSegment(SEGMENTS.get(0), false, CommonCallback.NOOP); final List> futures = ImmutableList.of(SEGMENTS.get(1), SEGMENTS.get(2)) .stream() .map( @@ -375,7 +376,7 @@ public void testNonExistingSegmentsInParallel() @Test public void testRemoveEmptyTimeline() throws SegmentLoadingException { - segmentManager.loadSegment(SEGMENTS.get(0), false); + segmentManager.loadSegment(SEGMENTS.get(0), false, CommonCallback.NOOP); assertResult(ImmutableList.of(SEGMENTS.get(0))); Assert.assertEquals(1, segmentManager.getDataSources().size()); segmentManager.dropSegment(SEGMENTS.get(0)); @@ -412,7 +413,7 @@ public void testLoadAndDropNonRootGenerationSegment() throws SegmentLoadingExcep 10 ); - segmentManager.loadSegment(segment, false); + segmentManager.loadSegment(segment, false, CommonCallback.NOOP); assertResult(ImmutableList.of(segment)); segmentManager.dropSegment(segment); @@ -442,7 +443,7 @@ private void assertResult(List expectedExistingSegments) throws Seg segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk( - ReferenceCountingSegment.wrapSegment(SEGMENT_LOADER.getSegment(segment, false), segment.getShardSpec()) + ReferenceCountingSegment.wrapSegment(SEGMENT_LOADER.getSegment(segment, false, CommonCallback.NOOP), segment.getShardSpec()) ) ); } diff --git a/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java b/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java index e5593f131988..bd60639024eb 100644 --- a/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java +++ b/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.jsontype.NamedType; import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableMap; +import org.apache.druid.coordination.CommonCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.FileUtils.FileCopyResult; @@ -129,7 +130,7 @@ public void testLoadSameSegment() throws IOException, ExecutionException, Interr final DataSegment segment = createSegment("2019-01-01/2019-01-02"); final List futures = IntStream .range(0, 16) - .mapToObj(i -> exec.submit(() -> segmentManager.loadSegment(segment, false))) + .mapToObj(i -> exec.submit(() -> segmentManager.loadSegment(segment, false, CommonCallback.NOOP))) .collect(Collectors.toList()); for (Future future : futures) { future.get(); @@ -154,7 +155,7 @@ public void testLoadMultipleSegments() throws IOException, ExecutionException, I .mapToObj(i -> exec.submit(() -> { for (DataSegment segment : segments) { try { - segmentManager.loadSegment(segment, false); + segmentManager.loadSegment(segment, false, CommonCallback.NOOP); } catch (SegmentLoadingException e) { throw new RuntimeException(e); @@ -222,7 +223,7 @@ public long size() private static class TestSegmentizerFactory implements SegmentizerFactory { @Override - public Segment factorize(DataSegment segment, File parentDir, boolean lazy) + public Segment factorize(DataSegment segment, File parentDir, boolean lazy, CommonCallback commonCallback) { return new Segment() { diff --git a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java index bf74ba66d7e8..5ce850f2abff 100644 --- a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java @@ -28,6 +28,7 @@ import org.apache.druid.client.cache.CachePopulatorStats; import org.apache.druid.client.cache.ForegroundCachePopulator; import org.apache.druid.client.cache.LocalCacheProvider; +import org.apache.druid.coordination.CommonCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.Intervals; @@ -143,7 +144,7 @@ public boolean isSegmentLoaded(DataSegment segment) } @Override - public Segment getSegment(final DataSegment segment, boolean lazy) + public Segment getSegment(final DataSegment segment, boolean lazy, CommonCallback commonCallback) { return new SegmentForTesting( MapUtils.getString(segment.getLoadSpec(), "version"), @@ -666,7 +667,8 @@ public void loadQueryable(String dataSource, String version, Interval interval) IndexIO.CURRENT_VERSION_ID, 123L ), - false + false, + CommonCallback.NOOP ); } catch (SegmentLoadingException e) { From 4c3e69770ccfed29ac0e5d77fbd3fa6495493da5 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 5 Jan 2021 11:47:57 +0800 Subject: [PATCH 02/29] tested on dev cluster --- .idea/misc.xml | 4 ++-- .../druid/{common/utils => coordination}/CommonCallback.java | 2 +- .../parallel/AbstractMultiPhaseParallelIndexingTest.java | 2 +- .../segment/loading/MMappedQueryableSegmentizerFactory.java | 2 +- .../org/apache/druid/segment/loading/SegmentizerFactory.java | 2 +- .../java/org/apache/druid/segment/loading/SegmentLoader.java | 2 +- .../druid/segment/loading/SegmentLoaderLocalCacheManager.java | 2 +- .../src/main/java/org/apache/druid/server/SegmentManager.java | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) rename core/src/main/java/org/apache/druid/{common/utils => coordination}/CommonCallback.java (95%) diff --git a/.idea/misc.xml b/.idea/misc.xml index fe39ed623c9c..bf2061d7392d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - \ No newline at end of file + diff --git a/core/src/main/java/org/apache/druid/common/utils/CommonCallback.java b/core/src/main/java/org/apache/druid/coordination/CommonCallback.java similarity index 95% rename from core/src/main/java/org/apache/druid/common/utils/CommonCallback.java rename to core/src/main/java/org/apache/druid/coordination/CommonCallback.java index 0b2ce09164df..6a256aeaafe8 100644 --- a/core/src/main/java/org/apache/druid/common/utils/CommonCallback.java +++ b/core/src/main/java/org/apache/druid/coordination/CommonCallback.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.common.utils; +package org.apache.druid.coordination; public interface CommonCallback { diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java index e9f46fa6c1ae..32c6ce3a33b2 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java @@ -20,7 +20,7 @@ package org.apache.druid.indexing.common.task.batch.parallel; import com.google.common.base.Preconditions; -import org.apache.druid.common.utils.CommonCallback; +import org.apache.druid.coordination.CommonCallback; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.LocalInputSource; diff --git a/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java b/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java index 0789ada12656..d0bccdb7f962 100644 --- a/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.google.common.base.Preconditions; -import org.apache.druid.common.utils.CommonCallback; +import org.apache.druid.coordination.CommonCallback; import org.apache.druid.segment.IndexIO; import org.apache.druid.segment.QueryableIndexSegment; import org.apache.druid.segment.Segment; diff --git a/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java b/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java index 4a8ea5a2481a..c0e25021b6f9 100644 --- a/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java @@ -20,7 +20,7 @@ package org.apache.druid.segment.loading; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import org.apache.druid.common.utils.CommonCallback; +import org.apache.druid.coordination.CommonCallback; import org.apache.druid.segment.Segment; import org.apache.druid.timeline.DataSegment; diff --git a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java index 715733747eeb..08c752babc37 100644 --- a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java +++ b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java @@ -19,7 +19,7 @@ package org.apache.druid.segment.loading; -import org.apache.druid.common.utils.CommonCallback; +import org.apache.druid.coordination.CommonCallback; import org.apache.druid.segment.Segment; import org.apache.druid.timeline.DataSegment; diff --git a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java index 927c5bf66e54..809869463fec 100644 --- a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java +++ b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; -import org.apache.druid.common.utils.CommonCallback; +import org.apache.druid.coordination.CommonCallback; import org.apache.druid.guice.annotations.Json; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.ISE; diff --git a/server/src/main/java/org/apache/druid/server/SegmentManager.java b/server/src/main/java/org/apache/druid/server/SegmentManager.java index dd7dc3edd2b6..e11ee453a161 100644 --- a/server/src/main/java/org/apache/druid/server/SegmentManager.java +++ b/server/src/main/java/org/apache/druid/server/SegmentManager.java @@ -23,7 +23,7 @@ import com.google.common.collect.Ordering; import com.google.inject.Inject; import org.apache.druid.common.guava.SettableSupplier; -import org.apache.druid.common.utils.CommonCallback; +import org.apache.druid.coordination.CommonCallback; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.io.Closer; From 7775658a57209265762bc41e30b90d7774a3e362 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 5 Jan 2021 12:38:27 +0800 Subject: [PATCH 03/29] tested --- .../main/java/org/apache/druid/segment/IndexIO.java | 4 ++-- ...astJoinableMMappedQueryableSegmentizerFactory.java | 5 +++-- .../druid/segment/CustomSegmentizerFactoryTest.java | 5 +++-- .../join/table/BroadcastSegmentIndexedTableTest.java | 3 ++- ...oinableMMappedQueryableSegmentizerFactoryTest.java | 3 ++- .../server/coordination/SegmentLoadDropHandler.java | 2 +- .../SegmentManagerBroadcastJoinIndexedTableTest.java | 11 ++++++----- 7 files changed, 19 insertions(+), 14 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index 2532f839de50..a2da9fed1ad7 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -600,7 +600,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Common return deserializeColumn(mapper, colBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e,"Exception when deserialize Column " + columnName); + log.warn(e, "Exception when deserialize Column " + columnName); loadFailed.execute(); throw Throwables.propagate(e); } @@ -622,7 +622,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Common return deserializeColumn(mapper, timeBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e,"Exception when deserialize Column " + ColumnHolder.TIME_COLUMN_NAME); + log.warn(e, "Exception when deserialize Column " + ColumnHolder.TIME_COLUMN_NAME); loadFailed.execute(); throw Throwables.propagate(e); } diff --git a/processing/src/main/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactory.java b/processing/src/main/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactory.java index c675ea5ff7bb..3c4667954fd0 100644 --- a/processing/src/main/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactory.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.coordination.CommonCallback; import org.apache.druid.segment.IndexIO; import org.apache.druid.segment.QueryableIndexSegment; import org.apache.druid.segment.Segment; @@ -57,10 +58,10 @@ public Set getKeyColumns() } @Override - public Segment factorize(DataSegment dataSegment, File parentDir, boolean lazy) throws SegmentLoadingException + public Segment factorize(DataSegment dataSegment, File parentDir, boolean lazy, CommonCallback loadFailed) throws SegmentLoadingException { try { - return new QueryableIndexSegment(indexIO.loadIndex(parentDir, lazy), dataSegment.getId()) { + return new QueryableIndexSegment(indexIO.loadIndex(parentDir, lazy, loadFailed), dataSegment.getId()) { @Nullable @Override public T as(Class clazz) diff --git a/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java b/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java index 8b4a8d3f5313..fe7bbf7e0d8b 100644 --- a/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.InjectableValues; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.jsontype.NamedType; +import org.apache.druid.coordination.CommonCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.jackson.SegmentizerModule; import org.apache.druid.java.util.common.Intervals; @@ -126,10 +127,10 @@ public void testCustomSegmentizerPersist() throws IOException private static class CustomSegmentizerFactory implements SegmentizerFactory { @Override - public Segment factorize(DataSegment segment, File parentDir, boolean lazy) throws SegmentLoadingException + public Segment factorize(DataSegment segment, File parentDir, boolean lazy, CommonCallback loadFailed) throws SegmentLoadingException { try { - return new QueryableIndexSegment(INDEX_IO.loadIndex(parentDir, lazy), segment.getId()); + return new QueryableIndexSegment(INDEX_IO.loadIndex(parentDir, lazy, loadFailed), segment.getId()); } catch (IOException e) { throw new SegmentLoadingException(e, "%s", e.getMessage()); diff --git a/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java b/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java index 524f2ac1f185..70a75e815f81 100644 --- a/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java +++ b/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java @@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableSet; import it.unimi.dsi.fastutil.ints.IntList; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.coordination.CommonCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.jackson.SegmentizerModule; import org.apache.druid.java.util.common.DateTimes; @@ -137,7 +138,7 @@ public void setup() throws IOException, SegmentLoadingException null, segment.getTotalSpace() ); - backingSegment = (QueryableIndexSegment) factory.factorize(dataSegment, segment, false); + backingSegment = (QueryableIndexSegment) factory.factorize(dataSegment, segment, false, CommonCallback.NOOP); columnNames = ImmutableList.builder().add(ColumnHolder.TIME_COLUMN_NAME) .addAll(backingSegment.asQueryableIndex().getColumnNames()).build(); diff --git a/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java b/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java index ab4c45339cdc..55925ba921ab 100644 --- a/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import org.apache.druid.coordination.CommonCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.jackson.SegmentizerModule; import org.apache.druid.java.util.common.DateTimes; @@ -119,7 +120,7 @@ public void testSegmentizer() throws IOException, SegmentLoadingException null, persistedSegmentRoot.getTotalSpace() ); - final Segment loaded = factory.factorize(dataSegment, persistedSegmentRoot, false); + final Segment loaded = factory.factorize(dataSegment, persistedSegmentRoot, false, CommonCallback.NOOP); final BroadcastSegmentIndexedTable table = (BroadcastSegmentIndexedTable) loaded.as(IndexedTable.class); Assert.assertNotNull(table); diff --git a/server/src/main/java/org/apache/druid/server/coordination/SegmentLoadDropHandler.java b/server/src/main/java/org/apache/druid/server/coordination/SegmentLoadDropHandler.java index e96c4a54efaa..dcc03097b558 100644 --- a/server/src/main/java/org/apache/druid/server/coordination/SegmentLoadDropHandler.java +++ b/server/src/main/java/org/apache/druid/server/coordination/SegmentLoadDropHandler.java @@ -271,7 +271,7 @@ private void loadSegment(DataSegment segment, DataSegmentChangeCallback callback try { loaded = segmentManager.loadSegment(segment, lazy, - () -> this.removeSegment(segment, DataSegmentChangeCallback.NOOP, false) + () -> this.removeSegment(segment, DataSegmentChangeCallback.NOOP, false) ); } catch (Exception e) { diff --git a/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java b/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java index 3bb87686b759..e49fa2c26bfa 100644 --- a/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java +++ b/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import org.apache.druid.coordination.CommonCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.jackson.SegmentizerModule; import org.apache.druid.java.util.common.DateTimes; @@ -159,7 +160,7 @@ public void testLoadIndexedTable() throws IOException, SegmentLoadingException IncrementalIndex data = TestIndex.makeRealtimeIndex("druid.sample.numeric.tsv"); final String interval = "2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z"; DataSegment segment = createSegment(data, interval, version); - Assert.assertTrue(segmentManager.loadSegment(segment, false)); + Assert.assertTrue(segmentManager.loadSegment(segment, false, CommonCallback.NOOP)); Assert.assertTrue(joinableFactory.isDirectlyJoinable(dataSource)); Optional maybeJoinable = makeJoinable(dataSource); @@ -208,8 +209,8 @@ public void testLoadMultipleIndexedTableOverwrite() throws IOException, SegmentL IncrementalIndex data2 = TestIndex.makeRealtimeIndex("druid.sample.numeric.tsv.bottom"); DataSegment segment1 = createSegment(data, interval, version); DataSegment segment2 = createSegment(data2, interval2, version2); - Assert.assertTrue(segmentManager.loadSegment(segment1, false)); - Assert.assertTrue(segmentManager.loadSegment(segment2, false)); + Assert.assertTrue(segmentManager.loadSegment(segment1, false, CommonCallback.NOOP)); + Assert.assertTrue(segmentManager.loadSegment(segment2, false, CommonCallback.NOOP)); Assert.assertTrue(joinableFactory.isDirectlyJoinable(dataSource)); Optional maybeJoinable = makeJoinable(dataSource); @@ -271,7 +272,7 @@ public void testLoadMultipleIndexedTable() throws IOException, SegmentLoadingExc final String interval2 = "2011-01-12T00:00:00.000Z/2011-03-28T00:00:00.000Z"; IncrementalIndex data = TestIndex.makeRealtimeIndex("druid.sample.numeric.tsv.bottom"); IncrementalIndex data2 = TestIndex.makeRealtimeIndex("druid.sample.numeric.tsv.top"); - Assert.assertTrue(segmentManager.loadSegment(createSegment(data, interval, version), false)); + Assert.assertTrue(segmentManager.loadSegment(createSegment(data, interval, version), false, CommonCallback.NOOP)); Assert.assertTrue(joinableFactory.isDirectlyJoinable(dataSource)); Optional maybeJoinable = makeJoinable(dataSource); @@ -293,7 +294,7 @@ public void testLoadMultipleIndexedTable() throws IOException, SegmentLoadingExc ); // add another segment with smaller interval, only partially overshadows so there will be 2 segments in timeline - Assert.assertTrue(segmentManager.loadSegment(createSegment(data2, interval2, version2), false)); + Assert.assertTrue(segmentManager.loadSegment(createSegment(data2, interval2, version2), false, CommonCallback.NOOP)); expectedException.expect(ISE.class); From 6b9595e5ebe3b67ae0795402a1c799a69d7a821f Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 7 Jan 2021 10:26:04 +0800 Subject: [PATCH 04/29] code review --- .idea/misc.xml | 4 +-- ...bstractMultiPhaseParallelIndexingTest.java | 4 +-- .../SegmentLazyLoadFailCallback.java | 4 +-- .../org/apache/druid/segment/IndexIO.java | 15 +++++----- ...bleMMappedQueryableSegmentizerFactory.java | 4 +-- .../MMappedQueryableSegmentizerFactory.java | 4 +-- .../segment/loading/SegmentizerFactory.java | 4 +-- .../segment/CustomSegmentizerFactoryTest.java | 4 +-- .../BroadcastSegmentIndexedTableTest.java | 4 +-- ...MappedQueryableSegmentizerFactoryTest.java | 4 +-- .../druid/segment/loading/SegmentLoader.java | 4 +-- .../SegmentLoaderLocalCacheManager.java | 4 +-- .../apache/druid/server/SegmentManager.java | 7 +++-- .../loading/CacheTestSegmentLoader.java | 4 +-- ...tManagerBroadcastJoinIndexedTableTest.java | 12 ++++---- .../druid/server/SegmentManagerTest.java | 30 +++++++++---------- .../SegmentManagerThreadSafetyTest.java | 8 ++--- .../coordination/ServerManagerTest.java | 6 ++-- 18 files changed, 64 insertions(+), 62 deletions(-) rename core/src/main/java/org/apache/druid/coordination/CommonCallback.java => processing/src/main/java/org/apache/druid/coordination/SegmentLazyLoadFailCallback.java (90%) diff --git a/.idea/misc.xml b/.idea/misc.xml index bf2061d7392d..fe39ed623c9c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - + \ No newline at end of file diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java index 32c6ce3a33b2..9e035d536a7d 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java @@ -20,7 +20,7 @@ package org.apache.druid.indexing.common.task.batch.parallel; import com.google.common.base.Preconditions; -import org.apache.druid.coordination.CommonCallback; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.LocalInputSource; @@ -283,7 +283,7 @@ private Segment loadSegment(DataSegment dataSegment, File tempSegmentDir) final SegmentLoader loader = new SegmentLoaderFactory(getIndexIO(), getObjectMapper()) .manufacturate(tempSegmentDir); try { - return loader.getSegment(dataSegment, false, CommonCallback.NOOP); + return loader.getSegment(dataSegment, false, SegmentLazyLoadFailCallback.NOOP); } catch (SegmentLoadingException e) { throw new RuntimeException(e); diff --git a/core/src/main/java/org/apache/druid/coordination/CommonCallback.java b/processing/src/main/java/org/apache/druid/coordination/SegmentLazyLoadFailCallback.java similarity index 90% rename from core/src/main/java/org/apache/druid/coordination/CommonCallback.java rename to processing/src/main/java/org/apache/druid/coordination/SegmentLazyLoadFailCallback.java index 6a256aeaafe8..a92e9e849b57 100644 --- a/core/src/main/java/org/apache/druid/coordination/CommonCallback.java +++ b/processing/src/main/java/org/apache/druid/coordination/SegmentLazyLoadFailCallback.java @@ -19,8 +19,8 @@ package org.apache.druid.coordination; -public interface CommonCallback +public interface SegmentLazyLoadFailCallback { void execute(); - CommonCallback NOOP = () -> {}; + SegmentLazyLoadFailCallback NOOP = () -> {}; } diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index a2da9fed1ad7..abe2514cd362 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -38,7 +38,7 @@ import org.apache.druid.collections.bitmap.ImmutableBitmap; import org.apache.druid.collections.spatial.ImmutableRTree; import org.apache.druid.common.utils.SerializerUtils; -import org.apache.druid.coordination.CommonCallback; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.IOE; import org.apache.druid.java.util.common.ISE; @@ -183,9 +183,10 @@ public void validateTwoSegments(final IndexableAdapter adapter1, final Indexable public QueryableIndex loadIndex(File inDir) throws IOException { - return loadIndex(inDir, false, CommonCallback.NOOP); + return loadIndex(inDir, false, SegmentLazyLoadFailCallback.NOOP); } - public QueryableIndex loadIndex(File inDir, boolean lazy, CommonCallback loadFailed) throws IOException + + public QueryableIndex loadIndex(File inDir, boolean lazy, SegmentLazyLoadFailCallback loadFailed) throws IOException { final int version = SegmentUtils.getVersionFromDir(inDir); @@ -413,7 +414,7 @@ public MMappedIndex mapDir(File inDir) throws IOException interface IndexLoader { - QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, CommonCallback loadFailed) throws IOException; + QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, SegmentLazyLoadFailCallback loadFailed) throws IOException; } static class LegacyIndexLoader implements IndexLoader @@ -428,7 +429,7 @@ static class LegacyIndexLoader implements IndexLoader } @Override - public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, CommonCallback loadFailed) throws IOException + public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, SegmentLazyLoadFailCallback loadFailed) throws IOException { MMappedIndex index = legacyHandler.mapDir(inDir); @@ -523,7 +524,7 @@ static class V9IndexLoader implements IndexLoader } @Override - public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, CommonCallback loadFailed) throws IOException + public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, SegmentLazyLoadFailCallback loadFailed) throws IOException { log.debug("Mapping v9 index[%s]", inDir); long startTime = System.currentTimeMillis(); @@ -649,7 +650,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Common } private ColumnHolder deserializeColumn(ObjectMapper mapper, ByteBuffer byteBuffer, SmooshedFileMapper smooshedFiles) - throws IOException, RuntimeException + throws IOException { ColumnDescriptor serde = mapper.readValue( SERIALIZER_UTILS.readString(byteBuffer), ColumnDescriptor.class diff --git a/processing/src/main/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactory.java b/processing/src/main/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactory.java index 3c4667954fd0..7dd1e5c0094b 100644 --- a/processing/src/main/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactory.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.druid.coordination.CommonCallback; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.IndexIO; import org.apache.druid.segment.QueryableIndexSegment; import org.apache.druid.segment.Segment; @@ -58,7 +58,7 @@ public Set getKeyColumns() } @Override - public Segment factorize(DataSegment dataSegment, File parentDir, boolean lazy, CommonCallback loadFailed) throws SegmentLoadingException + public Segment factorize(DataSegment dataSegment, File parentDir, boolean lazy, SegmentLazyLoadFailCallback loadFailed) throws SegmentLoadingException { try { return new QueryableIndexSegment(indexIO.loadIndex(parentDir, lazy, loadFailed), dataSegment.getId()) { diff --git a/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java b/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java index d0bccdb7f962..1ef626a9144b 100644 --- a/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.google.common.base.Preconditions; -import org.apache.druid.coordination.CommonCallback; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.IndexIO; import org.apache.druid.segment.QueryableIndexSegment; import org.apache.druid.segment.Segment; @@ -43,7 +43,7 @@ public MMappedQueryableSegmentizerFactory(@JacksonInject IndexIO indexIO) } @Override - public Segment factorize(DataSegment dataSegment, File parentDir, boolean lazy, CommonCallback loadFailed) throws SegmentLoadingException + public Segment factorize(DataSegment dataSegment, File parentDir, boolean lazy, SegmentLazyLoadFailCallback loadFailed) throws SegmentLoadingException { try { return new QueryableIndexSegment(indexIO.loadIndex(parentDir, lazy, loadFailed), dataSegment.getId()); diff --git a/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java b/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java index c0e25021b6f9..0e29f2eeb0f1 100644 --- a/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java @@ -20,7 +20,7 @@ package org.apache.druid.segment.loading; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import org.apache.druid.coordination.CommonCallback; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.Segment; import org.apache.druid.timeline.DataSegment; @@ -32,5 +32,5 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = MMappedQueryableSegmentizerFactory.class) public interface SegmentizerFactory { - Segment factorize(DataSegment segment, File parentDir, boolean lazy, CommonCallback loadFailed) throws SegmentLoadingException; + Segment factorize(DataSegment segment, File parentDir, boolean lazy, SegmentLazyLoadFailCallback loadFailed) throws SegmentLoadingException; } diff --git a/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java b/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java index fe7bbf7e0d8b..4f230ed5cb91 100644 --- a/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.databind.InjectableValues; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.jsontype.NamedType; -import org.apache.druid.coordination.CommonCallback; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.jackson.SegmentizerModule; import org.apache.druid.java.util.common.Intervals; @@ -127,7 +127,7 @@ public void testCustomSegmentizerPersist() throws IOException private static class CustomSegmentizerFactory implements SegmentizerFactory { @Override - public Segment factorize(DataSegment segment, File parentDir, boolean lazy, CommonCallback loadFailed) throws SegmentLoadingException + public Segment factorize(DataSegment segment, File parentDir, boolean lazy, SegmentLazyLoadFailCallback loadFailed) throws SegmentLoadingException { try { return new QueryableIndexSegment(INDEX_IO.loadIndex(parentDir, lazy, loadFailed), segment.getId()); diff --git a/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java b/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java index 70a75e815f81..b1cdc7e4b5b8 100644 --- a/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java +++ b/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java @@ -26,7 +26,7 @@ import com.google.common.collect.ImmutableSet; import it.unimi.dsi.fastutil.ints.IntList; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.coordination.CommonCallback; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.jackson.SegmentizerModule; import org.apache.druid.java.util.common.DateTimes; @@ -138,7 +138,7 @@ public void setup() throws IOException, SegmentLoadingException null, segment.getTotalSpace() ); - backingSegment = (QueryableIndexSegment) factory.factorize(dataSegment, segment, false, CommonCallback.NOOP); + backingSegment = (QueryableIndexSegment) factory.factorize(dataSegment, segment, false, SegmentLazyLoadFailCallback.NOOP); columnNames = ImmutableList.builder().add(ColumnHolder.TIME_COLUMN_NAME) .addAll(backingSegment.asQueryableIndex().getColumnNames()).build(); diff --git a/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java b/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java index 55925ba921ab..f9ad4ec66a32 100644 --- a/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import org.apache.druid.coordination.CommonCallback; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.jackson.SegmentizerModule; import org.apache.druid.java.util.common.DateTimes; @@ -120,7 +120,7 @@ public void testSegmentizer() throws IOException, SegmentLoadingException null, persistedSegmentRoot.getTotalSpace() ); - final Segment loaded = factory.factorize(dataSegment, persistedSegmentRoot, false, CommonCallback.NOOP); + final Segment loaded = factory.factorize(dataSegment, persistedSegmentRoot, false, SegmentLazyLoadFailCallback.NOOP); final BroadcastSegmentIndexedTable table = (BroadcastSegmentIndexedTable) loaded.as(IndexedTable.class); Assert.assertNotNull(table); diff --git a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java index 08c752babc37..3cd8a85ca81c 100644 --- a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java +++ b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java @@ -19,7 +19,7 @@ package org.apache.druid.segment.loading; -import org.apache.druid.coordination.CommonCallback; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.Segment; import org.apache.druid.timeline.DataSegment; @@ -32,7 +32,7 @@ public interface SegmentLoader { boolean isSegmentLoaded(DataSegment segment); - Segment getSegment(DataSegment segment, boolean lazy, CommonCallback loadFailed) throws SegmentLoadingException; + Segment getSegment(DataSegment segment, boolean lazy, SegmentLazyLoadFailCallback loadFailed) throws SegmentLoadingException; File getSegmentFiles(DataSegment segment) throws SegmentLoadingException; void cleanup(DataSegment segment); } diff --git a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java index 809869463fec..3bde319bea5b 100644 --- a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java +++ b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; -import org.apache.druid.coordination.CommonCallback; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.guice.annotations.Json; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.ISE; @@ -173,7 +173,7 @@ private boolean checkSegmentFilesIntactWithStartMarker(File localStorageDir) } @Override - public Segment getSegment(DataSegment segment, boolean lazy, CommonCallback loadFailed) throws SegmentLoadingException + public Segment getSegment(DataSegment segment, boolean lazy, SegmentLazyLoadFailCallback loadFailed) throws SegmentLoadingException { final ReferenceCountingLock lock = createOrGetLock(segment); final File segmentFiles; diff --git a/server/src/main/java/org/apache/druid/server/SegmentManager.java b/server/src/main/java/org/apache/druid/server/SegmentManager.java index e11ee453a161..3ea137ec3cb8 100644 --- a/server/src/main/java/org/apache/druid/server/SegmentManager.java +++ b/server/src/main/java/org/apache/druid/server/SegmentManager.java @@ -23,7 +23,7 @@ import com.google.common.collect.Ordering; import com.google.inject.Inject; import org.apache.druid.common.guava.SettableSupplier; -import org.apache.druid.coordination.CommonCallback; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.io.Closer; @@ -215,12 +215,13 @@ private TableDataSource getTableDataSource(DataSourceAnalysis analysis) * * @param segment segment to load * @param lazy whether to lazy load columns metadata + * @param loadFailed callBack to execute when segment lazy load failed * * @return true if the segment was newly loaded, false if it was already loaded * * @throws SegmentLoadingException if the segment cannot be loaded */ - public boolean loadSegment(final DataSegment segment, boolean lazy, CommonCallback loadFailed) throws SegmentLoadingException + public boolean loadSegment(final DataSegment segment, boolean lazy, SegmentLazyLoadFailCallback loadFailed) throws SegmentLoadingException { final Segment adapter = getAdapter(segment, lazy, loadFailed); @@ -272,7 +273,7 @@ public boolean loadSegment(final DataSegment segment, boolean lazy, CommonCallba return resultSupplier.get(); } - private Segment getAdapter(final DataSegment segment, boolean lazy, CommonCallback loadFailed) throws SegmentLoadingException + private Segment getAdapter(final DataSegment segment, boolean lazy, SegmentLazyLoadFailCallback loadFailed) throws SegmentLoadingException { final Segment adapter; try { diff --git a/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java b/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java index daaed6034b80..3653b82eb0ca 100644 --- a/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java +++ b/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java @@ -19,7 +19,7 @@ package org.apache.druid.segment.loading; -import org.apache.druid.coordination.CommonCallback; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.java.util.common.MapUtils; import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.Segment; @@ -47,7 +47,7 @@ public boolean isSegmentLoaded(DataSegment segment) } @Override - public Segment getSegment(final DataSegment segment, boolean lazy, CommonCallback commonCallback) + public Segment getSegment(final DataSegment segment, boolean lazy, SegmentLazyLoadFailCallback SegmentLazyLoadFailCallback) { return new Segment() { diff --git a/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java b/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java index e49fa2c26bfa..838b970e4192 100644 --- a/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java +++ b/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java @@ -25,7 +25,7 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import org.apache.druid.coordination.CommonCallback; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.jackson.SegmentizerModule; import org.apache.druid.java.util.common.DateTimes; @@ -160,7 +160,7 @@ public void testLoadIndexedTable() throws IOException, SegmentLoadingException IncrementalIndex data = TestIndex.makeRealtimeIndex("druid.sample.numeric.tsv"); final String interval = "2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z"; DataSegment segment = createSegment(data, interval, version); - Assert.assertTrue(segmentManager.loadSegment(segment, false, CommonCallback.NOOP)); + Assert.assertTrue(segmentManager.loadSegment(segment, false, SegmentLazyLoadFailCallback.NOOP)); Assert.assertTrue(joinableFactory.isDirectlyJoinable(dataSource)); Optional maybeJoinable = makeJoinable(dataSource); @@ -209,8 +209,8 @@ public void testLoadMultipleIndexedTableOverwrite() throws IOException, SegmentL IncrementalIndex data2 = TestIndex.makeRealtimeIndex("druid.sample.numeric.tsv.bottom"); DataSegment segment1 = createSegment(data, interval, version); DataSegment segment2 = createSegment(data2, interval2, version2); - Assert.assertTrue(segmentManager.loadSegment(segment1, false, CommonCallback.NOOP)); - Assert.assertTrue(segmentManager.loadSegment(segment2, false, CommonCallback.NOOP)); + Assert.assertTrue(segmentManager.loadSegment(segment1, false, SegmentLazyLoadFailCallback.NOOP)); + Assert.assertTrue(segmentManager.loadSegment(segment2, false, SegmentLazyLoadFailCallback.NOOP)); Assert.assertTrue(joinableFactory.isDirectlyJoinable(dataSource)); Optional maybeJoinable = makeJoinable(dataSource); @@ -272,7 +272,7 @@ public void testLoadMultipleIndexedTable() throws IOException, SegmentLoadingExc final String interval2 = "2011-01-12T00:00:00.000Z/2011-03-28T00:00:00.000Z"; IncrementalIndex data = TestIndex.makeRealtimeIndex("druid.sample.numeric.tsv.bottom"); IncrementalIndex data2 = TestIndex.makeRealtimeIndex("druid.sample.numeric.tsv.top"); - Assert.assertTrue(segmentManager.loadSegment(createSegment(data, interval, version), false, CommonCallback.NOOP)); + Assert.assertTrue(segmentManager.loadSegment(createSegment(data, interval, version), false, SegmentLazyLoadFailCallback.NOOP)); Assert.assertTrue(joinableFactory.isDirectlyJoinable(dataSource)); Optional maybeJoinable = makeJoinable(dataSource); @@ -294,7 +294,7 @@ public void testLoadMultipleIndexedTable() throws IOException, SegmentLoadingExc ); // add another segment with smaller interval, only partially overshadows so there will be 2 segments in timeline - Assert.assertTrue(segmentManager.loadSegment(createSegment(data2, interval2, version2), false, CommonCallback.NOOP)); + Assert.assertTrue(segmentManager.loadSegment(createSegment(data2, interval2, version2), false, SegmentLazyLoadFailCallback.NOOP)); expectedException.expect(ISE.class); diff --git a/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java b/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java index 1ac5035c92f2..7ba5f6fa86b9 100644 --- a/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; -import org.apache.druid.coordination.CommonCallback; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.MapUtils; import org.apache.druid.query.TableDataSource; @@ -70,7 +70,7 @@ public boolean isSegmentLoaded(DataSegment segment) } @Override - public Segment getSegment(final DataSegment segment, boolean lazy, CommonCallback loadFailed) + public Segment getSegment(final DataSegment segment, boolean lazy, SegmentLazyLoadFailCallback loadFailed) { return new SegmentForTesting( MapUtils.getString(segment.getLoadSpec(), "version"), @@ -223,7 +223,7 @@ public void testLoadSegment() throws ExecutionException, InterruptedException, S final List> futures = SEGMENTS.stream() .map( segment -> executor.submit( - () -> segmentManager.loadSegment(segment, false, CommonCallback.NOOP) + () -> segmentManager.loadSegment(segment, false, SegmentLazyLoadFailCallback.NOOP) ) ) .collect(Collectors.toList()); @@ -239,7 +239,7 @@ public void testLoadSegment() throws ExecutionException, InterruptedException, S public void testDropSegment() throws SegmentLoadingException, ExecutionException, InterruptedException { for (DataSegment eachSegment : SEGMENTS) { - Assert.assertTrue(segmentManager.loadSegment(eachSegment, false, CommonCallback.NOOP)); + Assert.assertTrue(segmentManager.loadSegment(eachSegment, false, SegmentLazyLoadFailCallback.NOOP)); } final List> futures = ImmutableList.of(SEGMENTS.get(0), SEGMENTS.get(2)).stream() @@ -265,14 +265,14 @@ public void testDropSegment() throws SegmentLoadingException, ExecutionException @Test public void testLoadDropSegment() throws SegmentLoadingException, ExecutionException, InterruptedException { - Assert.assertTrue(segmentManager.loadSegment(SEGMENTS.get(0), false, CommonCallback.NOOP)); - Assert.assertTrue(segmentManager.loadSegment(SEGMENTS.get(2), false, CommonCallback.NOOP)); + Assert.assertTrue(segmentManager.loadSegment(SEGMENTS.get(0), false, SegmentLazyLoadFailCallback.NOOP)); + Assert.assertTrue(segmentManager.loadSegment(SEGMENTS.get(2), false, SegmentLazyLoadFailCallback.NOOP)); final List> loadFutures = ImmutableList.of(SEGMENTS.get(1), SEGMENTS.get(3), SEGMENTS.get(4)) .stream() .map( segment -> executor.submit( - () -> segmentManager.loadSegment(segment, false, CommonCallback.NOOP) + () -> segmentManager.loadSegment(segment, false, SegmentLazyLoadFailCallback.NOOP) ) ) .collect(Collectors.toList()); @@ -303,10 +303,10 @@ public void testLoadDropSegment() throws SegmentLoadingException, ExecutionExcep public void testLoadDuplicatedSegmentsSequentially() throws SegmentLoadingException { for (DataSegment segment : SEGMENTS) { - Assert.assertTrue(segmentManager.loadSegment(segment, false, CommonCallback.NOOP)); + Assert.assertTrue(segmentManager.loadSegment(segment, false, SegmentLazyLoadFailCallback.NOOP)); } // try to load an existing segment - Assert.assertFalse(segmentManager.loadSegment(SEGMENTS.get(0), false, CommonCallback.NOOP)); + Assert.assertFalse(segmentManager.loadSegment(SEGMENTS.get(0), false, SegmentLazyLoadFailCallback.NOOP)); assertResult(SEGMENTS); } @@ -319,7 +319,7 @@ public void testLoadDuplicatedSegmentsInParallel() .stream() .map( segment -> executor.submit( - () -> segmentManager.loadSegment(segment, false, CommonCallback.NOOP) + () -> segmentManager.loadSegment(segment, false, SegmentLazyLoadFailCallback.NOOP) ) ) .collect(Collectors.toList()); @@ -340,7 +340,7 @@ public void testLoadDuplicatedSegmentsInParallel() @Test public void testNonExistingSegmentsSequentially() throws SegmentLoadingException { - Assert.assertTrue(segmentManager.loadSegment(SEGMENTS.get(0), false, CommonCallback.NOOP)); + Assert.assertTrue(segmentManager.loadSegment(SEGMENTS.get(0), false, SegmentLazyLoadFailCallback.NOOP)); // try to drop a non-existing segment of different data source segmentManager.dropSegment(SEGMENTS.get(2)); @@ -353,7 +353,7 @@ public void testNonExistingSegmentsSequentially() throws SegmentLoadingException public void testNonExistingSegmentsInParallel() throws SegmentLoadingException, ExecutionException, InterruptedException { - segmentManager.loadSegment(SEGMENTS.get(0), false, CommonCallback.NOOP); + segmentManager.loadSegment(SEGMENTS.get(0), false, SegmentLazyLoadFailCallback.NOOP); final List> futures = ImmutableList.of(SEGMENTS.get(1), SEGMENTS.get(2)) .stream() .map( @@ -376,7 +376,7 @@ public void testNonExistingSegmentsInParallel() @Test public void testRemoveEmptyTimeline() throws SegmentLoadingException { - segmentManager.loadSegment(SEGMENTS.get(0), false, CommonCallback.NOOP); + segmentManager.loadSegment(SEGMENTS.get(0), false, SegmentLazyLoadFailCallback.NOOP); assertResult(ImmutableList.of(SEGMENTS.get(0))); Assert.assertEquals(1, segmentManager.getDataSources().size()); segmentManager.dropSegment(SEGMENTS.get(0)); @@ -413,7 +413,7 @@ public void testLoadAndDropNonRootGenerationSegment() throws SegmentLoadingExcep 10 ); - segmentManager.loadSegment(segment, false, CommonCallback.NOOP); + segmentManager.loadSegment(segment, false, SegmentLazyLoadFailCallback.NOOP); assertResult(ImmutableList.of(segment)); segmentManager.dropSegment(segment); @@ -443,7 +443,7 @@ private void assertResult(List expectedExistingSegments) throws Seg segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk( - ReferenceCountingSegment.wrapSegment(SEGMENT_LOADER.getSegment(segment, false, CommonCallback.NOOP), segment.getShardSpec()) + ReferenceCountingSegment.wrapSegment(SEGMENT_LOADER.getSegment(segment, false, SegmentLazyLoadFailCallback.NOOP), segment.getShardSpec()) ) ); } diff --git a/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java b/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java index bd60639024eb..fa52f5766dd0 100644 --- a/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java +++ b/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java @@ -24,7 +24,7 @@ import com.fasterxml.jackson.databind.jsontype.NamedType; import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableMap; -import org.apache.druid.coordination.CommonCallback; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.FileUtils.FileCopyResult; @@ -130,7 +130,7 @@ public void testLoadSameSegment() throws IOException, ExecutionException, Interr final DataSegment segment = createSegment("2019-01-01/2019-01-02"); final List futures = IntStream .range(0, 16) - .mapToObj(i -> exec.submit(() -> segmentManager.loadSegment(segment, false, CommonCallback.NOOP))) + .mapToObj(i -> exec.submit(() -> segmentManager.loadSegment(segment, false, SegmentLazyLoadFailCallback.NOOP))) .collect(Collectors.toList()); for (Future future : futures) { future.get(); @@ -155,7 +155,7 @@ public void testLoadMultipleSegments() throws IOException, ExecutionException, I .mapToObj(i -> exec.submit(() -> { for (DataSegment segment : segments) { try { - segmentManager.loadSegment(segment, false, CommonCallback.NOOP); + segmentManager.loadSegment(segment, false, SegmentLazyLoadFailCallback.NOOP); } catch (SegmentLoadingException e) { throw new RuntimeException(e); @@ -223,7 +223,7 @@ public long size() private static class TestSegmentizerFactory implements SegmentizerFactory { @Override - public Segment factorize(DataSegment segment, File parentDir, boolean lazy, CommonCallback commonCallback) + public Segment factorize(DataSegment segment, File parentDir, boolean lazy, SegmentLazyLoadFailCallback SegmentLazyLoadFailCallback) { return new Segment() { diff --git a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java index 5ce850f2abff..80e5544c64d5 100644 --- a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java @@ -28,7 +28,7 @@ import org.apache.druid.client.cache.CachePopulatorStats; import org.apache.druid.client.cache.ForegroundCachePopulator; import org.apache.druid.client.cache.LocalCacheProvider; -import org.apache.druid.coordination.CommonCallback; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.Intervals; @@ -144,7 +144,7 @@ public boolean isSegmentLoaded(DataSegment segment) } @Override - public Segment getSegment(final DataSegment segment, boolean lazy, CommonCallback commonCallback) + public Segment getSegment(final DataSegment segment, boolean lazy, SegmentLazyLoadFailCallback SegmentLazyLoadFailCallback) { return new SegmentForTesting( MapUtils.getString(segment.getLoadSpec(), "version"), @@ -668,7 +668,7 @@ public void loadQueryable(String dataSource, String version, Interval interval) 123L ), false, - CommonCallback.NOOP + SegmentLazyLoadFailCallback.NOOP ); } catch (SegmentLoadingException e) { From ff6deb5a1c99f725902e0655efbdfbbdfc3b6c5e Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 7 Jan 2021 11:40:41 +0800 Subject: [PATCH 05/29] add UTs --- .../org/apache/druid/segment/IndexIOTest.java | 67 ++++++++++++++++++ .../v9SegmentPersistDir/segment/00000.smoosh | Bin 0 -> 1587 bytes .../v9SegmentPersistDir/segment/factory.json | 1 + .../v9SegmentPersistDir/segment/meta.smoosh | 9 +++ .../v9SegmentPersistDir/segment/version.bin | Bin 0 -> 4 bytes .../segmentWithDamagedFile/00000.smoosh | Bin 0 -> 1587 bytes .../segmentWithDamagedFile/factory.json | 1 + .../segmentWithDamagedFile/meta.smoosh | 9 +++ .../segmentWithDamagedFile/version.bin | Bin 0 -> 4 bytes 9 files changed, 87 insertions(+) create mode 100644 processing/src/test/resources/v9SegmentPersistDir/segment/00000.smoosh create mode 100644 processing/src/test/resources/v9SegmentPersistDir/segment/factory.json create mode 100644 processing/src/test/resources/v9SegmentPersistDir/segment/meta.smoosh create mode 100644 processing/src/test/resources/v9SegmentPersistDir/segment/version.bin create mode 100644 processing/src/test/resources/v9SegmentPersistDir/segmentWithDamagedFile/00000.smoosh create mode 100644 processing/src/test/resources/v9SegmentPersistDir/segmentWithDamagedFile/factory.json create mode 100644 processing/src/test/resources/v9SegmentPersistDir/segmentWithDamagedFile/meta.smoosh create mode 100644 processing/src/test/resources/v9SegmentPersistDir/segmentWithDamagedFile/version.bin diff --git a/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java b/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java index 2fb6bc39eaa4..8302d9965171 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java @@ -19,6 +19,7 @@ package org.apache.druid.segment; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; @@ -27,8 +28,10 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.UOE; import org.apache.druid.query.aggregation.Aggregator; @@ -49,6 +52,8 @@ import org.junit.runners.Parameterized; import javax.annotation.Nullable; +import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; @@ -337,4 +342,66 @@ public void testRowValidatorEquals() throws Exception } } } + + @Test + public void testLoadSegmentWithLazy() + { + final ObjectMapper mapper = new DefaultObjectMapper(); + final IndexIO indexIO = new IndexIO(mapper, () -> 0); + String path = this.getClass().getClassLoader().getResource("v9SegmentPersistDir/segment/").getPath(); + File inDir = new File(path); + try { + QueryableIndex queryableIndex = indexIO.loadIndex(inDir, true, SegmentLazyLoadFailCallback.NOOP); + Assert.assertNotNull(queryableIndex); + } + catch (IOException e) { + Assert.assertEquals("Failed to load current segment : " + inDir.getAbsolutePath(), e.getMessage()); + } + } + + @Test + public void testLoadSegmentDamagedFileWithLazy() throws IOException + { + final ObjectMapper mapper = new DefaultObjectMapper(); + final IndexIO indexIO = new IndexIO(mapper, () -> 0); + String path = this.getClass().getClassLoader().getResource("v9SegmentPersistDir/segmentWithDamagedFile/").getPath(); + + ForkSegmentLoadDropHandler segmentLoadDropHandler = new ForkSegmentLoadDropHandler(); + ForkSegment segment = new ForkSegment(true); + + File inDir = new File(path); + indexIO.loadIndex(inDir, true, () -> segmentLoadDropHandler.removeSegment(segment)); + Assert.assertFalse(segment.getSegmentExist()); + } + + public static class ForkSegmentLoadDropHandler + { + public void addSegment() + { + } + public void removeSegment(ForkSegment segment) + { + segment.setSegmentExist(false); + } + } + + public static class ForkSegment + { + private Boolean segmentExist; + + ForkSegment(Boolean segmentExist) + { + this.segmentExist = segmentExist; + } + + void setSegmentExist(Boolean value) + { + this.segmentExist = value; + } + + Boolean getSegmentExist() + { + return this.segmentExist; + } + } } diff --git a/processing/src/test/resources/v9SegmentPersistDir/segment/00000.smoosh b/processing/src/test/resources/v9SegmentPersistDir/segment/00000.smoosh new file mode 100644 index 0000000000000000000000000000000000000000..5bd4e75a27874bba498d052d0109e4408c562914 GIT binary patch literal 1587 zcmdT^O-~y!5Ot7>T2-wOv>Z|8s<$XodqG+*tptUYXo5)KQXz01v$g84`U5qJ7Ug&J zPw*Ez<1K-Z9tx+R)vP_wqT<&XL4>Im~d5{l$16=F{m~<7FW@F&6&#tyECwcT516Jo1Fs2}e$fUtuGX_Dhs`8>@ zpqRPF)K40eWthg^mRW;)E`=I)XEE{1c@YALkrpN?Rkx=f#s)W&XIk`~T(|N2=KFhY z|Hyc=GMDqDO{3HzM5bNe36A%LTa`2vakGLst=B99D1!x8OQ_UTxwBuFOMfQeN4uUu Z^~}o%myhoCjt(K*TcwhtMf6G3e*ox4qNM-; literal 0 HcmV?d00001 diff --git a/processing/src/test/resources/v9SegmentPersistDir/segment/factory.json b/processing/src/test/resources/v9SegmentPersistDir/segment/factory.json new file mode 100644 index 000000000000..f7b2cc330765 --- /dev/null +++ b/processing/src/test/resources/v9SegmentPersistDir/segment/factory.json @@ -0,0 +1 @@ +{"type":"mMapSegmentFactory"} \ No newline at end of file diff --git a/processing/src/test/resources/v9SegmentPersistDir/segment/meta.smoosh b/processing/src/test/resources/v9SegmentPersistDir/segment/meta.smoosh new file mode 100644 index 000000000000..70a831feef32 --- /dev/null +++ b/processing/src/test/resources/v9SegmentPersistDir/segment/meta.smoosh @@ -0,0 +1,9 @@ +v1,2147483647,1 +__time,0,0,141 +count,0,141,282 +dstIP,0,564,805 +index.drd,0,1046,1205 +metadata.drd,0,1205,1587 +srcIP,0,805,1046 +sum_bytes,0,282,423 +sum_packets,0,423,564 diff --git a/processing/src/test/resources/v9SegmentPersistDir/segment/version.bin b/processing/src/test/resources/v9SegmentPersistDir/segment/version.bin new file mode 100644 index 0000000000000000000000000000000000000000..3dd5ace49c65e7fd143d228be939732534095e78 GIT binary patch literal 4 LcmZQzVBiD*01W^N literal 0 HcmV?d00001 diff --git a/processing/src/test/resources/v9SegmentPersistDir/segmentWithDamagedFile/00000.smoosh b/processing/src/test/resources/v9SegmentPersistDir/segmentWithDamagedFile/00000.smoosh new file mode 100644 index 0000000000000000000000000000000000000000..2c58053664e0ccb333cd8db94701a7e8841e20f7 GIT binary patch literal 1587 zcmdT^OKTKC5Uvd|s8^_JTcnHc+JN;Ex*L+p`bpgP1&c1S$#AN0KE7;CxxXt?PGY{{Qiqa|Zfp!Bc zj<^b<&m8v=(Z@N9?FnT~jVC_Mq8Rz9@RX#SOm?;owuYPK>mh6Qn>7IF0X%^o{sKH~ zA>1LbSe$H}9%ld5k?C+bCT<{!OiP1SSH}gvd2wT2J-Q4iU2x$~h(skPE#$dA#Pd82$`Rz3Ytgk9y&QJ`I zOGBe(j)LHI?M2N|F)MRvoHQ!Sa808vvqpF%g_;h#KJm?Y5dw;l6(%Xw^p_u|`PRv^ zEc&-xzxpqm?`PjG$auc9l5@Y$qSPWpuHD#)itbBgRnl0*`3~i@Uh{}f87x9Au~Jj# l!G2aQJ5IvSwmrw{xt9?yAA37`zlY)8DwP~7qEDi^^#jm9qWb^< literal 0 HcmV?d00001 diff --git a/processing/src/test/resources/v9SegmentPersistDir/segmentWithDamagedFile/factory.json b/processing/src/test/resources/v9SegmentPersistDir/segmentWithDamagedFile/factory.json new file mode 100644 index 000000000000..f7b2cc330765 --- /dev/null +++ b/processing/src/test/resources/v9SegmentPersistDir/segmentWithDamagedFile/factory.json @@ -0,0 +1 @@ +{"type":"mMapSegmentFactory"} \ No newline at end of file diff --git a/processing/src/test/resources/v9SegmentPersistDir/segmentWithDamagedFile/meta.smoosh b/processing/src/test/resources/v9SegmentPersistDir/segmentWithDamagedFile/meta.smoosh new file mode 100644 index 000000000000..70a831feef32 --- /dev/null +++ b/processing/src/test/resources/v9SegmentPersistDir/segmentWithDamagedFile/meta.smoosh @@ -0,0 +1,9 @@ +v1,2147483647,1 +__time,0,0,141 +count,0,141,282 +dstIP,0,564,805 +index.drd,0,1046,1205 +metadata.drd,0,1205,1587 +srcIP,0,805,1046 +sum_bytes,0,282,423 +sum_packets,0,423,564 diff --git a/processing/src/test/resources/v9SegmentPersistDir/segmentWithDamagedFile/version.bin b/processing/src/test/resources/v9SegmentPersistDir/segmentWithDamagedFile/version.bin new file mode 100644 index 0000000000000000000000000000000000000000..3dd5ace49c65e7fd143d228be939732534095e78 GIT binary patch literal 4 LcmZQzVBiD*01W^N literal 0 HcmV?d00001 From 9fefdff30f57c621a9dead0ecb1b68b0453b7c69 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 7 Jan 2021 11:53:07 +0800 Subject: [PATCH 06/29] add UTs --- .idea/misc.xml | 4 ++-- .../java/org/apache/druid/segment/IndexIOTest.java | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index fe39ed623c9c..bf2061d7392d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - \ No newline at end of file + diff --git a/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java b/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java index 8302d9965171..f51ae4b8f899 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java @@ -370,11 +370,15 @@ public void testLoadSegmentDamagedFileWithLazy() throws IOException ForkSegment segment = new ForkSegment(true); File inDir = new File(path); - indexIO.loadIndex(inDir, true, () -> segmentLoadDropHandler.removeSegment(segment)); + QueryableIndex queryableIndex = indexIO.loadIndex(inDir, true, () -> segmentLoadDropHandler.removeSegment(segment)); + List columnNames = queryableIndex.getColumnNames(); + for (String columnName : columnNames) { + queryableIndex.getColumnHolder(columnName); + } Assert.assertFalse(segment.getSegmentExist()); } - public static class ForkSegmentLoadDropHandler + private static class ForkSegmentLoadDropHandler { public void addSegment() { @@ -385,7 +389,7 @@ public void removeSegment(ForkSegment segment) } } - public static class ForkSegment + private static class ForkSegment { private Boolean segmentExist; From 13710d9d2eb46d57a8fb4ec920706303024bd33e Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 7 Jan 2021 14:36:39 +0800 Subject: [PATCH 07/29] ut passed --- .../org/apache/druid/segment/IndexIOTest.java | 39 +++++++++++------- .../segmentWithDamagedFile/00000.smoosh | Bin 1587 -> 1588 bytes 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java b/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java index f51ae4b8f899..80e836471a3e 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java @@ -34,6 +34,7 @@ import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.UOE; +import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.segment.data.CompressionFactory; @@ -70,6 +71,7 @@ @RunWith(Parameterized.class) public class IndexIOTest extends InitializedNullHandlingTest { + private static final EmittingLogger log = new EmittingLogger(IndexIOTest.class); private static Interval DEFAULT_INTERVAL = Intervals.of("1970-01-01/2000-01-01"); private static final IndexSpec INDEX_SPEC = IndexMergerTestBase.makeIndexSpec( new ConciseBitmapSerdeFactory(), @@ -344,23 +346,18 @@ public void testRowValidatorEquals() throws Exception } @Test - public void testLoadSegmentWithLazy() + public void testLoadSegmentWithLazy() throws IOException { final ObjectMapper mapper = new DefaultObjectMapper(); final IndexIO indexIO = new IndexIO(mapper, () -> 0); String path = this.getClass().getClassLoader().getResource("v9SegmentPersistDir/segment/").getPath(); File inDir = new File(path); - try { - QueryableIndex queryableIndex = indexIO.loadIndex(inDir, true, SegmentLazyLoadFailCallback.NOOP); - Assert.assertNotNull(queryableIndex); - } - catch (IOException e) { - Assert.assertEquals("Failed to load current segment : " + inDir.getAbsolutePath(), e.getMessage()); - } + QueryableIndex queryableIndex = indexIO.loadIndex(inDir, true, SegmentLazyLoadFailCallback.NOOP); + Assert.assertNotNull(queryableIndex); } @Test - public void testLoadSegmentDamagedFileWithLazy() throws IOException + public void testLoadSegmentDamagedFileWithLazy() { final ObjectMapper mapper = new DefaultObjectMapper(); final IndexIO indexIO = new IndexIO(mapper, () -> 0); @@ -368,14 +365,28 @@ public void testLoadSegmentDamagedFileWithLazy() throws IOException ForkSegmentLoadDropHandler segmentLoadDropHandler = new ForkSegmentLoadDropHandler(); ForkSegment segment = new ForkSegment(true); - + Assert.assertTrue(segment.getSegmentExist()); File inDir = new File(path); - QueryableIndex queryableIndex = indexIO.loadIndex(inDir, true, () -> segmentLoadDropHandler.removeSegment(segment)); - List columnNames = queryableIndex.getColumnNames(); - for (String columnName : columnNames) { - queryableIndex.getColumnHolder(columnName); + Exception e = null; + + try { + QueryableIndex queryableIndex = indexIO.loadIndex(inDir, true, () -> segmentLoadDropHandler.removeSegment(segment)); + Assert.assertNotNull(queryableIndex); + queryableIndex.getDimensionHandlers(); + List columnNames = queryableIndex.getColumnNames(); + log.info(columnNames.toString()); + for (String columnName : columnNames) { + queryableIndex.getColumnHolder(columnName).toString(); + } } + catch (Exception ex) { + // Do nothing. Can ignore exceptions here. + e = ex; + log.warn(ex, "exception"); + } + Assert.assertNotNull(e); Assert.assertFalse(segment.getSegmentExist()); + } private static class ForkSegmentLoadDropHandler diff --git a/processing/src/test/resources/v9SegmentPersistDir/segmentWithDamagedFile/00000.smoosh b/processing/src/test/resources/v9SegmentPersistDir/segmentWithDamagedFile/00000.smoosh index 2c58053664e0ccb333cd8db94701a7e8841e20f7..aaf3449a90ef3edc80b1b6321d9d46afd7ae6a61 100644 GIT binary patch delta 19 acmdnYvxR5ECFY!*oXLrdwwnbRl~@5vo(4w% delta 46 wcmdnOvzceYrOEM(>QW303Ji>l3_u{tz);8lVr7DOj0}?(GInpC&p4F@0M`Ww)c^nh From f67b334ad9a0a6d9e46855a4505e8c0fda6cf13b Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 7 Jan 2021 14:37:45 +0800 Subject: [PATCH 08/29] ut passed --- .../src/test/java/org/apache/druid/segment/IndexIOTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java b/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java index 80e836471a3e..8bb153772c48 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java @@ -34,7 +34,6 @@ import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.UOE; -import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.segment.data.CompressionFactory; @@ -71,7 +70,6 @@ @RunWith(Parameterized.class) public class IndexIOTest extends InitializedNullHandlingTest { - private static final EmittingLogger log = new EmittingLogger(IndexIOTest.class); private static Interval DEFAULT_INTERVAL = Intervals.of("1970-01-01/2000-01-01"); private static final IndexSpec INDEX_SPEC = IndexMergerTestBase.makeIndexSpec( new ConciseBitmapSerdeFactory(), @@ -374,7 +372,6 @@ public void testLoadSegmentDamagedFileWithLazy() Assert.assertNotNull(queryableIndex); queryableIndex.getDimensionHandlers(); List columnNames = queryableIndex.getColumnNames(); - log.info(columnNames.toString()); for (String columnName : columnNames) { queryableIndex.getColumnHolder(columnName).toString(); } @@ -382,7 +379,6 @@ public void testLoadSegmentDamagedFileWithLazy() catch (Exception ex) { // Do nothing. Can ignore exceptions here. e = ex; - log.warn(ex, "exception"); } Assert.assertNotNull(e); Assert.assertFalse(segment.getSegmentExist()); From 7dd33e2571f913c7c1b4dcb8d36dbf4f2c209389 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 7 Jan 2021 15:04:35 +0800 Subject: [PATCH 09/29] opti imports --- .../batch/parallel/AbstractMultiPhaseParallelIndexingTest.java | 2 +- processing/src/main/java/org/apache/druid/segment/IndexIO.java | 2 +- .../{ => segment}/coordination/SegmentLazyLoadFailCallback.java | 2 +- .../BroadcastJoinableMMappedQueryableSegmentizerFactory.java | 2 +- .../segment/loading/MMappedQueryableSegmentizerFactory.java | 2 +- .../org/apache/druid/segment/loading/SegmentizerFactory.java | 2 +- .../org/apache/druid/segment/CustomSegmentizerFactoryTest.java | 2 +- .../src/test/java/org/apache/druid/segment/IndexIOTest.java | 2 +- .../segment/join/table/BroadcastSegmentIndexedTableTest.java | 2 +- ...BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java | 2 +- .../java/org/apache/druid/segment/loading/SegmentLoader.java | 2 +- .../druid/segment/loading/SegmentLoaderLocalCacheManager.java | 2 +- .../src/main/java/org/apache/druid/server/SegmentManager.java | 2 +- .../apache/druid/segment/loading/CacheTestSegmentLoader.java | 2 +- .../server/SegmentManagerBroadcastJoinIndexedTableTest.java | 2 +- .../test/java/org/apache/druid/server/SegmentManagerTest.java | 2 +- .../org/apache/druid/server/SegmentManagerThreadSafetyTest.java | 2 +- .../org/apache/druid/server/coordination/ServerManagerTest.java | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) rename processing/src/main/java/org/apache/druid/{ => segment}/coordination/SegmentLazyLoadFailCallback.java (95%) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java index 9e035d536a7d..c1566ef6f9ee 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java @@ -20,7 +20,6 @@ package org.apache.druid.indexing.common.task.batch.parallel; import com.google.common.base.Preconditions; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.LocalInputSource; @@ -52,6 +51,7 @@ import org.apache.druid.query.scan.ScanResultValue; import org.apache.druid.query.spec.SpecificSegmentSpec; import org.apache.druid.segment.Segment; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.GranularitySpec; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index abe2514cd362..5ba61deab603 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -38,7 +38,6 @@ import org.apache.druid.collections.bitmap.ImmutableBitmap; import org.apache.druid.collections.spatial.ImmutableRTree; import org.apache.druid.common.utils.SerializerUtils; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.IOE; import org.apache.druid.java.util.common.ISE; @@ -54,6 +53,7 @@ import org.apache.druid.segment.column.ColumnDescriptor; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.data.BitmapSerde; import org.apache.druid.segment.data.BitmapSerdeFactory; import org.apache.druid.segment.data.CompressedColumnarLongsSupplier; diff --git a/processing/src/main/java/org/apache/druid/coordination/SegmentLazyLoadFailCallback.java b/processing/src/main/java/org/apache/druid/segment/coordination/SegmentLazyLoadFailCallback.java similarity index 95% rename from processing/src/main/java/org/apache/druid/coordination/SegmentLazyLoadFailCallback.java rename to processing/src/main/java/org/apache/druid/segment/coordination/SegmentLazyLoadFailCallback.java index a92e9e849b57..58b144054a6e 100644 --- a/processing/src/main/java/org/apache/druid/coordination/SegmentLazyLoadFailCallback.java +++ b/processing/src/main/java/org/apache/druid/segment/coordination/SegmentLazyLoadFailCallback.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.coordination; +package org.apache.druid.segment.coordination; public interface SegmentLazyLoadFailCallback { diff --git a/processing/src/main/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactory.java b/processing/src/main/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactory.java index 7dd1e5c0094b..9802ccd0a9b7 100644 --- a/processing/src/main/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactory.java @@ -22,10 +22,10 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.IndexIO; import org.apache.druid.segment.QueryableIndexSegment; import org.apache.druid.segment.Segment; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.join.table.BroadcastSegmentIndexedTable; import org.apache.druid.segment.join.table.IndexedTable; import org.apache.druid.timeline.DataSegment; diff --git a/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java b/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java index 1ef626a9144b..00f44bc16500 100644 --- a/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java @@ -21,10 +21,10 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.google.common.base.Preconditions; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.IndexIO; import org.apache.druid.segment.QueryableIndexSegment; import org.apache.druid.segment.Segment; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.timeline.DataSegment; import java.io.File; diff --git a/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java b/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java index 0e29f2eeb0f1..3d16313ccf6c 100644 --- a/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java @@ -20,8 +20,8 @@ package org.apache.druid.segment.loading; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.Segment; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.timeline.DataSegment; import java.io.File; diff --git a/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java b/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java index 4f230ed5cb91..e49d0ac95475 100644 --- a/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java @@ -22,12 +22,12 @@ import com.fasterxml.jackson.databind.InjectableValues; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.jsontype.NamedType; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.jackson.SegmentizerModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.query.expression.TestExprMacroTable; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.loading.MMappedQueryableSegmentizerFactory; import org.apache.druid.segment.loading.SegmentLoadingException; diff --git a/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java b/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java index 8bb153772c48..9062663ef3d3 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java @@ -28,7 +28,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.jackson.DefaultObjectMapper; @@ -36,6 +35,7 @@ import org.apache.druid.java.util.common.UOE; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.data.CompressionFactory; import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.data.ConciseBitmapSerdeFactory; diff --git a/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java b/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java index b1cdc7e4b5b8..ea689a0e6ee8 100644 --- a/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java +++ b/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java @@ -26,7 +26,6 @@ import com.google.common.collect.ImmutableSet; import it.unimi.dsi.fastutil.ints.IntList; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.jackson.SegmentizerModule; import org.apache.druid.java.util.common.DateTimes; @@ -47,6 +46,7 @@ import org.apache.druid.segment.TestIndex; import org.apache.druid.segment.column.BaseColumn; import org.apache.druid.segment.column.ColumnHolder; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.loading.MMappedQueryableSegmentizerFactory; import org.apache.druid.segment.loading.SegmentLoadingException; diff --git a/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java b/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java index f9ad4ec66a32..2fecd3f447bb 100644 --- a/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java @@ -24,7 +24,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.jackson.SegmentizerModule; import org.apache.druid.java.util.common.DateTimes; @@ -37,6 +36,7 @@ import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.Segment; import org.apache.druid.segment.TestIndex; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.join.table.BroadcastSegmentIndexedTable; import org.apache.druid.segment.join.table.IndexedTable; diff --git a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java index 3cd8a85ca81c..a5a5c14ae2e5 100644 --- a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java +++ b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java @@ -19,8 +19,8 @@ package org.apache.druid.segment.loading; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.Segment; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.timeline.DataSegment; import java.io.File; diff --git a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java index 3bde319bea5b..18a27901a6a3 100644 --- a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java +++ b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.guice.annotations.Json; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.ISE; diff --git a/server/src/main/java/org/apache/druid/server/SegmentManager.java b/server/src/main/java/org/apache/druid/server/SegmentManager.java index 3ea137ec3cb8..62d3906d7b5d 100644 --- a/server/src/main/java/org/apache/druid/server/SegmentManager.java +++ b/server/src/main/java/org/apache/druid/server/SegmentManager.java @@ -23,7 +23,6 @@ import com.google.common.collect.Ordering; import com.google.inject.Inject; import org.apache.druid.common.guava.SettableSupplier; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.io.Closer; @@ -32,6 +31,7 @@ import org.apache.druid.query.planning.DataSourceAnalysis; import org.apache.druid.segment.ReferenceCountingSegment; import org.apache.druid.segment.Segment; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.join.table.IndexedTable; import org.apache.druid.segment.join.table.ReferenceCountingIndexedTable; import org.apache.druid.segment.loading.SegmentLoader; diff --git a/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java b/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java index 3653b82eb0ca..b48a126102f0 100644 --- a/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java +++ b/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java @@ -19,11 +19,11 @@ package org.apache.druid.segment.loading; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.java.util.common.MapUtils; import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.Segment; import org.apache.druid.segment.StorageAdapter; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.joda.time.Interval; diff --git a/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java b/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java index 838b970e4192..7fd5448cbedb 100644 --- a/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java +++ b/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java @@ -25,7 +25,6 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.jackson.SegmentizerModule; import org.apache.druid.java.util.common.DateTimes; @@ -43,6 +42,7 @@ import org.apache.druid.segment.IndexMergerV9; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.TestIndex; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.join.BroadcastTableJoinableFactory; import org.apache.druid.segment.join.JoinConditionAnalysis; diff --git a/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java b/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java index 7ba5f6fa86b9..990cad4a5098 100644 --- a/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java @@ -22,7 +22,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.MapUtils; import org.apache.druid.query.TableDataSource; @@ -31,6 +30,7 @@ import org.apache.druid.segment.ReferenceCountingSegment; import org.apache.druid.segment.Segment; import org.apache.druid.segment.StorageAdapter; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.loading.SegmentLoader; import org.apache.druid.segment.loading.SegmentLoadingException; import org.apache.druid.server.SegmentManager.DataSourceState; diff --git a/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java b/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java index fa52f5766dd0..efbbd80db17c 100644 --- a/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java +++ b/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.databind.jsontype.NamedType; import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableMap; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.FileUtils.FileCopyResult; @@ -36,6 +35,7 @@ import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.Segment; import org.apache.druid.segment.StorageAdapter; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.loading.DataSegmentPusher; import org.apache.druid.segment.loading.LocalDataSegmentPuller; import org.apache.druid.segment.loading.LocalLoadSpec; diff --git a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java index 80e5544c64d5..446fefd1e628 100644 --- a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java @@ -28,7 +28,6 @@ import org.apache.druid.client.cache.CachePopulatorStats; import org.apache.druid.client.cache.ForegroundCachePopulator; import org.apache.druid.client.cache.LocalCacheProvider; -import org.apache.druid.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.Intervals; @@ -74,6 +73,7 @@ import org.apache.druid.segment.ReferenceCountingSegment; import org.apache.druid.segment.Segment; import org.apache.druid.segment.StorageAdapter; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.join.NoopJoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; import org.apache.druid.segment.loading.SegmentLoadingException; From eac89198e4079772fa56d100b18ba4b8aa964e93 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 7 Jan 2021 16:10:30 +0800 Subject: [PATCH 10/29] done --- .../src/main/java/org/apache/druid/segment/IndexMergerV9.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java b/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java index cb65289d78d6..df4552a60466 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java @@ -45,6 +45,7 @@ import org.apache.druid.segment.column.ColumnDescriptor; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.data.GenericIndexed; import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.incremental.IncrementalIndexAdapter; @@ -1011,7 +1012,7 @@ private File multiphaseMerge( // convert Files to QueryableIndexIndexableAdapter and do another merge phase List qIndexAdapters = new ArrayList<>(); for (File outputFile : currentOutputs) { - QueryableIndex qIndex = indexIO.loadIndex(outputFile, true); + QueryableIndex qIndex = indexIO.loadIndex(outputFile, true, SegmentLazyLoadFailCallback.NOOP); qIndexAdapters.add(new QueryableIndexIndexableAdapter(qIndex)); } currentPhases = getMergePhases(qIndexAdapters, maxColumnsToMerge); From e1eca76b4ae50733b9d922c4a94c356d18d51201 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 7 Jan 2021 16:39:46 +0800 Subject: [PATCH 11/29] done --- .../src/main/java/org/apache/druid/segment/IndexIO.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index 5ba61deab603..83e174e01061 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -622,9 +622,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen try { return deserializeColumn(mapper, timeBuffer, smooshedFiles); } - catch (IOException | RuntimeException e) { - log.warn(e, "Exception when deserialize Column " + ColumnHolder.TIME_COLUMN_NAME); - loadFailed.execute(); + catch (IOException e) { throw Throwables.propagate(e); } } From 3d567da3d1a493d63fac36705a371cb9742dc0da Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 8 Jan 2021 10:40:11 +0800 Subject: [PATCH 12/29] fix checkstyle --- .../druid/segment/loading/SegmentLoaderLocalCacheManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java index 18a27901a6a3..bafa4e50d0d8 100644 --- a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java +++ b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java @@ -22,13 +22,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.guice.annotations.Json; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.segment.IndexIO; import org.apache.druid.segment.Segment; +import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.timeline.DataSegment; import javax.annotation.Nonnull; From 6b82e6c64ded714a0285afac83876e1d4827edaf Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 8 Jan 2021 12:22:01 +0800 Subject: [PATCH 13/29] modify uts --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a25ea08f2ab2..2e71beeb71e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -154,7 +154,7 @@ jobs: # Set MAVEN_OPTS for Surefire launcher. Skip remoteresources to avoid intermittent connection timeouts when # resolving the SIGAR dependency. - > - MAVEN_OPTS='-Xmx800m' ${MVN} test -pl ${MAVEN_PROJECTS} + MAVEN_OPTS='-Xmx1400m' ${MVN} test -pl ${MAVEN_PROJECTS} ${MAVEN_SKIP} -Dremoteresources.skip=true -Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL} - sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0" - free -m From f113ed3e803b1bc8e110190b86bf7362a1844c69 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Sat, 9 Jan 2021 19:50:27 +0800 Subject: [PATCH 14/29] modify logs --- processing/src/main/java/org/apache/druid/segment/IndexIO.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index 83e174e01061..9e835e365d30 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -601,7 +601,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen return deserializeColumn(mapper, colBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e, "Exception when deserialize Column " + columnName); + log.warn(e, "Exception when deserialize Column [%s].", columnName); loadFailed.execute(); throw Throwables.propagate(e); } From 5bfca47513fad5274ec0c9578291622e1b8ed2c3 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 12 Jan 2021 11:27:19 +0800 Subject: [PATCH 15/29] changing the package of SegmentLazyLoadFailCallback.java to org.apache.druid.segment --- .../batch/parallel/AbstractMultiPhaseParallelIndexingTest.java | 2 +- processing/src/main/java/org/apache/druid/segment/IndexIO.java | 1 - .../src/main/java/org/apache/druid/segment/IndexMergerV9.java | 1 - .../segment/{coordination => }/SegmentLazyLoadFailCallback.java | 2 +- .../BroadcastJoinableMMappedQueryableSegmentizerFactory.java | 2 +- .../segment/loading/MMappedQueryableSegmentizerFactory.java | 2 +- .../org/apache/druid/segment/loading/SegmentizerFactory.java | 2 +- .../org/apache/druid/segment/CustomSegmentizerFactoryTest.java | 1 - .../src/test/java/org/apache/druid/segment/IndexIOTest.java | 1 - .../segment/join/table/BroadcastSegmentIndexedTableTest.java | 2 +- ...BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java | 2 +- .../java/org/apache/druid/segment/loading/SegmentLoader.java | 2 +- .../druid/segment/loading/SegmentLoaderLocalCacheManager.java | 2 +- .../src/main/java/org/apache/druid/server/SegmentManager.java | 2 +- .../apache/druid/segment/loading/CacheTestSegmentLoader.java | 2 +- .../server/SegmentManagerBroadcastJoinIndexedTableTest.java | 2 +- .../test/java/org/apache/druid/server/SegmentManagerTest.java | 2 +- .../org/apache/druid/server/SegmentManagerThreadSafetyTest.java | 2 +- .../org/apache/druid/server/coordination/ServerManagerTest.java | 2 +- 19 files changed, 15 insertions(+), 19 deletions(-) rename processing/src/main/java/org/apache/druid/segment/{coordination => }/SegmentLazyLoadFailCallback.java (95%) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java index c1566ef6f9ee..d06d17c39309 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractMultiPhaseParallelIndexingTest.java @@ -51,7 +51,7 @@ import org.apache.druid.query.scan.ScanResultValue; import org.apache.druid.query.spec.SpecificSegmentSpec; import org.apache.druid.segment.Segment; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.segment.indexing.DataSchema; import org.apache.druid.segment.indexing.granularity.GranularitySpec; import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index 9e835e365d30..5f792b315dbd 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -53,7 +53,6 @@ import org.apache.druid.segment.column.ColumnDescriptor; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ValueType; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.data.BitmapSerde; import org.apache.druid.segment.data.BitmapSerdeFactory; import org.apache.druid.segment.data.CompressedColumnarLongsSupplier; diff --git a/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java b/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java index df4552a60466..02618daf929a 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexMergerV9.java @@ -45,7 +45,6 @@ import org.apache.druid.segment.column.ColumnDescriptor; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ValueType; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.data.GenericIndexed; import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.incremental.IncrementalIndexAdapter; diff --git a/processing/src/main/java/org/apache/druid/segment/coordination/SegmentLazyLoadFailCallback.java b/processing/src/main/java/org/apache/druid/segment/SegmentLazyLoadFailCallback.java similarity index 95% rename from processing/src/main/java/org/apache/druid/segment/coordination/SegmentLazyLoadFailCallback.java rename to processing/src/main/java/org/apache/druid/segment/SegmentLazyLoadFailCallback.java index 58b144054a6e..babc6ba11237 100644 --- a/processing/src/main/java/org/apache/druid/segment/coordination/SegmentLazyLoadFailCallback.java +++ b/processing/src/main/java/org/apache/druid/segment/SegmentLazyLoadFailCallback.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.segment.coordination; +package org.apache.druid.segment; public interface SegmentLazyLoadFailCallback { diff --git a/processing/src/main/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactory.java b/processing/src/main/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactory.java index 9802ccd0a9b7..75f2882529a7 100644 --- a/processing/src/main/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactory.java @@ -25,7 +25,7 @@ import org.apache.druid.segment.IndexIO; import org.apache.druid.segment.QueryableIndexSegment; import org.apache.druid.segment.Segment; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.segment.join.table.BroadcastSegmentIndexedTable; import org.apache.druid.segment.join.table.IndexedTable; import org.apache.druid.timeline.DataSegment; diff --git a/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java b/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java index 00f44bc16500..d3124cb122f9 100644 --- a/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/loading/MMappedQueryableSegmentizerFactory.java @@ -24,7 +24,7 @@ import org.apache.druid.segment.IndexIO; import org.apache.druid.segment.QueryableIndexSegment; import org.apache.druid.segment.Segment; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.timeline.DataSegment; import java.io.File; diff --git a/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java b/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java index 3d16313ccf6c..25df3c56f317 100644 --- a/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java +++ b/processing/src/main/java/org/apache/druid/segment/loading/SegmentizerFactory.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.apache.druid.segment.Segment; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.timeline.DataSegment; import java.io.File; diff --git a/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java b/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java index e49d0ac95475..9728f230f024 100644 --- a/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/segment/CustomSegmentizerFactoryTest.java @@ -27,7 +27,6 @@ import org.apache.druid.java.util.common.Intervals; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.query.expression.TestExprMacroTable; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.loading.MMappedQueryableSegmentizerFactory; import org.apache.druid.segment.loading.SegmentLoadingException; diff --git a/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java b/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java index 3ff8325388c6..a8c557c75801 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java @@ -35,7 +35,6 @@ import org.apache.druid.java.util.common.UOE; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.CountAggregatorFactory; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; import org.apache.druid.segment.data.CompressionFactory; import org.apache.druid.segment.data.CompressionStrategy; import org.apache.druid.segment.data.ConciseBitmapSerdeFactory; diff --git a/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java b/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java index ea689a0e6ee8..d49192c4ef2e 100644 --- a/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java +++ b/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java @@ -46,7 +46,7 @@ import org.apache.druid.segment.TestIndex; import org.apache.druid.segment.column.BaseColumn; import org.apache.druid.segment.column.ColumnHolder; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.loading.MMappedQueryableSegmentizerFactory; import org.apache.druid.segment.loading.SegmentLoadingException; diff --git a/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java b/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java index 2fecd3f447bb..ca4bfbbc30ec 100644 --- a/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java @@ -36,7 +36,7 @@ import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.Segment; import org.apache.druid.segment.TestIndex; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.join.table.BroadcastSegmentIndexedTable; import org.apache.druid.segment.join.table.IndexedTable; diff --git a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java index a5a5c14ae2e5..741cfa1373ad 100644 --- a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java +++ b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoader.java @@ -20,7 +20,7 @@ package org.apache.druid.segment.loading; import org.apache.druid.segment.Segment; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.timeline.DataSegment; import java.io.File; diff --git a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java index bafa4e50d0d8..59917515aa59 100644 --- a/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java +++ b/server/src/main/java/org/apache/druid/segment/loading/SegmentLoaderLocalCacheManager.java @@ -28,7 +28,7 @@ import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.segment.IndexIO; import org.apache.druid.segment.Segment; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.timeline.DataSegment; import javax.annotation.Nonnull; diff --git a/server/src/main/java/org/apache/druid/server/SegmentManager.java b/server/src/main/java/org/apache/druid/server/SegmentManager.java index 62d3906d7b5d..c3636d3a9780 100644 --- a/server/src/main/java/org/apache/druid/server/SegmentManager.java +++ b/server/src/main/java/org/apache/druid/server/SegmentManager.java @@ -31,7 +31,7 @@ import org.apache.druid.query.planning.DataSourceAnalysis; import org.apache.druid.segment.ReferenceCountingSegment; import org.apache.druid.segment.Segment; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.segment.join.table.IndexedTable; import org.apache.druid.segment.join.table.ReferenceCountingIndexedTable; import org.apache.druid.segment.loading.SegmentLoader; diff --git a/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java b/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java index b48a126102f0..3a2fe279326d 100644 --- a/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java +++ b/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java @@ -23,7 +23,7 @@ import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.Segment; import org.apache.druid.segment.StorageAdapter; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.joda.time.Interval; diff --git a/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java b/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java index 7fd5448cbedb..542264b4c256 100644 --- a/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java +++ b/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java @@ -42,7 +42,7 @@ import org.apache.druid.segment.IndexMergerV9; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.TestIndex; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.join.BroadcastTableJoinableFactory; import org.apache.druid.segment.join.JoinConditionAnalysis; diff --git a/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java b/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java index 990cad4a5098..15b68856c8a4 100644 --- a/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java @@ -30,7 +30,7 @@ import org.apache.druid.segment.ReferenceCountingSegment; import org.apache.druid.segment.Segment; import org.apache.druid.segment.StorageAdapter; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.segment.loading.SegmentLoader; import org.apache.druid.segment.loading.SegmentLoadingException; import org.apache.druid.server.SegmentManager.DataSourceState; diff --git a/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java b/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java index efbbd80db17c..a1546a712237 100644 --- a/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java +++ b/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java @@ -35,7 +35,7 @@ import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.Segment; import org.apache.druid.segment.StorageAdapter; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.segment.loading.DataSegmentPusher; import org.apache.druid.segment.loading.LocalDataSegmentPuller; import org.apache.druid.segment.loading.LocalLoadSpec; diff --git a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java index 446fefd1e628..8b9fc40d0f46 100644 --- a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java @@ -73,7 +73,7 @@ import org.apache.druid.segment.ReferenceCountingSegment; import org.apache.druid.segment.Segment; import org.apache.druid.segment.StorageAdapter; -import org.apache.druid.segment.coordination.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.segment.join.NoopJoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; import org.apache.druid.segment.loading.SegmentLoadingException; From c7d9115a568b8e98a407362cac1f6cca7644d1e8 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 12 Jan 2021 11:48:28 +0800 Subject: [PATCH 16/29] merge from master --- .../druid/server/coordination/SegmentLoadDropHandlerTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/apache/druid/server/coordination/SegmentLoadDropHandlerTest.java b/server/src/test/java/org/apache/druid/server/coordination/SegmentLoadDropHandlerTest.java index a9797846e64f..521508399160 100644 --- a/server/src/test/java/org/apache/druid/server/coordination/SegmentLoadDropHandlerTest.java +++ b/server/src/test/java/org/apache/druid/server/coordination/SegmentLoadDropHandlerTest.java @@ -31,6 +31,7 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.segment.IndexIO; +import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.loading.CacheTestSegmentLoader; import org.apache.druid.segment.loading.SegmentLoaderConfig; @@ -539,7 +540,7 @@ public void testProcessBatch() throws Exception public void testProcessBatchDuplicateLoadRequestsWhenFirstRequestFailsSecondRequestShouldSucceed() throws Exception { final SegmentManager segmentManager = Mockito.mock(SegmentManager.class); - Mockito.when(segmentManager.loadSegment(ArgumentMatchers.any(), ArgumentMatchers.anyBoolean())) + Mockito.when(segmentManager.loadSegment(ArgumentMatchers.any(), ArgumentMatchers.anyBoolean(), SegmentLazyLoadFailCallback.NOOP)) .thenThrow(new RuntimeException("segment loading failure test")) .thenReturn(true); final SegmentLoadDropHandler segmentLoadDropHandler = new SegmentLoadDropHandler( From ba5dd23355f5aad070743702a5ad501c068a2e72 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 12 Jan 2021 16:54:34 +0800 Subject: [PATCH 17/29] modify import orders --- .idea/misc.xml | 4 ++-- .../segment/join/table/BroadcastSegmentIndexedTableTest.java | 2 +- ...oadcastJoinableMMappedQueryableSegmentizerFactoryTest.java | 2 +- .../apache/druid/segment/loading/CacheTestSegmentLoader.java | 2 +- .../server/SegmentManagerBroadcastJoinIndexedTableTest.java | 2 +- .../test/java/org/apache/druid/server/SegmentManagerTest.java | 2 +- .../apache/druid/server/SegmentManagerThreadSafetyTest.java | 2 +- .../apache/druid/server/coordination/ServerManagerTest.java | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index bf2061d7392d..fe39ed623c9c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - + \ No newline at end of file diff --git a/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java b/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java index d49192c4ef2e..4635a7bf25a8 100644 --- a/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java +++ b/processing/src/test/java/org/apache/druid/segment/join/table/BroadcastSegmentIndexedTableTest.java @@ -42,11 +42,11 @@ import org.apache.druid.segment.IndexMergerV9; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.QueryableIndexSegment; +import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.segment.SimpleAscendingOffset; import org.apache.druid.segment.TestIndex; import org.apache.druid.segment.column.BaseColumn; import org.apache.druid.segment.column.ColumnHolder; -import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.loading.MMappedQueryableSegmentizerFactory; import org.apache.druid.segment.loading.SegmentLoadingException; diff --git a/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java b/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java index ca4bfbbc30ec..0e8ac933a6ce 100644 --- a/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java +++ b/processing/src/test/java/org/apache/druid/segment/loading/BroadcastJoinableMMappedQueryableSegmentizerFactoryTest.java @@ -35,8 +35,8 @@ import org.apache.druid.segment.IndexMergerV9; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.Segment; -import org.apache.druid.segment.TestIndex; import org.apache.druid.segment.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.TestIndex; import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.join.table.BroadcastSegmentIndexedTable; import org.apache.druid.segment.join.table.IndexedTable; diff --git a/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java b/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java index 3a2fe279326d..557537c06811 100644 --- a/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java +++ b/server/src/test/java/org/apache/druid/segment/loading/CacheTestSegmentLoader.java @@ -22,8 +22,8 @@ import org.apache.druid.java.util.common.MapUtils; import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.Segment; -import org.apache.druid.segment.StorageAdapter; import org.apache.druid.segment.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.StorageAdapter; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.joda.time.Interval; diff --git a/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java b/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java index 542264b4c256..38c410019400 100644 --- a/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java +++ b/server/src/test/java/org/apache/druid/server/SegmentManagerBroadcastJoinIndexedTableTest.java @@ -41,8 +41,8 @@ import org.apache.druid.segment.IndexMerger; import org.apache.druid.segment.IndexMergerV9; import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.TestIndex; import org.apache.druid.segment.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.TestIndex; import org.apache.druid.segment.incremental.IncrementalIndex; import org.apache.druid.segment.join.BroadcastTableJoinableFactory; import org.apache.druid.segment.join.JoinConditionAnalysis; diff --git a/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java b/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java index 15b68856c8a4..762339f7f1f8 100644 --- a/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/SegmentManagerTest.java @@ -29,8 +29,8 @@ import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.ReferenceCountingSegment; import org.apache.druid.segment.Segment; -import org.apache.druid.segment.StorageAdapter; import org.apache.druid.segment.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.StorageAdapter; import org.apache.druid.segment.loading.SegmentLoader; import org.apache.druid.segment.loading.SegmentLoadingException; import org.apache.druid.server.SegmentManager.DataSourceState; diff --git a/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java b/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java index a1546a712237..87587ce69a66 100644 --- a/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java +++ b/server/src/test/java/org/apache/druid/server/SegmentManagerThreadSafetyTest.java @@ -34,8 +34,8 @@ import org.apache.druid.segment.IndexIO; import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.Segment; -import org.apache.druid.segment.StorageAdapter; import org.apache.druid.segment.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.StorageAdapter; import org.apache.druid.segment.loading.DataSegmentPusher; import org.apache.druid.segment.loading.LocalDataSegmentPuller; import org.apache.druid.segment.loading.LocalLoadSpec; diff --git a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java index 8b9fc40d0f46..8de42a86a1c7 100644 --- a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java @@ -72,8 +72,8 @@ import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.ReferenceCountingSegment; import org.apache.druid.segment.Segment; -import org.apache.druid.segment.StorageAdapter; import org.apache.druid.segment.SegmentLazyLoadFailCallback; +import org.apache.druid.segment.StorageAdapter; import org.apache.druid.segment.join.NoopJoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; import org.apache.druid.segment.loading.SegmentLoadingException; From 5c892323446b07a33284a983f86094e8f7902b32 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 12 Jan 2021 17:07:53 +0800 Subject: [PATCH 18/29] merge from master --- .idea/misc.xml | 4 ++-- .../druid/server/coordination/SegmentLoadDropHandlerTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index fe39ed623c9c..bf2061d7392d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - \ No newline at end of file + diff --git a/server/src/test/java/org/apache/druid/server/coordination/SegmentLoadDropHandlerTest.java b/server/src/test/java/org/apache/druid/server/coordination/SegmentLoadDropHandlerTest.java index 521508399160..fc97854d39d6 100644 --- a/server/src/test/java/org/apache/druid/server/coordination/SegmentLoadDropHandlerTest.java +++ b/server/src/test/java/org/apache/druid/server/coordination/SegmentLoadDropHandlerTest.java @@ -540,7 +540,7 @@ public void testProcessBatch() throws Exception public void testProcessBatchDuplicateLoadRequestsWhenFirstRequestFailsSecondRequestShouldSucceed() throws Exception { final SegmentManager segmentManager = Mockito.mock(SegmentManager.class); - Mockito.when(segmentManager.loadSegment(ArgumentMatchers.any(), ArgumentMatchers.anyBoolean(), SegmentLazyLoadFailCallback.NOOP)) + Mockito.when(segmentManager.loadSegment(ArgumentMatchers.any(), ArgumentMatchers.anyBoolean(), ArgumentMatchers.any())) .thenThrow(new RuntimeException("segment loading failure test")) .thenReturn(true); final SegmentLoadDropHandler segmentLoadDropHandler = new SegmentLoadDropHandler( From 7714f81bfcf21bf0c581a2b1135a9c27a0f82b07 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 12 Jan 2021 17:17:31 +0800 Subject: [PATCH 19/29] merge from master --- .../druid/server/coordination/SegmentLoadDropHandlerTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/apache/druid/server/coordination/SegmentLoadDropHandlerTest.java b/server/src/test/java/org/apache/druid/server/coordination/SegmentLoadDropHandlerTest.java index fc97854d39d6..8aa9f3678350 100644 --- a/server/src/test/java/org/apache/druid/server/coordination/SegmentLoadDropHandlerTest.java +++ b/server/src/test/java/org/apache/druid/server/coordination/SegmentLoadDropHandlerTest.java @@ -31,7 +31,6 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.segment.IndexIO; -import org.apache.druid.segment.SegmentLazyLoadFailCallback; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.loading.CacheTestSegmentLoader; import org.apache.druid.segment.loading.SegmentLoaderConfig; From 65f17df837e066379b2ef0f1d4c8465aecf35875 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 12 Jan 2021 22:25:25 +0800 Subject: [PATCH 20/29] modify logs --- .../src/main/java/org/apache/druid/segment/IndexIO.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index 5f792b315dbd..986258e47b92 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -621,7 +621,9 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen try { return deserializeColumn(mapper, timeBuffer, smooshedFiles); } - catch (IOException e) { + catch (IOException | RuntimeException e) { + log.warn(e, "Exception when deserialize Column [%s]", ColumnHolder.TIME_COLUMN_NAME); + loadFailed.execute(); throw Throwables.propagate(e); } } From dd46d753abd1d5417d55566b9d88aea62aa819de Mon Sep 17 00:00:00 2001 From: yuezhang Date: Wed, 13 Jan 2021 00:30:26 +0800 Subject: [PATCH 21/29] modify docs --- .../src/main/java/org/apache/druid/segment/IndexIO.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index 986258e47b92..589a4b6c6072 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -600,7 +600,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen return deserializeColumn(mapper, colBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e, "Exception when deserialize Column [%s].", columnName); + log.warn(e, "Throw Exception when deserialize Column [%s].", columnName); loadFailed.execute(); throw Throwables.propagate(e); } @@ -622,7 +622,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen return deserializeColumn(mapper, timeBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e, "Exception when deserialize Column [%s]", ColumnHolder.TIME_COLUMN_NAME); + log.warn(e, "Throw exception when deserialize Column [%s]", ColumnHolder.TIME_COLUMN_NAME); loadFailed.execute(); throw Throwables.propagate(e); } From 15a4c3e8770acb1a992114d7121de8f6f0b89ed8 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Wed, 13 Jan 2021 03:58:21 +0800 Subject: [PATCH 22/29] modify logs to rerun ci --- .../src/main/java/org/apache/druid/segment/IndexIO.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index 589a4b6c6072..b4682c228062 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -600,7 +600,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen return deserializeColumn(mapper, colBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e, "Throw Exception when deserialize Column [%s].", columnName); + log.warn(e, "Throw Exception when deserialize column [%s].", columnName); loadFailed.execute(); throw Throwables.propagate(e); } @@ -622,7 +622,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen return deserializeColumn(mapper, timeBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e, "Throw exception when deserialize Column [%s]", ColumnHolder.TIME_COLUMN_NAME); + log.warn(e, "Throw exception when deserialize column [%s]", ColumnHolder.TIME_COLUMN_NAME); loadFailed.execute(); throw Throwables.propagate(e); } From 80accce200c0c9bf555f644ee88eafd82225e983 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Wed, 13 Jan 2021 05:23:57 +0800 Subject: [PATCH 23/29] modify logs to rerun ci --- .../src/main/java/org/apache/druid/segment/IndexIO.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index b4682c228062..1ecccc67e538 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -600,7 +600,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen return deserializeColumn(mapper, colBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e, "Throw Exception when deserialize column [%s].", columnName); + log.warn(e, "Throw Exception when deserialize Column [%s].", columnName); loadFailed.execute(); throw Throwables.propagate(e); } @@ -622,7 +622,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen return deserializeColumn(mapper, timeBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e, "Throw exception when deserialize column [%s]", ColumnHolder.TIME_COLUMN_NAME); + log.warn(e, "Throw exception when deserialize olumn [%s]", ColumnHolder.TIME_COLUMN_NAME); loadFailed.execute(); throw Throwables.propagate(e); } From 78f5ff3d836c8e66ee1c2b7313710a8eaf7e8290 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Wed, 13 Jan 2021 11:43:12 +0800 Subject: [PATCH 24/29] modify logs to rerun ci --- .../src/main/java/org/apache/druid/segment/IndexIO.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index 1ecccc67e538..b4682c228062 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -600,7 +600,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen return deserializeColumn(mapper, colBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e, "Throw Exception when deserialize Column [%s].", columnName); + log.warn(e, "Throw Exception when deserialize column [%s].", columnName); loadFailed.execute(); throw Throwables.propagate(e); } @@ -622,7 +622,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen return deserializeColumn(mapper, timeBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e, "Throw exception when deserialize olumn [%s]", ColumnHolder.TIME_COLUMN_NAME); + log.warn(e, "Throw exception when deserialize column [%s]", ColumnHolder.TIME_COLUMN_NAME); loadFailed.execute(); throw Throwables.propagate(e); } From 9dec7bd5130d2c244c83ad8fbf15faafaa7357e3 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Wed, 13 Jan 2021 18:33:23 +0800 Subject: [PATCH 25/29] modify logs to rerun ci --- processing/src/main/java/org/apache/druid/segment/IndexIO.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index b4682c228062..bff25e22eb7a 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -600,7 +600,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen return deserializeColumn(mapper, colBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e, "Throw Exception when deserialize column [%s].", columnName); + log.warn(e, "Throw exception when deserialize column [%s].", columnName); loadFailed.execute(); throw Throwables.propagate(e); } From 6bb59ce5a3ddacffc68cad659a12e5fc9735c599 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 14 Jan 2021 00:23:25 +0800 Subject: [PATCH 26/29] modify logs to rerun ci --- .../src/main/java/org/apache/druid/segment/IndexIO.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index bff25e22eb7a..df43f6374a58 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -600,7 +600,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen return deserializeColumn(mapper, colBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e, "Throw exception when deserialize column [%s].", columnName); + log.warn(e, "Throw exceptions when deserialize column [%s].", columnName); loadFailed.execute(); throw Throwables.propagate(e); } @@ -622,7 +622,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen return deserializeColumn(mapper, timeBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e, "Throw exception when deserialize column [%s]", ColumnHolder.TIME_COLUMN_NAME); + log.warn(e, "Throw exceptions when deserialize column [%s]", ColumnHolder.TIME_COLUMN_NAME); loadFailed.execute(); throw Throwables.propagate(e); } From b5aa6102aac6cb8d6399962c819bebe21666cf1d Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 14 Jan 2021 07:48:39 +0800 Subject: [PATCH 27/29] modify logs to rerun ci --- .travis.yml | 2 +- .../org/apache/druid/segment/IndexIOTest.java | 12 ------------ .../v9SegmentPersistDir/segment/00000.smoosh | Bin 1587 -> 0 bytes .../v9SegmentPersistDir/segment/factory.json | 1 - .../v9SegmentPersistDir/segment/meta.smoosh | 9 --------- .../v9SegmentPersistDir/segment/version.bin | Bin 4 -> 0 bytes 6 files changed, 1 insertion(+), 23 deletions(-) delete mode 100644 processing/src/test/resources/v9SegmentPersistDir/segment/00000.smoosh delete mode 100644 processing/src/test/resources/v9SegmentPersistDir/segment/factory.json delete mode 100644 processing/src/test/resources/v9SegmentPersistDir/segment/meta.smoosh delete mode 100644 processing/src/test/resources/v9SegmentPersistDir/segment/version.bin diff --git a/.travis.yml b/.travis.yml index 2e71beeb71e2..aa03eee01466 100644 --- a/.travis.yml +++ b/.travis.yml @@ -154,7 +154,7 @@ jobs: # Set MAVEN_OPTS for Surefire launcher. Skip remoteresources to avoid intermittent connection timeouts when # resolving the SIGAR dependency. - > - MAVEN_OPTS='-Xmx1400m' ${MVN} test -pl ${MAVEN_PROJECTS} + MAVEN_OPTS='-Xmx1100m' ${MVN} test -pl ${MAVEN_PROJECTS} ${MAVEN_SKIP} -Dremoteresources.skip=true -Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL} - sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0" - free -m diff --git a/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java b/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java index a8c557c75801..b6a9155566be 100644 --- a/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java +++ b/processing/src/test/java/org/apache/druid/segment/IndexIOTest.java @@ -53,7 +53,6 @@ import javax.annotation.Nullable; import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; @@ -343,17 +342,6 @@ public void testRowValidatorEquals() throws Exception } } - @Test - public void testLoadSegmentWithLazy() throws IOException - { - final ObjectMapper mapper = new DefaultObjectMapper(); - final IndexIO indexIO = new IndexIO(mapper, () -> 0); - String path = this.getClass().getClassLoader().getResource("v9SegmentPersistDir/segment/").getPath(); - File inDir = new File(path); - QueryableIndex queryableIndex = indexIO.loadIndex(inDir, true, SegmentLazyLoadFailCallback.NOOP); - Assert.assertNotNull(queryableIndex); - } - @Test public void testLoadSegmentDamagedFileWithLazy() { diff --git a/processing/src/test/resources/v9SegmentPersistDir/segment/00000.smoosh b/processing/src/test/resources/v9SegmentPersistDir/segment/00000.smoosh deleted file mode 100644 index 5bd4e75a27874bba498d052d0109e4408c562914..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1587 zcmdT^O-~y!5Ot7>T2-wOv>Z|8s<$XodqG+*tptUYXo5)KQXz01v$g84`U5qJ7Ug&J zPw*Ez<1K-Z9tx+R)vP_wqT<&XL4>Im~d5{l$16=F{m~<7FW@F&6&#tyECwcT516Jo1Fs2}e$fUtuGX_Dhs`8>@ zpqRPF)K40eWthg^mRW;)E`=I)XEE{1c@YALkrpN?Rkx=f#s)W&XIk`~T(|N2=KFhY z|Hyc=GMDqDO{3HzM5bNe36A%LTa`2vakGLst=B99D1!x8OQ_UTxwBuFOMfQeN4uUu Z^~}o%myhoCjt(K*TcwhtMf6G3e*ox4qNM-; diff --git a/processing/src/test/resources/v9SegmentPersistDir/segment/factory.json b/processing/src/test/resources/v9SegmentPersistDir/segment/factory.json deleted file mode 100644 index f7b2cc330765..000000000000 --- a/processing/src/test/resources/v9SegmentPersistDir/segment/factory.json +++ /dev/null @@ -1 +0,0 @@ -{"type":"mMapSegmentFactory"} \ No newline at end of file diff --git a/processing/src/test/resources/v9SegmentPersistDir/segment/meta.smoosh b/processing/src/test/resources/v9SegmentPersistDir/segment/meta.smoosh deleted file mode 100644 index 70a831feef32..000000000000 --- a/processing/src/test/resources/v9SegmentPersistDir/segment/meta.smoosh +++ /dev/null @@ -1,9 +0,0 @@ -v1,2147483647,1 -__time,0,0,141 -count,0,141,282 -dstIP,0,564,805 -index.drd,0,1046,1205 -metadata.drd,0,1205,1587 -srcIP,0,805,1046 -sum_bytes,0,282,423 -sum_packets,0,423,564 diff --git a/processing/src/test/resources/v9SegmentPersistDir/segment/version.bin b/processing/src/test/resources/v9SegmentPersistDir/segment/version.bin deleted file mode 100644 index 3dd5ace49c65e7fd143d228be939732534095e78..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmZQzVBiD*01W^N From a510bee83be87f85720d87c51ed82755d52b7d25 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 15 Jan 2021 07:56:01 +0800 Subject: [PATCH 28/29] modify logs to rerun ci --- .../src/main/java/org/apache/druid/segment/IndexIO.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index df43f6374a58..bff25e22eb7a 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -600,7 +600,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen return deserializeColumn(mapper, colBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e, "Throw exceptions when deserialize column [%s].", columnName); + log.warn(e, "Throw exception when deserialize column [%s].", columnName); loadFailed.execute(); throw Throwables.propagate(e); } @@ -622,7 +622,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen return deserializeColumn(mapper, timeBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e, "Throw exceptions when deserialize column [%s]", ColumnHolder.TIME_COLUMN_NAME); + log.warn(e, "Throw exception when deserialize column [%s]", ColumnHolder.TIME_COLUMN_NAME); loadFailed.execute(); throw Throwables.propagate(e); } From fef72a7ad29c1a0937b7cc45f8f706c7e546b62a Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 15 Jan 2021 09:50:33 +0800 Subject: [PATCH 29/29] modify logs to rerun ci --- .../src/main/java/org/apache/druid/segment/IndexIO.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/IndexIO.java b/processing/src/main/java/org/apache/druid/segment/IndexIO.java index bff25e22eb7a..df43f6374a58 100644 --- a/processing/src/main/java/org/apache/druid/segment/IndexIO.java +++ b/processing/src/main/java/org/apache/druid/segment/IndexIO.java @@ -600,7 +600,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen return deserializeColumn(mapper, colBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e, "Throw exception when deserialize column [%s].", columnName); + log.warn(e, "Throw exceptions when deserialize column [%s].", columnName); loadFailed.execute(); throw Throwables.propagate(e); } @@ -622,7 +622,7 @@ public QueryableIndex load(File inDir, ObjectMapper mapper, boolean lazy, Segmen return deserializeColumn(mapper, timeBuffer, smooshedFiles); } catch (IOException | RuntimeException e) { - log.warn(e, "Throw exception when deserialize column [%s]", ColumnHolder.TIME_COLUMN_NAME); + log.warn(e, "Throw exceptions when deserialize column [%s]", ColumnHolder.TIME_COLUMN_NAME); loadFailed.execute(); throw Throwables.propagate(e); }