From ce48a5dda4ee269dd560b75decab3b5e83e8a131 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Wed, 28 Jun 2017 14:28:55 +0900 Subject: [PATCH 1/3] Reduce number of tasks in ITUnionQueryTest (#4476) --- .../druid/tests/indexer/ITUnionQueryTest.java | 2 +- .../test/resources/indexer/union_queries.json | 128 +++++++++--------- 2 files changed, 65 insertions(+), 65 deletions(-) diff --git a/integration-tests/src/test/java/io/druid/tests/indexer/ITUnionQueryTest.java b/integration-tests/src/test/java/io/druid/tests/indexer/ITUnionQueryTest.java index dbdc49f01074..f83b2f44179f 100644 --- a/integration-tests/src/test/java/io/druid/tests/indexer/ITUnionQueryTest.java +++ b/integration-tests/src/test/java/io/druid/tests/indexer/ITUnionQueryTest.java @@ -63,7 +63,7 @@ public class ITUnionQueryTest extends AbstractIndexerTest @Test public void testUnionQuery() throws Exception { - final int numTasks = 4; + final int numTasks = 3; try { // Load 4 datasources with same dimensions diff --git a/integration-tests/src/test/resources/indexer/union_queries.json b/integration-tests/src/test/resources/indexer/union_queries.json index 08b7776e7ab9..ec13d507d42f 100644 --- a/integration-tests/src/test/resources/indexer/union_queries.json +++ b/integration-tests/src/test/resources/indexer/union_queries.json @@ -53,11 +53,11 @@ { "timestamp": "2013-08-31T01:02:33.000Z", "result": { - "added": 2064.0, - "count": 8, - "delta": 748.0, - "deleted": 1316.0, - "rows": 8 + "added": 1548.0, + "count": 6, + "delta": 561.0, + "deleted": 987.0, + "rows": 6 } } ] @@ -115,28 +115,28 @@ "timestamp": "2013-08-31T01:02:33.000Z", "result": [ { - "added": 3620.0, - "count": 4, + "added": 2715.0, + "count": 3, "page": "Crimson Typhoon", - "delta": 3600.0, - "deleted": 20.0, - "rows": 4 + "delta": 2700.0, + "deleted": 15.0, + "rows": 3 }, { - "added": 1836.0, - "count": 4, + "added": 1377.0, + "count": 3, "page": "Striker Eureka", - "delta": 1320.0, - "deleted": 516.0, - "rows": 4 + "delta": 990.0, + "deleted": 387.0, + "rows": 3 }, { - "added": 492.0, - "count": 4, + "added": 369.0, + "count": 3, "page": "Cherno Alpha", - "delta": 444.0, - "deleted": 48.0, - "rows": 4 + "delta": 333.0, + "deleted": 36.0, + "rows": 3 } ] } @@ -226,31 +226,31 @@ "timestamp": "2013-08-31T01:02:33.000Z", "result": [ { - "added": 3620.0, - "count": 4, + "added": 2715.0, + "count": 3, "page": "Crimson Typhoon", - "delta": 3600.0, - "deleted": 20.0, - "sumOfAddedDeletedConst": 4640.0, - "rows": 4 + "delta": 2700.0, + "deleted": 15.0, + "sumOfAddedDeletedConst": 3730.0, + "rows": 3 }, { - "added": 1836.0, - "count": 4, + "added": 1377.0, + "count": 3, "page": "Striker Eureka", - "delta": 1320.0, - "deleted": 516.0, - "sumOfAddedDeletedConst": 3352.0, - "rows": 4 + "delta": 990.0, + "deleted": 387.0, + "sumOfAddedDeletedConst": 2764.0, + "rows": 3 }, { - "added": 492.0, - "count": 4, + "added": 369.0, + "count": 3, "page": "Cherno Alpha", - "delta": 444.0, - "deleted": 48.0, - "sumOfAddedDeletedConst": 1540.0, - "rows": 4 + "delta": 333.0, + "deleted": 36.0, + "sumOfAddedDeletedConst": 1405.0, + "rows": 3 } ] } @@ -316,22 +316,22 @@ "timestamp": "2013-08-31T01:02:33.000Z", "result": [ { - "sumOfRowsAndCount": 16.0, - "count": 8, + "sumOfRowsAndCount": 12.0, + "count": 6, "language": "en", - "rows": 8 + "rows": 6 }, { - "sumOfRowsAndCount": 8.0, - "count": 4, + "sumOfRowsAndCount": 6.0, + "count": 3, "language": "ja", - "rows": 4 + "rows": 3 }, { - "sumOfRowsAndCount": 8.0, - "count": 4, + "sumOfRowsAndCount": 6.0, + "count": 3, "language": "ru", - "rows": 4 + "rows": 3 } ] } @@ -392,9 +392,9 @@ "version": "v1", "timestamp": "2013-08-31T00:00:00.000Z", "event": { - "sumOfRowsAndCount": 16.0, - "count": 8, - "rows": 8, + "sumOfRowsAndCount": 12.0, + "count": 6, + "rows": 6, "namespace": "article" } }, @@ -402,9 +402,9 @@ "version": "v1", "timestamp": "2013-08-31T00:00:00.000Z", "event": { - "sumOfRowsAndCount": 24.0, - "count": 12, - "rows": 12, + "sumOfRowsAndCount": 18.0, + "count": 9, + "rows": 9, "namespace": "wikipedia" } } @@ -470,10 +470,10 @@ "version": "v1", "timestamp": "2013-08-31T00:00:00.000Z", "event": { - "sumOfRowsAndCount": 8.0, - "count": 4, + "sumOfRowsAndCount": 6.0, + "count": 3, "robot": "false", - "rows": 4, + "rows": 3, "namespace": "article" } }, @@ -481,10 +481,10 @@ "version": "v1", "timestamp": "2013-08-31T00:00:00.000Z", "event": { - "sumOfRowsAndCount": 8.0, - "count": 4, + "sumOfRowsAndCount": 6.0, + "count": 3, "robot": "true", - "rows": 4, + "rows": 3, "namespace": "article" } }, @@ -492,10 +492,10 @@ "version": "v1", "timestamp": "2013-08-31T00:00:00.000Z", "event": { - "sumOfRowsAndCount": 24.0, - "count": 12, + "sumOfRowsAndCount": 18.0, + "count": 9, "robot": "true", - "rows": 12, + "rows": 9, "namespace": "wikipedia" } } @@ -530,12 +530,12 @@ { "dimension": "user", "value": "triplets", - "count":4 + "count":3 }, { "dimension": "namespace", "value": "wikipedia", - "count":12 + "count":9 } ] } From 4ced4e75c3b09d6fbdcf4b39ee629e891030e6c9 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 27 Jun 2017 00:15:28 +0900 Subject: [PATCH 2/3] Make KafkaLookupExtractorFactoryTest fast (#4466) * Make KafkaLookupExtractorFactoryTest fast * Use list * Use Bytes --- .../KafkaLookupExtractorFactoryTest.java | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/extensions-core/kafka-extraction-namespace/src/test/java/io/druid/query/lookup/KafkaLookupExtractorFactoryTest.java b/extensions-core/kafka-extraction-namespace/src/test/java/io/druid/query/lookup/KafkaLookupExtractorFactoryTest.java index e2fe7d6e2676..13d7f191651c 100644 --- a/extensions-core/kafka-extraction-namespace/src/test/java/io/druid/query/lookup/KafkaLookupExtractorFactoryTest.java +++ b/extensions-core/kafka-extraction-namespace/src/test/java/io/druid/query/lookup/KafkaLookupExtractorFactoryTest.java @@ -26,6 +26,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.primitives.Bytes; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.StringUtils; import io.druid.server.lookup.namespace.cache.CacheHandler; @@ -47,11 +48,12 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -129,18 +131,15 @@ public void testCacheKeyScramblesOnNewData() TOPIC, DEFAULT_PROPERTIES ); - factory.getMapRef().set(ImmutableMap.of()); + factory.getMapRef().set(ImmutableMap.of()); final AtomicLong events = factory.getDoubleEventCount(); final LookupExtractor extractor = factory.get(); - final List byteArrays = new ArrayList<>(n); + final Set> byteArrays = new HashSet<>(n); for (int i = 0; i < n; ++i) { - final byte[] myKey = extractor.getCacheKey(); - // Not terribly efficient.. but who cares - for (byte[] byteArray : byteArrays) { - Assert.assertFalse(Arrays.equals(byteArray, myKey)); - } + final List myKey = Bytes.asList(extractor.getCacheKey()); + Assert.assertFalse(byteArrays.contains(myKey)); byteArrays.add(myKey); events.incrementAndGet(); } @@ -156,17 +155,14 @@ public void testCacheKeyScramblesDifferentStarts() TOPIC, DEFAULT_PROPERTIES ); - factory.getMapRef().set(ImmutableMap.of()); + factory.getMapRef().set(ImmutableMap.of()); final AtomicLong events = factory.getDoubleEventCount(); - final List byteArrays = new ArrayList<>(n); + final Set> byteArrays = new HashSet<>(n); for (int i = 0; i < n; ++i) { final LookupExtractor extractor = factory.get(); - final byte[] myKey = extractor.getCacheKey(); - // Not terribly efficient.. but who cares - for (byte[] byteArray : byteArrays) { - Assert.assertFalse(Arrays.equals(byteArray, myKey)); - } + final List myKey = Bytes.asList(extractor.getCacheKey()); + Assert.assertFalse(byteArrays.contains(myKey)); byteArrays.add(myKey); events.incrementAndGet(); } From 0c5dad29df4151a4df8c64bdc8c931d826ec4026 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Tue, 27 Jun 2017 10:51:48 +0900 Subject: [PATCH 3/3] Split travis test (#4468) --- .travis.yml | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 54fa4bcd8964..908c9b78a119 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,16 +9,23 @@ cache: matrix: include: + # strict compilation - sudo: false install: true - script: - # The script execution consists of 4 steps: - # 1) Increase the jvm max heap size. Strict compilation in 2) requires more than 2 GB. - # 2) Strict compilation using error-prone - # 3) Reset the maven options. Parallel testing in 4) launches 2 processes, each requires 1 GB of memory. Increasing the maximum memory may cause OutOfMemory error because the instance of Travis has 4 GB of memory space. - # 4) Parallel unit testing - # Using && instead of independent script steps to make Travis build fail faster, if each step is not successful - - echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn clean -Pstrict -pl '!benchmarks' compile test-compile -B && rm ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2 + # Strict compilation requires more than 2 GB + script: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn clean -Pstrict -pl '!benchmarks' compile test-compile -B + + # processing module test + - sudo: false + install: mvn install -q -ff -DskipTests -B + script: mvn test -B -Pparallel-test -Dmaven.fork.count=2 -pl processing + + # non-processing modules test + - sudo: false + install: mvn install -q -ff -DskipTests -B + script: mvn test -B -Pparallel-test -Dmaven.fork.count=2 -pl '!processing' + + # run integration tests - sudo: required services: - docker @@ -26,6 +33,6 @@ matrix: - DOCKER_IP=172.17.0.1 install: # Only errors will be shown with the -q option. This is to avoid generating too many logs which make travis build failed. - - mvn clean install -q -ff -DskipTests -B + - mvn install -q -ff -DskipTests -B script: - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh