Skip to content

[Backport] Unit test backports (#4466, #4468, #4476) #4569

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jon-wei wants to merge 3 commits into from
Closed

[Backport] Unit test backports (#4466, #4468, #4476) #4569

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ce48a5d
Reduce number of tasks in ITUnionQueryTest (#4476)
jihoonson Jun 28, 2017
4ced4e7
Make KafkaLookupExtractorFactoryTest fast (#4466)
jihoonson Jun 26, 2017
0c5dad2
Split travis test (#4468)
jihoonson Jun 27, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 16 additions & 9 deletions .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,30 @@ cache:

matrix:
include:
# strict compilation
- sudo: false
install: true
script:
# The script execution consists of 4 steps:
# 1) Increase the jvm max heap size. Strict compilation in 2) requires more than 2 GB.
# 2) Strict compilation using error-prone
# 3) Reset the maven options. Parallel testing in 4) launches 2 processes, each requires 1 GB of memory. Increasing the maximum memory may cause OutOfMemory error because the instance of Travis has 4 GB of memory space.
# 4) Parallel unit testing
# Using && instead of independent script steps to make Travis build fail faster, if each step is not successful
- echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn clean -Pstrict -pl '!benchmarks' compile test-compile -B && rm ~/.mavenrc && mvn test -B -Pparallel-test -Dmaven.fork.count=2
# Strict compilation requires more than 2 GB
script: echo "MAVEN_OPTS='-Xmx3000m'" > ~/.mavenrc && mvn clean -Pstrict -pl '!benchmarks' compile test-compile -B

# processing module test
- sudo: false
install: mvn install -q -ff -DskipTests -B
script: mvn test -B -Pparallel-test -Dmaven.fork.count=2 -pl processing

# non-processing modules test
- sudo: false
install: mvn install -q -ff -DskipTests -B
script: mvn test -B -Pparallel-test -Dmaven.fork.count=2 -pl '!processing'

# run integration tests
- sudo: required
services:
- docker
env:
- DOCKER_IP=172.17.0.1
install:
# Only errors will be shown with the -q option. This is to avoid generating too many logs which make travis build failed.
- mvn clean install -q -ff -DskipTests -B
- mvn install -q -ff -DskipTests -B
script:
- $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh
26 changes: 11 additions & 15 deletions ...action-namespace/src/test/java/io/druid/query/lookup/KafkaLookupExtractorFactoryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.primitives.Bytes;
import io.druid.jackson.DefaultObjectMapper;
import io.druid.java.util.common.StringUtils;
import io.druid.server.lookup.namespace.cache.CacheHandler;
Expand All @@ -47,11 +48,12 @@
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
Expand Down Expand Up @@ -129,18 +131,15 @@ public void testCacheKeyScramblesOnNewData()
TOPIC,
DEFAULT_PROPERTIES
);
factory.getMapRef().set(ImmutableMap.<String, String>of());
factory.getMapRef().set(ImmutableMap.of());
final AtomicLong events = factory.getDoubleEventCount();

final LookupExtractor extractor = factory.get();

final List<byte[]> byteArrays = new ArrayList<>(n);
final Set<List<Byte>> byteArrays = new HashSet<>(n);
for (int i = 0; i < n; ++i) {
final byte[] myKey = extractor.getCacheKey();
// Not terribly efficient.. but who cares
for (byte[] byteArray : byteArrays) {
Assert.assertFalse(Arrays.equals(byteArray, myKey));
}
final List<Byte> myKey = Bytes.asList(extractor.getCacheKey());
Assert.assertFalse(byteArrays.contains(myKey));
byteArrays.add(myKey);
events.incrementAndGet();
}
Expand All @@ -156,17 +155,14 @@ public void testCacheKeyScramblesDifferentStarts()
TOPIC,
DEFAULT_PROPERTIES
);
factory.getMapRef().set(ImmutableMap.<String, String>of());
factory.getMapRef().set(ImmutableMap.of());
final AtomicLong events = factory.getDoubleEventCount();

final List<byte[]> byteArrays = new ArrayList<>(n);
final Set<List<Byte>> byteArrays = new HashSet<>(n);
for (int i = 0; i < n; ++i) {
final LookupExtractor extractor = factory.get();
final byte[] myKey = extractor.getCacheKey();
// Not terribly efficient.. but who cares
for (byte[] byteArray : byteArrays) {
Assert.assertFalse(Arrays.equals(byteArray, myKey));
}
final List<Byte> myKey = Bytes.asList(extractor.getCacheKey());
Assert.assertFalse(byteArrays.contains(myKey));
byteArrays.add(myKey);
events.incrementAndGet();
}
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/src/test/java/io/druid/tests/indexer/ITUnionQueryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public class ITUnionQueryTest extends AbstractIndexerTest
@Test
public void testUnionQuery() throws Exception
{
final int numTasks = 4;
final int numTasks = 3;

try {
// Load 4 datasources with same dimensions
Expand Down
128 changes: 64 additions & 64 deletions integration-tests/src/test/resources/indexer/union_queries.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,11 @@
{
"timestamp": "2013-08-31T01:02:33.000Z",
"result": {
"added": 2064.0,
"count": 8,
"delta": 748.0,
"deleted": 1316.0,
"rows": 8
"added": 1548.0,
"count": 6,
"delta": 561.0,
"deleted": 987.0,
"rows": 6
}
}
]
Expand Down Expand Up @@ -115,28 +115,28 @@
"timestamp": "2013-08-31T01:02:33.000Z",
"result": [
{
"added": 3620.0,
"count": 4,
"added": 2715.0,
"count": 3,
"page": "Crimson Typhoon",
"delta": 3600.0,
"deleted": 20.0,
"rows": 4
"delta": 2700.0,
"deleted": 15.0,
"rows": 3
},
{
"added": 1836.0,
"count": 4,
"added": 1377.0,
"count": 3,
"page": "Striker Eureka",
"delta": 1320.0,
"deleted": 516.0,
"rows": 4
"delta": 990.0,
"deleted": 387.0,
"rows": 3
},
{
"added": 492.0,
"count": 4,
"added": 369.0,
"count": 3,
"page": "Cherno Alpha",
"delta": 444.0,
"deleted": 48.0,
"rows": 4
"delta": 333.0,
"deleted": 36.0,
"rows": 3
}
]
}
Expand Down Expand Up @@ -226,31 +226,31 @@
"timestamp": "2013-08-31T01:02:33.000Z",
"result": [
{
"added": 3620.0,
"count": 4,
"added": 2715.0,
"count": 3,
"page": "Crimson Typhoon",
"delta": 3600.0,
"deleted": 20.0,
"sumOfAddedDeletedConst": 4640.0,
"rows": 4
"delta": 2700.0,
"deleted": 15.0,
"sumOfAddedDeletedConst": 3730.0,
"rows": 3
},
{
"added": 1836.0,
"count": 4,
"added": 1377.0,
"count": 3,
"page": "Striker Eureka",
"delta": 1320.0,
"deleted": 516.0,
"sumOfAddedDeletedConst": 3352.0,
"rows": 4
"delta": 990.0,
"deleted": 387.0,
"sumOfAddedDeletedConst": 2764.0,
"rows": 3
},
{
"added": 492.0,
"count": 4,
"added": 369.0,
"count": 3,
"page": "Cherno Alpha",
"delta": 444.0,
"deleted": 48.0,
"sumOfAddedDeletedConst": 1540.0,
"rows": 4
"delta": 333.0,
"deleted": 36.0,
"sumOfAddedDeletedConst": 1405.0,
"rows": 3
}
]
}
Expand Down Expand Up @@ -316,22 +316,22 @@
"timestamp": "2013-08-31T01:02:33.000Z",
"result": [
{
"sumOfRowsAndCount": 16.0,
"count": 8,
"sumOfRowsAndCount": 12.0,
"count": 6,
"language": "en",
"rows": 8
"rows": 6
},
{
"sumOfRowsAndCount": 8.0,
"count": 4,
"sumOfRowsAndCount": 6.0,
"count": 3,
"language": "ja",
"rows": 4
"rows": 3
},
{
"sumOfRowsAndCount": 8.0,
"count": 4,
"sumOfRowsAndCount": 6.0,
"count": 3,
"language": "ru",
"rows": 4
"rows": 3
}
]
}
Expand Down Expand Up @@ -392,19 +392,19 @@
"version": "v1",
"timestamp": "2013-08-31T00:00:00.000Z",
"event": {
"sumOfRowsAndCount": 16.0,
"count": 8,
"rows": 8,
"sumOfRowsAndCount": 12.0,
"count": 6,
"rows": 6,
"namespace": "article"
}
},
{
"version": "v1",
"timestamp": "2013-08-31T00:00:00.000Z",
"event": {
"sumOfRowsAndCount": 24.0,
"count": 12,
"rows": 12,
"sumOfRowsAndCount": 18.0,
"count": 9,
"rows": 9,
"namespace": "wikipedia"
}
}
Expand Down Expand Up @@ -470,32 +470,32 @@
"version": "v1",
"timestamp": "2013-08-31T00:00:00.000Z",
"event": {
"sumOfRowsAndCount": 8.0,
"count": 4,
"sumOfRowsAndCount": 6.0,
"count": 3,
"robot": "false",
"rows": 4,
"rows": 3,
"namespace": "article"
}
},
{
"version": "v1",
"timestamp": "2013-08-31T00:00:00.000Z",
"event": {
"sumOfRowsAndCount": 8.0,
"count": 4,
"sumOfRowsAndCount": 6.0,
"count": 3,
"robot": "true",
"rows": 4,
"rows": 3,
"namespace": "article"
}
},
{
"version": "v1",
"timestamp": "2013-08-31T00:00:00.000Z",
"event": {
"sumOfRowsAndCount": 24.0,
"count": 12,
"sumOfRowsAndCount": 18.0,
"count": 9,
"robot": "true",
"rows": 12,
"rows": 9,
"namespace": "wikipedia"
}
}
Expand Down Expand Up @@ -530,12 +530,12 @@
{
"dimension": "user",
"value": "triplets",
"count":4
"count":3
},
{
"dimension": "namespace",
"value": "wikipedia",
"count":12
"count":9
}
]
}
Expand Down