From 179deca77f0cf359e4ecb793f91e4f4c9f98ddf4 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 25 Jun 2020 11:23:59 -0700 Subject: [PATCH 1/4] Move shardSpec tests to core --- .../HashBasedNumberedShardSpecTest.java | 25 +++++++++---------- .../partition}/NumberedShardSpecTest.java | 19 ++++++-------- .../SingleDimensionShardSpecTest.java | 8 +----- 3 files changed, 20 insertions(+), 32 deletions(-) rename {server => core}/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java (92%) rename {server/src/test/java/org/apache/druid/server/shard => core/src/test/java/org/apache/druid/timeline/partition}/NumberedShardSpecTest.java (92%) rename {server/src/test/java/org/apache/druid/server/shard => core/src/test/java/org/apache/druid/timeline/partition}/SingleDimensionShardSpecTest.java (93%) diff --git a/server/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java similarity index 92% rename from server/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java rename to core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java index be24fe5c25a8..da6d600e87ca 100644 --- a/server/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java +++ b/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java @@ -30,7 +30,6 @@ import org.apache.druid.data.input.Row; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.ISE; -import org.apache.druid.server.ServerTestHelper; import org.joda.time.DateTime; import org.junit.Assert; import org.junit.Test; @@ -43,6 +42,7 @@ public class HashBasedNumberedShardSpecTest { + private final ObjectMapper objectMapper = ShardSpecTestUtils.initObjectMapper(); @Test public void testEquals() { @@ -56,16 +56,15 @@ public void testEquals() @Test public void testSerdeRoundTrip() throws Exception { - - final ShardSpec spec = ServerTestHelper.MAPPER.readValue( - ServerTestHelper.MAPPER.writeValueAsBytes( + final ShardSpec spec = objectMapper.readValue( + objectMapper.writeValueAsBytes( new HashBasedNumberedShardSpec( 1, 2, 1, 3, ImmutableList.of("visitor_id"), - ServerTestHelper.MAPPER + objectMapper ) ), ShardSpec.class @@ -80,14 +79,14 @@ public void testSerdeRoundTrip() throws Exception @Test public void testSerdeBackwardsCompat() throws Exception { - final ShardSpec spec = ServerTestHelper.MAPPER.readValue( + final ShardSpec spec = objectMapper.readValue( "{\"type\": \"hashed\", \"partitions\": 2, \"partitionNum\": 1}", ShardSpec.class ); Assert.assertEquals(1, spec.getPartitionNum()); Assert.assertEquals(2, spec.getNumCorePartitions()); - final ShardSpec specWithPartitionDimensions = ServerTestHelper.MAPPER.readValue( + final ShardSpec specWithPartitionDimensions = objectMapper.readValue( "{\"type\": \"hashed\", \"partitions\": 2, \"partitionNum\": 1, \"partitionDimensions\":[\"visitor_id\"]}", ShardSpec.class ); @@ -104,9 +103,9 @@ public void testSerdeBackwardsCompat() throws Exception public void testPartitionChunks() { final List specs = ImmutableList.of( - new HashBasedNumberedShardSpec(0, 3, 0, 3, null, ServerTestHelper.MAPPER), - new HashBasedNumberedShardSpec(1, 3, 1, 3, null, ServerTestHelper.MAPPER), - new HashBasedNumberedShardSpec(2, 3, 2, 3, null, ServerTestHelper.MAPPER) + new HashBasedNumberedShardSpec(0, 3, 0, 3, null, objectMapper), + new HashBasedNumberedShardSpec(1, 3, 1, 3, null, objectMapper), + new HashBasedNumberedShardSpec(2, 3, 2, 3, null, objectMapper) ); final List> chunks = Lists.transform( @@ -208,7 +207,7 @@ public void testSharePartitionSpace() 1, 3, ImmutableList.of("visitor_id"), - ServerTestHelper.MAPPER + objectMapper ); Assert.assertTrue(shardSpec.sharePartitionSpace(NumberedPartialShardSpec.instance())); Assert.assertTrue(shardSpec.sharePartitionSpace(new HashBasedNumberedPartialShardSpec(null, 0, 1))); @@ -226,11 +225,11 @@ public boolean assertExistsInOneSpec(List specs, InputRow row) throw new ISE("None of the partition matches"); } - public static class HashOverridenShardSpec extends HashBasedNumberedShardSpec + public class HashOverridenShardSpec extends HashBasedNumberedShardSpec { public HashOverridenShardSpec(int partitionNum, int partitions) { - super(partitionNum, partitions, partitionNum, partitions, null, ServerTestHelper.MAPPER); + super(partitionNum, partitions, partitionNum, partitions, null, objectMapper); } @Override diff --git a/server/src/test/java/org/apache/druid/server/shard/NumberedShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/NumberedShardSpecTest.java similarity index 92% rename from server/src/test/java/org/apache/druid/server/shard/NumberedShardSpecTest.java rename to core/src/test/java/org/apache/druid/timeline/partition/NumberedShardSpecTest.java index 04c0c553bcf9..8b8d5da1b2c2 100644 --- a/server/src/test/java/org/apache/druid/server/shard/NumberedShardSpecTest.java +++ b/core/src/test/java/org/apache/druid/timeline/partition/NumberedShardSpecTest.java @@ -17,8 +17,9 @@ * under the License. */ -package org.apache.druid.server.shard; +package org.apache.druid.timeline.partition; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -26,17 +27,9 @@ import com.google.common.collect.Ordering; import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.server.ServerTestHelper; import org.apache.druid.timeline.Overshadowable; import org.apache.druid.timeline.TimelineObjectHolder; import org.apache.druid.timeline.VersionedIntervalTimeline; -import org.apache.druid.timeline.partition.HashBasedNumberedPartialShardSpec; -import org.apache.druid.timeline.partition.NumberedOverwritePartialShardSpec; -import org.apache.druid.timeline.partition.NumberedPartialShardSpec; -import org.apache.druid.timeline.partition.NumberedShardSpec; -import org.apache.druid.timeline.partition.PartitionChunk; -import org.apache.druid.timeline.partition.ShardSpec; -import org.apache.druid.timeline.partition.SingleDimensionPartialShardSpec; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Test; @@ -58,8 +51,9 @@ public void testEquals() @Test public void testSerdeRoundTrip() throws Exception { - final ShardSpec spec = ServerTestHelper.MAPPER.readValue( - ServerTestHelper.MAPPER.writeValueAsBytes(new NumberedShardSpec(1, 2)), + final ObjectMapper objectMapper = ShardSpecTestUtils.initObjectMapper(); + final ShardSpec spec = objectMapper.readValue( + objectMapper.writeValueAsBytes(new NumberedShardSpec(1, 2)), ShardSpec.class ); Assert.assertEquals(1, spec.getPartitionNum()); @@ -69,7 +63,8 @@ public void testSerdeRoundTrip() throws Exception @Test public void testSerdeBackwardsCompat() throws Exception { - final ShardSpec spec = ServerTestHelper.MAPPER.readValue( + final ObjectMapper objectMapper = ShardSpecTestUtils.initObjectMapper(); + final ShardSpec spec = objectMapper.readValue( "{\"type\": \"numbered\", \"partitions\": 2, \"partitionNum\": 1}", ShardSpec.class ); diff --git a/server/src/test/java/org/apache/druid/server/shard/SingleDimensionShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/SingleDimensionShardSpecTest.java similarity index 93% rename from server/src/test/java/org/apache/druid/server/shard/SingleDimensionShardSpecTest.java rename to core/src/test/java/org/apache/druid/timeline/partition/SingleDimensionShardSpecTest.java index 6928dfa735e3..a8f1aa9d33ef 100644 --- a/server/src/test/java/org/apache/druid/server/shard/SingleDimensionShardSpecTest.java +++ b/core/src/test/java/org/apache/druid/timeline/partition/SingleDimensionShardSpecTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.server.shard; +package org.apache.druid.timeline.partition; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -30,12 +30,6 @@ import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.timeline.partition.HashBasedNumberedPartialShardSpec; -import org.apache.druid.timeline.partition.NumberedOverwritePartialShardSpec; -import org.apache.druid.timeline.partition.NumberedPartialShardSpec; -import org.apache.druid.timeline.partition.ShardSpec; -import org.apache.druid.timeline.partition.SingleDimensionPartialShardSpec; -import org.apache.druid.timeline.partition.SingleDimensionShardSpec; import org.junit.Assert; import org.junit.Test; From 5f14e17bc3836fc8b4f63dceb9dd7caba4b2f90c Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 25 Jun 2020 23:36:10 -0700 Subject: [PATCH 2/4] checkstyle --- .../druid/timeline/partition/HashBasedNumberedShardSpecTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java index da6d600e87ca..624f3d34f169 100644 --- a/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java +++ b/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java @@ -43,6 +43,7 @@ public class HashBasedNumberedShardSpecTest { private final ObjectMapper objectMapper = ShardSpecTestUtils.initObjectMapper(); + @Test public void testEquals() { From 330470beef48ad01e34bdd0c7fa95f84c6709397 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Fri, 26 Jun 2020 09:32:15 -0700 Subject: [PATCH 3/4] inject object mapper for testing --- .../timeline/partition/HashBasedNumberedShardSpecTest.java | 1 + .../org/apache/druid/timeline/partition/ShardSpecTestUtils.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java index 624f3d34f169..e7b8caff151d 100644 --- a/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java +++ b/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java @@ -19,6 +19,7 @@ package org.apache.druid.timeline.partition; +import com.fasterxml.jackson.databind.InjectableValues.Std; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.collect.ImmutableList; diff --git a/core/src/test/java/org/apache/druid/timeline/partition/ShardSpecTestUtils.java b/core/src/test/java/org/apache/druid/timeline/partition/ShardSpecTestUtils.java index 2f153651513b..dd613afbc762 100644 --- a/core/src/test/java/org/apache/druid/timeline/partition/ShardSpecTestUtils.java +++ b/core/src/test/java/org/apache/druid/timeline/partition/ShardSpecTestUtils.java @@ -20,6 +20,7 @@ package org.apache.druid.timeline.partition; import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.InjectableValues.Std; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; @@ -30,6 +31,7 @@ public static ObjectMapper initObjectMapper() { // Copied configurations from org.apache.druid.jackson.DefaultObjectMapper final ObjectMapper mapper = new ObjectMapper(); + mapper.setInjectableValues(new Std().addValue(ObjectMapper.class, mapper)); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); mapper.configure(MapperFeature.AUTO_DETECT_GETTERS, false); // See https://github.com/FasterXML/jackson-databind/issues/170 From 6fea79a878ac14430717362461ab20f8ca3bde73 Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Fri, 26 Jun 2020 10:46:07 -0700 Subject: [PATCH 4/4] unused import --- .../druid/timeline/partition/HashBasedNumberedShardSpecTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java index e7b8caff151d..624f3d34f169 100644 --- a/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java +++ b/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java @@ -19,7 +19,6 @@ package org.apache.druid.timeline.partition; -import com.fasterxml.jackson.databind.InjectableValues.Std; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.collect.ImmutableList;