From 1626b4e8392520cf796964c9b8b1b9832745bd88 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Tue, 30 Jul 2024 07:51:07 -0700 Subject: [PATCH 01/14] Add multiValueHandlingMode config. For users that aren't able to migrate to arrays yet, this system-wide global property will influence what default mode to use for multi value handling. Note that this addresses the outstanding issue with compaction that always defautls to SORTED_ARRAY even if the data is ingested as a SORTED_SET or ARRAY. --- .../data/input/impl/DimensionSchema.java | 4 +- .../segment/DefaultColumnFormatConfig.java | 47 +++++++++++++++++-- .../segment/MultiValueConfigHandling.java | 43 +++++++++++++++++ .../DefaultColumnFormatsConfigTest.java | 4 +- .../segment/NestedDataColumnSchemaTest.java | 4 +- .../apache/druid/guice/StorageNodeModule.java | 2 + .../org/apache/druid/cli/DumpSegmentTest.java | 8 ++-- .../sql/calcite/util/SqlTestFramework.java | 2 +- 8 files changed, 99 insertions(+), 15 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/segment/MultiValueConfigHandling.java diff --git a/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java b/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java index a65ddb7b9aec..28ca890f094d 100644 --- a/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java +++ b/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java @@ -32,6 +32,7 @@ import org.apache.druid.segment.AutoTypeColumnSchema; import org.apache.druid.segment.DimensionHandler; import org.apache.druid.segment.DimensionHandlerUtils; +import org.apache.druid.segment.MultiValueConfigHandling; import org.apache.druid.segment.NestedDataColumnSchema; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.TypeSignature; @@ -110,10 +111,9 @@ public static MultiValueHandling fromString(String name) return name == null ? ofDefault() : valueOf(StringUtils.toUpperCase(name)); } - // this can be system configuration public static MultiValueHandling ofDefault() { - return SORTED_ARRAY; + return MultiValueConfigHandling.getDefaultMultiValueHandlingMode(); } } diff --git a/processing/src/main/java/org/apache/druid/segment/DefaultColumnFormatConfig.java b/processing/src/main/java/org/apache/druid/segment/DefaultColumnFormatConfig.java index 3f41fb88c032..a643594f3beb 100644 --- a/processing/src/main/java/org/apache/druid/segment/DefaultColumnFormatConfig.java +++ b/processing/src/main/java/org/apache/druid/segment/DefaultColumnFormatConfig.java @@ -21,9 +21,11 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.data.input.impl.DimensionSchema; import org.apache.druid.error.DruidException; import javax.annotation.Nullable; +import java.util.Arrays; import java.util.Objects; public class DefaultColumnFormatConfig @@ -39,16 +41,45 @@ public static void validateNestedFormatVersion(@Nullable Integer formatVersion) } } + private static void validateMultiValueHandlingMode(@Nullable String multiValueHandlingMode) + { + if (multiValueHandlingMode != null) { + try { + DimensionSchema.MultiValueHandling.fromString(multiValueHandlingMode); + } + catch (IllegalArgumentException e) { + throw DruidException.forPersona(DruidException.Persona.OPERATOR) + .ofCategory(DruidException.Category.INVALID_INPUT) + .build( + "Invalid value[%s] specified for 'druid.indexing.formats.multiValueHandlingMode'." + + " Supported values are [%s].", + multiValueHandlingMode, + Arrays.toString(DimensionSchema.MultiValueHandling.values()) + ); + } + } + } + + @Nullable @JsonProperty("nestedColumnFormatVersion") private final Integer nestedColumnFormatVersion; + @JsonProperty("multiValueHandlingMode") + private final DimensionSchema.MultiValueHandling multiValueHandlingMode; + @JsonCreator public DefaultColumnFormatConfig( - @JsonProperty("nestedColumnFormatVersion") @Nullable Integer nestedColumnFormatVersion + @JsonProperty("nestedColumnFormatVersion") @Nullable Integer nestedColumnFormatVersion, + @JsonProperty("multiValueHandlingMode") @Nullable String multiValueHandlingMode ) { + validateNestedFormatVersion(nestedColumnFormatVersion); + validateMultiValueHandlingMode(multiValueHandlingMode); + this.nestedColumnFormatVersion = nestedColumnFormatVersion; - validateNestedFormatVersion(this.nestedColumnFormatVersion); + this.multiValueHandlingMode = multiValueHandlingMode == null ? + DimensionSchema.MultiValueHandling.SORTED_ARRAY : + DimensionSchema.MultiValueHandling.fromString(multiValueHandlingMode); } @Nullable @@ -58,6 +89,12 @@ public Integer getNestedColumnFormatVersion() return nestedColumnFormatVersion; } + @JsonProperty("multiValueHandlingMode") + public DimensionSchema.MultiValueHandling getMultiValueHandlingMode() + { + return multiValueHandlingMode; + } + @Override public boolean equals(Object o) { @@ -68,13 +105,14 @@ public boolean equals(Object o) return false; } DefaultColumnFormatConfig that = (DefaultColumnFormatConfig) o; - return Objects.equals(nestedColumnFormatVersion, that.nestedColumnFormatVersion); + return Objects.equals(nestedColumnFormatVersion, that.nestedColumnFormatVersion) + && multiValueHandlingMode == that.multiValueHandlingMode; } @Override public int hashCode() { - return Objects.hash(nestedColumnFormatVersion); + return Objects.hash(nestedColumnFormatVersion, multiValueHandlingMode); } @Override @@ -82,6 +120,7 @@ public String toString() { return "DefaultColumnFormatConfig{" + "nestedColumnFormatVersion=" + nestedColumnFormatVersion + + ", multiValueHandlingMode=" + multiValueHandlingMode + '}'; } } diff --git a/processing/src/main/java/org/apache/druid/segment/MultiValueConfigHandling.java b/processing/src/main/java/org/apache/druid/segment/MultiValueConfigHandling.java new file mode 100644 index 000000000000..e51e63350556 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/MultiValueConfigHandling.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment; + +import com.google.inject.Inject; +import org.apache.druid.data.input.impl.DimensionSchema; + +/** + * Helper class to determine the default mode of multi value handling. + */ +public class MultiValueConfigHandling +{ + /** + * INSTANCE is injected using static injection. + */ + @Inject + private static DefaultColumnFormatConfig INSTANCE; + + /** + * Return the defualt multi-value handling mode. + */ + public static DimensionSchema.MultiValueHandling getDefaultMultiValueHandlingMode() + { + return INSTANCE.getMultiValueHandlingMode(); + } +} diff --git a/processing/src/test/java/org/apache/druid/segment/DefaultColumnFormatsConfigTest.java b/processing/src/test/java/org/apache/druid/segment/DefaultColumnFormatsConfigTest.java index f259fc1dc237..7880b6735117 100644 --- a/processing/src/test/java/org/apache/druid/segment/DefaultColumnFormatsConfigTest.java +++ b/processing/src/test/java/org/apache/druid/segment/DefaultColumnFormatsConfigTest.java @@ -33,7 +33,7 @@ public class DefaultColumnFormatsConfigTest @Test public void testDefaultsSerde() throws JsonProcessingException { - DefaultColumnFormatConfig defaultColumnFormatConfig = new DefaultColumnFormatConfig(null); + DefaultColumnFormatConfig defaultColumnFormatConfig = new DefaultColumnFormatConfig(null, null); String there = MAPPER.writeValueAsString(defaultColumnFormatConfig); DefaultColumnFormatConfig andBack = MAPPER.readValue(there, DefaultColumnFormatConfig.class); Assert.assertEquals(defaultColumnFormatConfig, andBack); @@ -43,7 +43,7 @@ public void testDefaultsSerde() throws JsonProcessingException @Test public void testDefaultsSerdeOverride() throws JsonProcessingException { - DefaultColumnFormatConfig defaultColumnFormatConfig = new DefaultColumnFormatConfig(4); + DefaultColumnFormatConfig defaultColumnFormatConfig = new DefaultColumnFormatConfig(4, null); String there = MAPPER.writeValueAsString(defaultColumnFormatConfig); DefaultColumnFormatConfig andBack = MAPPER.readValue(there, DefaultColumnFormatConfig.class); Assert.assertEquals(defaultColumnFormatConfig, andBack); diff --git a/processing/src/test/java/org/apache/druid/segment/NestedDataColumnSchemaTest.java b/processing/src/test/java/org/apache/druid/segment/NestedDataColumnSchemaTest.java index 8bfcfdefa4f4..847f8a9d1965 100644 --- a/processing/src/test/java/org/apache/druid/segment/NestedDataColumnSchemaTest.java +++ b/processing/src/test/java/org/apache/druid/segment/NestedDataColumnSchemaTest.java @@ -29,8 +29,8 @@ public class NestedDataColumnSchemaTest { - private static final DefaultColumnFormatConfig DEFAULT_CONFIG = new DefaultColumnFormatConfig(null); - private static final DefaultColumnFormatConfig DEFAULT_CONFIG_V4 = new DefaultColumnFormatConfig(4); + private static final DefaultColumnFormatConfig DEFAULT_CONFIG = new DefaultColumnFormatConfig(null, null); + private static final DefaultColumnFormatConfig DEFAULT_CONFIG_V4 = new DefaultColumnFormatConfig(4, null); private static final ObjectMapper MAPPER; private static final ObjectMapper MAPPER_V4; diff --git a/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java b/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java index e788949f66b9..980a73311629 100644 --- a/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java +++ b/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java @@ -32,6 +32,7 @@ import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.query.DruidProcessingConfig; import org.apache.druid.segment.DefaultColumnFormatConfig; +import org.apache.druid.segment.MultiValueConfigHandling; import org.apache.druid.segment.column.ColumnConfig; import org.apache.druid.segment.loading.SegmentLoaderConfig; import org.apache.druid.segment.loading.StorageLocation; @@ -60,6 +61,7 @@ public void configure(Binder binder) bindLocationSelectorStrategy(binder); binder.bind(ServerTypeConfig.class).toProvider(Providers.of(null)); binder.bind(ColumnConfig.class).to(DruidProcessingConfig.class).in(LazySingleton.class); + binder.requestStaticInjection(MultiValueConfigHandling.class); } @Provides diff --git a/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java b/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java index 9369717bf1f6..83ca668acf93 100644 --- a/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java +++ b/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java @@ -161,10 +161,10 @@ public void testDumpNestedColumn() throws Exception new InjectableValues.Std() .addValue(ExprMacroTable.class.getName(), TestExprMacroTable.INSTANCE) .addValue(ObjectMapper.class.getName(), mapper) - .addValue(DefaultColumnFormatConfig.class, new DefaultColumnFormatConfig(null)) + .addValue(DefaultColumnFormatConfig.class, new DefaultColumnFormatConfig(null, null)) ); Mockito.when(injector.getInstance(Key.get(ObjectMapper.class, Json.class))).thenReturn(mapper); - Mockito.when(injector.getInstance(DefaultColumnFormatConfig.class)).thenReturn(new DefaultColumnFormatConfig(null)); + Mockito.when(injector.getInstance(DefaultColumnFormatConfig.class)).thenReturn(new DefaultColumnFormatConfig(null, null)); List segments = createSegments(tempFolder, closer); QueryableIndex queryableIndex = segments.get(0).asQueryableIndex(); @@ -201,10 +201,10 @@ public void testDumpNestedColumnPath() throws Exception new InjectableValues.Std() .addValue(ExprMacroTable.class.getName(), TestExprMacroTable.INSTANCE) .addValue(ObjectMapper.class.getName(), mapper) - .addValue(DefaultColumnFormatConfig.class, new DefaultColumnFormatConfig(null)) + .addValue(DefaultColumnFormatConfig.class, new DefaultColumnFormatConfig(null, null)) ); Mockito.when(injector.getInstance(Key.get(ObjectMapper.class, Json.class))).thenReturn(mapper); - Mockito.when(injector.getInstance(DefaultColumnFormatConfig.class)).thenReturn(new DefaultColumnFormatConfig(null)); + Mockito.when(injector.getInstance(DefaultColumnFormatConfig.class)).thenReturn(new DefaultColumnFormatConfig(null, null)); List segments = createSegments(tempFolder, closer); QueryableIndex queryableIndex = segments.get(0).asQueryableIndex(); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index d195e0cefb91..fa7acc00b744 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -548,7 +548,7 @@ public void configure(Binder binder) { binder.bind(DruidOperatorTable.class).in(LazySingleton.class); binder.bind(DataSegment.PruneSpecsHolder.class).toInstance(DataSegment.PruneSpecsHolder.DEFAULT); - binder.bind(DefaultColumnFormatConfig.class).toInstance(new DefaultColumnFormatConfig(null)); + binder.bind(DefaultColumnFormatConfig.class).toInstance(new DefaultColumnFormatConfig(null, null)); } @Provides From faf3028875e4442838cb870db675f49443d7237b Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 1 Aug 2024 09:32:12 -0700 Subject: [PATCH 02/14] Review comment: rename to stringMultiValueHandlingMode. Use side effect instead of static injection. --- .../data/input/impl/DimensionSchema.java | 4 +- .../guice/StringMultiValueHandlingModule.java | 73 +++++++++++++++++++ .../segment/DefaultColumnFormatConfig.java | 36 ++++----- .../segment/MultiValueConfigHandling.java | 43 ----------- .../apache/druid/guice/StorageNodeModule.java | 2 - .../initialization/CoreInjectorBuilder.java | 4 +- 6 files changed, 96 insertions(+), 66 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java delete mode 100644 processing/src/main/java/org/apache/druid/segment/MultiValueConfigHandling.java diff --git a/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java b/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java index 28ca890f094d..c37465dcbd22 100644 --- a/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java +++ b/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java @@ -26,13 +26,13 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.annotation.JsonValue; import com.google.common.base.Strings; +import org.apache.druid.guice.StringMultiValueHandlingModule; import org.apache.druid.guice.annotations.PublicApi; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.segment.AutoTypeColumnSchema; import org.apache.druid.segment.DimensionHandler; import org.apache.druid.segment.DimensionHandlerUtils; -import org.apache.druid.segment.MultiValueConfigHandling; import org.apache.druid.segment.NestedDataColumnSchema; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.TypeSignature; @@ -113,7 +113,7 @@ public static MultiValueHandling fromString(String name) public static MultiValueHandling ofDefault() { - return MultiValueConfigHandling.getDefaultMultiValueHandlingMode(); + return StringMultiValueHandlingModule.getConfiguredStringMultiValueHandlingMode(); } } diff --git a/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java b/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java new file mode 100644 index 000000000000..50f8bf62908f --- /dev/null +++ b/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Provides; +import org.apache.druid.data.input.impl.DimensionSchema; +import org.apache.druid.initialization.DruidModule; +import org.apache.druid.segment.DefaultColumnFormatConfig; + +import javax.annotation.Nonnull; + +/** + * Module to determine the default mode of string multi value handling. + */ +public class StringMultiValueHandlingModule implements DruidModule +{ + @Override + public void configure(Binder binder) + { + // binding our side effect class to the lifecycle causes setStringMultiValueHandlingMode to be called on service + // start, allowing use of the config to get the system default multi value handling mode. + LifecycleModule.register(binder, SideEffectHandlerRegisterer.class); + } + + /** + * The system property along with the default is managed in {@link DefaultColumnFormatConfig} itself. + * So the value is guaranteed to be non-null after lifecycle service start. + */ + @Nonnull + private static DimensionSchema.MultiValueHandling CONFIGURED_STRING_MULTI_HANDLING_MODE; + + public static DimensionSchema.MultiValueHandling getConfiguredStringMultiValueHandlingMode() + { + return CONFIGURED_STRING_MULTI_HANDLING_MODE; + } + + @Provides + @LazySingleton + public SideEffectHandlerRegisterer setStringMultiValueHandlingMode(DefaultColumnFormatConfig formatsConfig) + { + CONFIGURED_STRING_MULTI_HANDLING_MODE = formatsConfig.getStringMultiValueHandlingMode(); + return new SideEffectHandlerRegisterer(); + } + + /** + * This is used as a vehicle to register the correct version of the system default string mvd mode by side + * effect with the help of binding to {@link org.apache.druid.java.util.common.lifecycle.Lifecycle} so that + * {@link #setStringMultiValueHandlingMode(DefaultColumnFormatConfig)} can be called with the injected + * {@link DefaultColumnFormatConfig}. + */ + public static class SideEffectHandlerRegisterer + { + // nothing to see here + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/DefaultColumnFormatConfig.java b/processing/src/main/java/org/apache/druid/segment/DefaultColumnFormatConfig.java index a643594f3beb..86812ed31e77 100644 --- a/processing/src/main/java/org/apache/druid/segment/DefaultColumnFormatConfig.java +++ b/processing/src/main/java/org/apache/druid/segment/DefaultColumnFormatConfig.java @@ -41,19 +41,19 @@ public static void validateNestedFormatVersion(@Nullable Integer formatVersion) } } - private static void validateMultiValueHandlingMode(@Nullable String multiValueHandlingMode) + private static void validateMultiValueHandlingMode(@Nullable String stringMultiValueHandlingMode) { - if (multiValueHandlingMode != null) { + if (stringMultiValueHandlingMode != null) { try { - DimensionSchema.MultiValueHandling.fromString(multiValueHandlingMode); + DimensionSchema.MultiValueHandling.fromString(stringMultiValueHandlingMode); } catch (IllegalArgumentException e) { throw DruidException.forPersona(DruidException.Persona.OPERATOR) .ofCategory(DruidException.Category.INVALID_INPUT) .build( - "Invalid value[%s] specified for 'druid.indexing.formats.multiValueHandlingMode'." + "Invalid value[%s] specified for 'druid.indexing.formats.stringMultiValueHandlingMode'." + " Supported values are [%s].", - multiValueHandlingMode, + stringMultiValueHandlingMode, Arrays.toString(DimensionSchema.MultiValueHandling.values()) ); } @@ -64,22 +64,22 @@ private static void validateMultiValueHandlingMode(@Nullable String multiValueHa @JsonProperty("nestedColumnFormatVersion") private final Integer nestedColumnFormatVersion; - @JsonProperty("multiValueHandlingMode") - private final DimensionSchema.MultiValueHandling multiValueHandlingMode; + @JsonProperty("stringMultiValueHandlingMode") + private final DimensionSchema.MultiValueHandling stringMultiValueHandlingMode; @JsonCreator public DefaultColumnFormatConfig( @JsonProperty("nestedColumnFormatVersion") @Nullable Integer nestedColumnFormatVersion, - @JsonProperty("multiValueHandlingMode") @Nullable String multiValueHandlingMode + @JsonProperty("stringMultiValueHandlingMode") @Nullable String stringMultiValueHandlingMode ) { validateNestedFormatVersion(nestedColumnFormatVersion); - validateMultiValueHandlingMode(multiValueHandlingMode); + validateMultiValueHandlingMode(stringMultiValueHandlingMode); this.nestedColumnFormatVersion = nestedColumnFormatVersion; - this.multiValueHandlingMode = multiValueHandlingMode == null ? - DimensionSchema.MultiValueHandling.SORTED_ARRAY : - DimensionSchema.MultiValueHandling.fromString(multiValueHandlingMode); + this.stringMultiValueHandlingMode = stringMultiValueHandlingMode == null ? + DimensionSchema.MultiValueHandling.SORTED_ARRAY : + DimensionSchema.MultiValueHandling.fromString(stringMultiValueHandlingMode); } @Nullable @@ -89,10 +89,10 @@ public Integer getNestedColumnFormatVersion() return nestedColumnFormatVersion; } - @JsonProperty("multiValueHandlingMode") - public DimensionSchema.MultiValueHandling getMultiValueHandlingMode() + @JsonProperty("stringMultiValueHandlingMode") + public DimensionSchema.MultiValueHandling getStringMultiValueHandlingMode() { - return multiValueHandlingMode; + return stringMultiValueHandlingMode; } @Override @@ -106,13 +106,13 @@ public boolean equals(Object o) } DefaultColumnFormatConfig that = (DefaultColumnFormatConfig) o; return Objects.equals(nestedColumnFormatVersion, that.nestedColumnFormatVersion) - && multiValueHandlingMode == that.multiValueHandlingMode; + && stringMultiValueHandlingMode == that.stringMultiValueHandlingMode; } @Override public int hashCode() { - return Objects.hash(nestedColumnFormatVersion, multiValueHandlingMode); + return Objects.hash(nestedColumnFormatVersion, stringMultiValueHandlingMode); } @Override @@ -120,7 +120,7 @@ public String toString() { return "DefaultColumnFormatConfig{" + "nestedColumnFormatVersion=" + nestedColumnFormatVersion + - ", multiValueHandlingMode=" + multiValueHandlingMode + + ", stringMultiValueHandlingMode=" + stringMultiValueHandlingMode + '}'; } } diff --git a/processing/src/main/java/org/apache/druid/segment/MultiValueConfigHandling.java b/processing/src/main/java/org/apache/druid/segment/MultiValueConfigHandling.java deleted file mode 100644 index e51e63350556..000000000000 --- a/processing/src/main/java/org/apache/druid/segment/MultiValueConfigHandling.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.segment; - -import com.google.inject.Inject; -import org.apache.druid.data.input.impl.DimensionSchema; - -/** - * Helper class to determine the default mode of multi value handling. - */ -public class MultiValueConfigHandling -{ - /** - * INSTANCE is injected using static injection. - */ - @Inject - private static DefaultColumnFormatConfig INSTANCE; - - /** - * Return the defualt multi-value handling mode. - */ - public static DimensionSchema.MultiValueHandling getDefaultMultiValueHandlingMode() - { - return INSTANCE.getMultiValueHandlingMode(); - } -} diff --git a/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java b/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java index 980a73311629..e788949f66b9 100644 --- a/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java +++ b/server/src/main/java/org/apache/druid/guice/StorageNodeModule.java @@ -32,7 +32,6 @@ import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.query.DruidProcessingConfig; import org.apache.druid.segment.DefaultColumnFormatConfig; -import org.apache.druid.segment.MultiValueConfigHandling; import org.apache.druid.segment.column.ColumnConfig; import org.apache.druid.segment.loading.SegmentLoaderConfig; import org.apache.druid.segment.loading.StorageLocation; @@ -61,7 +60,6 @@ public void configure(Binder binder) bindLocationSelectorStrategy(binder); binder.bind(ServerTypeConfig.class).toProvider(Providers.of(null)); binder.bind(ColumnConfig.class).to(DruidProcessingConfig.class).in(LazySingleton.class); - binder.requestStaticInjection(MultiValueConfigHandling.class); } @Provides diff --git a/server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java b/server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java index e07ac5ed1155..05b2f90e3e76 100644 --- a/server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java +++ b/server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java @@ -39,6 +39,7 @@ import org.apache.druid.guice.ServerViewModule; import org.apache.druid.guice.StartupLoggingModule; import org.apache.druid.guice.StorageNodeModule; +import org.apache.druid.guice.StringMultiValueHandlingModule; import org.apache.druid.guice.annotations.Client; import org.apache.druid.guice.annotations.EscalatedClient; import org.apache.druid.guice.http.HttpClientModule; @@ -131,7 +132,8 @@ public CoreInjectorBuilder forServer() new StartupLoggingModule(), new ExternalStorageAccessSecurityModule(), new ServiceClientModule(), - new StorageConnectorModule() + new StorageConnectorModule(), + new StringMultiValueHandlingModule() ); return this; } From 5b250754b9c4daf1bba990a423fb3b9ccb7db7ae Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 1 Aug 2024 10:02:55 -0700 Subject: [PATCH 03/14] Add unit test for new module. --- .../StringMultiValueHandlingModuleTest.java | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 processing/src/test/java/org/apache/druid/guice/StringMultiValueHandlingModuleTest.java diff --git a/processing/src/test/java/org/apache/druid/guice/StringMultiValueHandlingModuleTest.java b/processing/src/test/java/org/apache/druid/guice/StringMultiValueHandlingModuleTest.java new file mode 100644 index 000000000000..441bb57f8978 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/guice/StringMultiValueHandlingModuleTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.guice; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Injector; +import org.apache.druid.data.input.impl.DimensionSchema; +import org.apache.druid.segment.DefaultColumnFormatConfig; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Properties; + +public class StringMultiValueHandlingModuleTest +{ + @Test + public void testDefaults() + { + final Properties props = new Properties(); + final Injector gadget = makeInjector(props); + + gadget.getInstance(StringMultiValueHandlingModule.SideEffectHandlerRegisterer.class); + + Assert.assertEquals( + DimensionSchema.MultiValueHandling.SORTED_ARRAY, + StringMultiValueHandlingModule.getConfiguredStringMultiValueHandlingMode() + ); + } + + @Test + public void testOverrides() + { + final Properties props = new Properties(); + props.setProperty("druid.indexing.formats.stringMultiValueHandlingMode", "ARRAY"); + final Injector gadget = makeInjector(props); + + gadget.getInstance(StringMultiValueHandlingModule.SideEffectHandlerRegisterer.class); + + Assert.assertEquals( + DimensionSchema.MultiValueHandling.ARRAY, + StringMultiValueHandlingModule.getConfiguredStringMultiValueHandlingMode() + ); + } + + + @Test + public void testOverridesCaseIsensitiveMode() + { + final Properties props = new Properties(); + props.setProperty("druid.indexing.formats.stringMultiValueHandlingMode", "sorted_array"); + final Injector gadget = makeInjector(props); + + gadget.getInstance(StringMultiValueHandlingModule.SideEffectHandlerRegisterer.class); + + Assert.assertEquals( + DimensionSchema.MultiValueHandling.SORTED_ARRAY, + StringMultiValueHandlingModule.getConfiguredStringMultiValueHandlingMode() + ); + } + + @Test + public void testInvalidMode() + { + final Properties props = new Properties(); + props.setProperty("druid.indexing.formats.stringMultiValueHandlingMode", "boo"); + final Injector gadget = makeInjector(props); + + final Exception exception = Assert.assertThrows( + Exception.class, + () -> gadget.getInstance(StringMultiValueHandlingModule.SideEffectHandlerRegisterer.class) + ); + Assert.assertTrue(exception.getMessage().contains( + "Invalid value[boo] specified for 'druid.indexing.formats.stringMultiValueHandlingMode'." + + " Supported values are [[SORTED_ARRAY, SORTED_SET, ARRAY]]." + )); + } + + private Injector makeInjector(final Properties props) + { + final StartupInjectorBuilder bob = new StartupInjectorBuilder().forTests().withProperties(props); + + bob.addAll( + ImmutableList.of( + binder -> { + JsonConfigProvider.bind(binder, "druid.indexing.formats", DefaultColumnFormatConfig.class); + }, + new StringMultiValueHandlingModule() + ) + ); + + return bob.build(); + } +} From 48018d1988b7bd5ed89071d71c29ef0a8d260c80 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 1 Aug 2024 10:07:00 -0700 Subject: [PATCH 04/14] rename test names. --- .../guice/StringMultiValueHandlingModuleTest.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/guice/StringMultiValueHandlingModuleTest.java b/processing/src/test/java/org/apache/druid/guice/StringMultiValueHandlingModuleTest.java index 441bb57f8978..b2845640f4a5 100644 --- a/processing/src/test/java/org/apache/druid/guice/StringMultiValueHandlingModuleTest.java +++ b/processing/src/test/java/org/apache/druid/guice/StringMultiValueHandlingModuleTest.java @@ -31,7 +31,7 @@ public class StringMultiValueHandlingModuleTest { @Test - public void testDefaults() + public void testDefaultMode() { final Properties props = new Properties(); final Injector gadget = makeInjector(props); @@ -45,32 +45,31 @@ public void testDefaults() } @Test - public void testOverrides() + public void testOverrideMode() { final Properties props = new Properties(); - props.setProperty("druid.indexing.formats.stringMultiValueHandlingMode", "ARRAY"); + props.setProperty("druid.indexing.formats.stringMultiValueHandlingMode", "sorted_array"); final Injector gadget = makeInjector(props); gadget.getInstance(StringMultiValueHandlingModule.SideEffectHandlerRegisterer.class); Assert.assertEquals( - DimensionSchema.MultiValueHandling.ARRAY, + DimensionSchema.MultiValueHandling.SORTED_ARRAY, StringMultiValueHandlingModule.getConfiguredStringMultiValueHandlingMode() ); } - @Test - public void testOverridesCaseIsensitiveMode() + public void testOverrideCaseInsensitiveMode() { final Properties props = new Properties(); - props.setProperty("druid.indexing.formats.stringMultiValueHandlingMode", "sorted_array"); + props.setProperty("druid.indexing.formats.stringMultiValueHandlingMode", "ARRAY"); final Injector gadget = makeInjector(props); gadget.getInstance(StringMultiValueHandlingModule.SideEffectHandlerRegisterer.class); Assert.assertEquals( - DimensionSchema.MultiValueHandling.SORTED_ARRAY, + DimensionSchema.MultiValueHandling.ARRAY, StringMultiValueHandlingModule.getConfiguredStringMultiValueHandlingMode() ); } From 93a5a6cd1061e1433e63a75878745e79fb38a64d Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 1 Aug 2024 11:09:23 -0700 Subject: [PATCH 05/14] Fix tests, static check failure and minor cleanup. --- .../indexer/IndexGeneratorCombinerTest.java | 2 ++ .../data/input/impl/DimensionSchema.java | 2 +- .../guice/StringMultiValueHandlingModule.java | 26 ++++++++++++++----- .../druid/segment/StringDimensionIndexer.java | 2 +- .../StringMultiValueHandlingModuleTest.java | 6 ++--- .../segment/realtime/FireHydrantTest.java | 2 ++ 6 files changed, 29 insertions(+), 11 deletions(-) diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorCombinerTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorCombinerTest.java index 385c28ff0fb0..2a9172ffb280 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorCombinerTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorCombinerTest.java @@ -29,6 +29,7 @@ import org.apache.druid.data.input.impl.StringInputRowParser; import org.apache.druid.data.input.impl.TimeAndDimsParseSpec; import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.guice.StringMultiValueHandlingModule; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; @@ -62,6 +63,7 @@ public class IndexGeneratorCombinerTest @Before public void setUp() throws Exception { + StringMultiValueHandlingModule.setStringMvForTests(); HadoopDruidIndexerConfig config = new HadoopDruidIndexerConfig( new HadoopIngestionSpec( new DataSchema( diff --git a/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java b/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java index c37465dcbd22..aa3bf593439b 100644 --- a/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java +++ b/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java @@ -113,7 +113,7 @@ public static MultiValueHandling fromString(String name) public static MultiValueHandling ofDefault() { - return StringMultiValueHandlingModule.getConfiguredStringMultiValueHandlingMode(); + return StringMultiValueHandlingModule.getConfiguredOrDefaultStringMvMode(); } } diff --git a/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java b/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java index 50f8bf62908f..ca1813ba9494 100644 --- a/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java +++ b/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java @@ -19,6 +19,7 @@ package org.apache.druid.guice; +import com.google.common.annotations.VisibleForTesting; import com.google.inject.Binder; import com.google.inject.Provides; import org.apache.druid.data.input.impl.DimensionSchema; @@ -45,25 +46,38 @@ public void configure(Binder binder) * So the value is guaranteed to be non-null after lifecycle service start. */ @Nonnull - private static DimensionSchema.MultiValueHandling CONFIGURED_STRING_MULTI_HANDLING_MODE; + private static DimensionSchema.MultiValueHandling STRING_MV_MODE; - public static DimensionSchema.MultiValueHandling getConfiguredStringMultiValueHandlingMode() + /** + * @return the configured string multi value handling mode from the system config if set; otherwise, returns + * the default. + */ + public static DimensionSchema.MultiValueHandling getConfiguredOrDefaultStringMvMode() { - return CONFIGURED_STRING_MULTI_HANDLING_MODE; + return STRING_MV_MODE; } @Provides @LazySingleton - public SideEffectHandlerRegisterer setStringMultiValueHandlingMode(DefaultColumnFormatConfig formatsConfig) + public static SideEffectHandlerRegisterer setStringMvMode(DefaultColumnFormatConfig formatsConfig) { - CONFIGURED_STRING_MULTI_HANDLING_MODE = formatsConfig.getStringMultiValueHandlingMode(); + STRING_MV_MODE = formatsConfig.getStringMultiValueHandlingMode(); return new SideEffectHandlerRegisterer(); } + /** + * Helper for wiring stuff up for tests. + */ + @VisibleForTesting + public static void setStringMvForTests() + { + STRING_MV_MODE = DimensionSchema.MultiValueHandling.SORTED_ARRAY; + } + /** * This is used as a vehicle to register the correct version of the system default string mvd mode by side * effect with the help of binding to {@link org.apache.druid.java.util.common.lifecycle.Lifecycle} so that - * {@link #setStringMultiValueHandlingMode(DefaultColumnFormatConfig)} can be called with the injected + * {@link #setStringMvMode(DefaultColumnFormatConfig)} can be called with the injected * {@link DefaultColumnFormatConfig}. */ public static class SideEffectHandlerRegisterer diff --git a/processing/src/main/java/org/apache/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/org/apache/druid/segment/StringDimensionIndexer.java index 923b48ca4784..46d45024437f 100644 --- a/processing/src/main/java/org/apache/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/org/apache/druid/segment/StringDimensionIndexer.java @@ -66,7 +66,7 @@ private static String emptyToNullIfNeeded(@Nullable Object o) private volatile boolean hasMultipleValues = false; public StringDimensionIndexer( - MultiValueHandling multiValueHandling, + @Nullable MultiValueHandling multiValueHandling, boolean hasBitmapIndexes, boolean hasSpatialIndexes, boolean useMaxMemoryEstimates diff --git a/processing/src/test/java/org/apache/druid/guice/StringMultiValueHandlingModuleTest.java b/processing/src/test/java/org/apache/druid/guice/StringMultiValueHandlingModuleTest.java index b2845640f4a5..5cfd2ee7eef8 100644 --- a/processing/src/test/java/org/apache/druid/guice/StringMultiValueHandlingModuleTest.java +++ b/processing/src/test/java/org/apache/druid/guice/StringMultiValueHandlingModuleTest.java @@ -40,7 +40,7 @@ public void testDefaultMode() Assert.assertEquals( DimensionSchema.MultiValueHandling.SORTED_ARRAY, - StringMultiValueHandlingModule.getConfiguredStringMultiValueHandlingMode() + StringMultiValueHandlingModule.getConfiguredOrDefaultStringMvMode() ); } @@ -55,7 +55,7 @@ public void testOverrideMode() Assert.assertEquals( DimensionSchema.MultiValueHandling.SORTED_ARRAY, - StringMultiValueHandlingModule.getConfiguredStringMultiValueHandlingMode() + StringMultiValueHandlingModule.getConfiguredOrDefaultStringMvMode() ); } @@ -70,7 +70,7 @@ public void testOverrideCaseInsensitiveMode() Assert.assertEquals( DimensionSchema.MultiValueHandling.ARRAY, - StringMultiValueHandlingModule.getConfiguredStringMultiValueHandlingMode() + StringMultiValueHandlingModule.getConfiguredOrDefaultStringMvMode() ); } diff --git a/server/src/test/java/org/apache/druid/segment/realtime/FireHydrantTest.java b/server/src/test/java/org/apache/druid/segment/realtime/FireHydrantTest.java index 38c3fda1e7e2..4f50ed3e7d0c 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/FireHydrantTest.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/FireHydrantTest.java @@ -19,6 +19,7 @@ package org.apache.druid.segment.realtime; +import org.apache.druid.guice.StringMultiValueHandlingModule; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Pair; import org.apache.druid.segment.IncrementalIndexSegment; @@ -55,6 +56,7 @@ public class FireHydrantTest extends InitializedNullHandlingTest @Before public void setup() { + StringMultiValueHandlingModule.setStringMvForTests(); incrementalIndexSegment = new IncrementalIndexSegment(TestIndex.getIncrementalTestIndex(), SegmentId.dummy("test")); queryableIndexSegment = new QueryableIndexSegment(TestIndex.getMMappedTestIndex(), SegmentId.dummy("test")); From 041fbdba2e8b1505d9405564a8b7796ee07c0ad4 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 1 Aug 2024 12:21:13 -0700 Subject: [PATCH 06/14] More test fixes. --- .../org/apache/druid/indexer/IndexGeneratorCombinerTest.java | 2 +- .../apache/druid/guice/StringMultiValueHandlingModule.java | 4 ++-- .../org/apache/druid/testing/InitializedNullHandlingTest.java | 2 ++ .../org/apache/druid/segment/realtime/FireHydrantTest.java | 2 +- .../org/apache/druid/sql/calcite/util/CalciteTestBase.java | 2 ++ .../org/apache/druid/sql/calcite/util/SqlTestFramework.java | 4 +++- 6 files changed, 11 insertions(+), 5 deletions(-) diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorCombinerTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorCombinerTest.java index 2a9172ffb280..5ccedb670e7b 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorCombinerTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorCombinerTest.java @@ -63,7 +63,7 @@ public class IndexGeneratorCombinerTest @Before public void setUp() throws Exception { - StringMultiValueHandlingModule.setStringMvForTests(); + StringMultiValueHandlingModule.initializeStringMvForTests(); HadoopDruidIndexerConfig config = new HadoopDruidIndexerConfig( new HadoopIngestionSpec( new DataSchema( diff --git a/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java b/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java index ca1813ba9494..5d00c8b7e7ac 100644 --- a/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java +++ b/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java @@ -66,10 +66,10 @@ public static SideEffectHandlerRegisterer setStringMvMode(DefaultColumnFormatCon } /** - * Helper for wiring stuff up for tests. + * Helper for wiring stuff up for tests that don't use guice injection. */ @VisibleForTesting - public static void setStringMvForTests() + public static void initializeStringMvForTests() { STRING_MV_MODE = DimensionSchema.MultiValueHandling.SORTED_ARRAY; } diff --git a/processing/src/test/java/org/apache/druid/testing/InitializedNullHandlingTest.java b/processing/src/test/java/org/apache/druid/testing/InitializedNullHandlingTest.java index b0334757001a..9a2bce752f24 100644 --- a/processing/src/test/java/org/apache/druid/testing/InitializedNullHandlingTest.java +++ b/processing/src/test/java/org/apache/druid/testing/InitializedNullHandlingTest.java @@ -20,6 +20,7 @@ package org.apache.druid.testing; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.guice.StringMultiValueHandlingModule; import org.apache.druid.math.expr.ExpressionProcessing; public class InitializedNullHandlingTest @@ -27,5 +28,6 @@ public class InitializedNullHandlingTest static { NullHandling.initializeForTests(); ExpressionProcessing.initializeForTests(); + StringMultiValueHandlingModule.initializeStringMvForTests(); } } diff --git a/server/src/test/java/org/apache/druid/segment/realtime/FireHydrantTest.java b/server/src/test/java/org/apache/druid/segment/realtime/FireHydrantTest.java index 4f50ed3e7d0c..502e42203daa 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/FireHydrantTest.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/FireHydrantTest.java @@ -56,7 +56,7 @@ public class FireHydrantTest extends InitializedNullHandlingTest @Before public void setup() { - StringMultiValueHandlingModule.setStringMvForTests(); + StringMultiValueHandlingModule.initializeStringMvForTests(); incrementalIndexSegment = new IncrementalIndexSegment(TestIndex.getIncrementalTestIndex(), SegmentId.dummy("test")); queryableIndexSegment = new QueryableIndexSegment(TestIndex.getMMappedTestIndex(), SegmentId.dummy("test")); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTestBase.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTestBase.java index 9a69c62dc734..36225cb8d031 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTestBase.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTestBase.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.guice.StringMultiValueHandlingModule; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.segment.column.ColumnType; @@ -52,6 +53,7 @@ public static void setupCalciteProperties() { NullHandling.initializeForTests(); ExpressionProcessing.initializeForTests(); + StringMultiValueHandlingModule.initializeStringMvForTests(); } /** diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index fa7acc00b744..f3f8bacbb012 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -31,6 +31,7 @@ import org.apache.druid.guice.LazySingleton; import org.apache.druid.guice.SegmentWranglerModule; import org.apache.druid.guice.StartupInjectorBuilder; +import org.apache.druid.guice.StringMultiValueHandlingModule; import org.apache.druid.initialization.CoreInjectorBuilder; import org.apache.druid.initialization.DruidModule; import org.apache.druid.initialization.ServiceInjectorBuilder; @@ -619,7 +620,8 @@ private SqlTestFramework(Builder builder) .addModule(new SegmentWranglerModule()) .addModule(new SqlAggregationModule()) .addModule(new ExpressionModule()) - .addModule(new TestSetupModule(builder)); + .addModule(new TestSetupModule(builder)) + .addModule(new StringMultiValueHandlingModule()); builder.componentSupplier.configureGuice(injectorBuilder); From de222d5f2f1dcb768f2b81f3f8f22a5d60a632f9 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 1 Aug 2024 15:53:29 -0700 Subject: [PATCH 07/14] Move default to the module and remove explicit test initialization. --- .../indexer/IndexGeneratorCombinerTest.java | 2 -- .../guice/StringMultiValueHandlingModule.java | 24 ++++++------------- .../segment/DefaultColumnFormatConfig.java | 12 +++++----- .../testing/InitializedNullHandlingTest.java | 2 -- .../segment/realtime/FireHydrantTest.java | 2 -- .../sql/calcite/util/CalciteTestBase.java | 1 - 6 files changed, 13 insertions(+), 30 deletions(-) diff --git a/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorCombinerTest.java b/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorCombinerTest.java index 5ccedb670e7b..385c28ff0fb0 100644 --- a/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorCombinerTest.java +++ b/indexing-hadoop/src/test/java/org/apache/druid/indexer/IndexGeneratorCombinerTest.java @@ -29,7 +29,6 @@ import org.apache.druid.data.input.impl.StringInputRowParser; import org.apache.druid.data.input.impl.TimeAndDimsParseSpec; import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.guice.StringMultiValueHandlingModule; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; @@ -63,7 +62,6 @@ public class IndexGeneratorCombinerTest @Before public void setUp() throws Exception { - StringMultiValueHandlingModule.initializeStringMvForTests(); HadoopDruidIndexerConfig config = new HadoopDruidIndexerConfig( new HadoopIngestionSpec( new DataSchema( diff --git a/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java b/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java index 5d00c8b7e7ac..4e33e0d460fa 100644 --- a/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java +++ b/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java @@ -19,15 +19,12 @@ package org.apache.druid.guice; -import com.google.common.annotations.VisibleForTesting; import com.google.inject.Binder; import com.google.inject.Provides; import org.apache.druid.data.input.impl.DimensionSchema; import org.apache.druid.initialization.DruidModule; import org.apache.druid.segment.DefaultColumnFormatConfig; -import javax.annotation.Nonnull; - /** * Module to determine the default mode of string multi value handling. */ @@ -42,11 +39,11 @@ public void configure(Binder binder) } /** - * The system property along with the default is managed in {@link DefaultColumnFormatConfig} itself. - * So the value is guaranteed to be non-null after lifecycle service start. + * The default value is set here so tests can just use it without any explicit initialization. + * In production, this default may be overridden if a value is configured via + * {@link #setStringMvMode(DefaultColumnFormatConfig)}. */ - @Nonnull - private static DimensionSchema.MultiValueHandling STRING_MV_MODE; + private static DimensionSchema.MultiValueHandling STRING_MV_MODE = DimensionSchema.MultiValueHandling.SORTED_ARRAY; /** * @return the configured string multi value handling mode from the system config if set; otherwise, returns @@ -61,19 +58,12 @@ public static DimensionSchema.MultiValueHandling getConfiguredOrDefaultStringMvM @LazySingleton public static SideEffectHandlerRegisterer setStringMvMode(DefaultColumnFormatConfig formatsConfig) { - STRING_MV_MODE = formatsConfig.getStringMultiValueHandlingMode(); + if (formatsConfig.getStringMultiValueHandlingMode() != null) { + STRING_MV_MODE = DimensionSchema.MultiValueHandling.fromString(formatsConfig.getStringMultiValueHandlingMode()); + } return new SideEffectHandlerRegisterer(); } - /** - * Helper for wiring stuff up for tests that don't use guice injection. - */ - @VisibleForTesting - public static void initializeStringMvForTests() - { - STRING_MV_MODE = DimensionSchema.MultiValueHandling.SORTED_ARRAY; - } - /** * This is used as a vehicle to register the correct version of the system default string mvd mode by side * effect with the help of binding to {@link org.apache.druid.java.util.common.lifecycle.Lifecycle} so that diff --git a/processing/src/main/java/org/apache/druid/segment/DefaultColumnFormatConfig.java b/processing/src/main/java/org/apache/druid/segment/DefaultColumnFormatConfig.java index 86812ed31e77..4df37f85b354 100644 --- a/processing/src/main/java/org/apache/druid/segment/DefaultColumnFormatConfig.java +++ b/processing/src/main/java/org/apache/druid/segment/DefaultColumnFormatConfig.java @@ -64,8 +64,9 @@ private static void validateMultiValueHandlingMode(@Nullable String stringMultiV @JsonProperty("nestedColumnFormatVersion") private final Integer nestedColumnFormatVersion; + @Nullable @JsonProperty("stringMultiValueHandlingMode") - private final DimensionSchema.MultiValueHandling stringMultiValueHandlingMode; + private final String stringMultiValueHandlingMode; @JsonCreator public DefaultColumnFormatConfig( @@ -77,9 +78,7 @@ public DefaultColumnFormatConfig( validateMultiValueHandlingMode(stringMultiValueHandlingMode); this.nestedColumnFormatVersion = nestedColumnFormatVersion; - this.stringMultiValueHandlingMode = stringMultiValueHandlingMode == null ? - DimensionSchema.MultiValueHandling.SORTED_ARRAY : - DimensionSchema.MultiValueHandling.fromString(stringMultiValueHandlingMode); + this.stringMultiValueHandlingMode = stringMultiValueHandlingMode; } @Nullable @@ -89,8 +88,9 @@ public Integer getNestedColumnFormatVersion() return nestedColumnFormatVersion; } + @Nullable @JsonProperty("stringMultiValueHandlingMode") - public DimensionSchema.MultiValueHandling getStringMultiValueHandlingMode() + public String getStringMultiValueHandlingMode() { return stringMultiValueHandlingMode; } @@ -106,7 +106,7 @@ public boolean equals(Object o) } DefaultColumnFormatConfig that = (DefaultColumnFormatConfig) o; return Objects.equals(nestedColumnFormatVersion, that.nestedColumnFormatVersion) - && stringMultiValueHandlingMode == that.stringMultiValueHandlingMode; + && Objects.equals(stringMultiValueHandlingMode, that.stringMultiValueHandlingMode); } @Override diff --git a/processing/src/test/java/org/apache/druid/testing/InitializedNullHandlingTest.java b/processing/src/test/java/org/apache/druid/testing/InitializedNullHandlingTest.java index 9a2bce752f24..b0334757001a 100644 --- a/processing/src/test/java/org/apache/druid/testing/InitializedNullHandlingTest.java +++ b/processing/src/test/java/org/apache/druid/testing/InitializedNullHandlingTest.java @@ -20,7 +20,6 @@ package org.apache.druid.testing; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.StringMultiValueHandlingModule; import org.apache.druid.math.expr.ExpressionProcessing; public class InitializedNullHandlingTest @@ -28,6 +27,5 @@ public class InitializedNullHandlingTest static { NullHandling.initializeForTests(); ExpressionProcessing.initializeForTests(); - StringMultiValueHandlingModule.initializeStringMvForTests(); } } diff --git a/server/src/test/java/org/apache/druid/segment/realtime/FireHydrantTest.java b/server/src/test/java/org/apache/druid/segment/realtime/FireHydrantTest.java index 502e42203daa..38c3fda1e7e2 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/FireHydrantTest.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/FireHydrantTest.java @@ -19,7 +19,6 @@ package org.apache.druid.segment.realtime; -import org.apache.druid.guice.StringMultiValueHandlingModule; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Pair; import org.apache.druid.segment.IncrementalIndexSegment; @@ -56,7 +55,6 @@ public class FireHydrantTest extends InitializedNullHandlingTest @Before public void setup() { - StringMultiValueHandlingModule.initializeStringMvForTests(); incrementalIndexSegment = new IncrementalIndexSegment(TestIndex.getIncrementalTestIndex(), SegmentId.dummy("test")); queryableIndexSegment = new QueryableIndexSegment(TestIndex.getMMappedTestIndex(), SegmentId.dummy("test")); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTestBase.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTestBase.java index 36225cb8d031..babd9fe60afc 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTestBase.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTestBase.java @@ -53,7 +53,6 @@ public static void setupCalciteProperties() { NullHandling.initializeForTests(); ExpressionProcessing.initializeForTests(); - StringMultiValueHandlingModule.initializeStringMvForTests(); } /** From deb08010e3933f242e9e8a1d9f2b67accdc38155 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 1 Aug 2024 16:02:17 -0700 Subject: [PATCH 08/14] Update DefaultColumnFormatsConfigTest --- .../apache/druid/segment/DefaultColumnFormatsConfigTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/processing/src/test/java/org/apache/druid/segment/DefaultColumnFormatsConfigTest.java b/processing/src/test/java/org/apache/druid/segment/DefaultColumnFormatsConfigTest.java index 7880b6735117..c4de7e41f2b1 100644 --- a/processing/src/test/java/org/apache/druid/segment/DefaultColumnFormatsConfigTest.java +++ b/processing/src/test/java/org/apache/druid/segment/DefaultColumnFormatsConfigTest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.data.input.impl.DimensionSchema; import org.apache.druid.jackson.DefaultObjectMapper; import org.junit.Assert; import org.junit.Test; @@ -38,16 +39,18 @@ public void testDefaultsSerde() throws JsonProcessingException DefaultColumnFormatConfig andBack = MAPPER.readValue(there, DefaultColumnFormatConfig.class); Assert.assertEquals(defaultColumnFormatConfig, andBack); Assert.assertNull(andBack.getNestedColumnFormatVersion()); + Assert.assertNull(andBack.getStringMultiValueHandlingMode()); } @Test public void testDefaultsSerdeOverride() throws JsonProcessingException { - DefaultColumnFormatConfig defaultColumnFormatConfig = new DefaultColumnFormatConfig(4, null); + DefaultColumnFormatConfig defaultColumnFormatConfig = new DefaultColumnFormatConfig(4, "ARRAY"); String there = MAPPER.writeValueAsString(defaultColumnFormatConfig); DefaultColumnFormatConfig andBack = MAPPER.readValue(there, DefaultColumnFormatConfig.class); Assert.assertEquals(defaultColumnFormatConfig, andBack); Assert.assertEquals(4, (int) andBack.getNestedColumnFormatVersion()); + Assert.assertEquals(DimensionSchema.MultiValueHandling.ARRAY.toString(), andBack.getStringMultiValueHandlingMode()); } @Test From 972439487a9fb954f2aef6449135f119c6f6fec9 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 1 Aug 2024 16:49:42 -0700 Subject: [PATCH 09/14] Rename NestedDataModule to DefaultColumnFormatConfigModule and handle both nested format & mv handling. --- .../GroupByDeserializationBenchmark.java | 6 +- .../frame/FrameChannelMergerBenchmark.java | 4 +- .../benchmark/query/SqlGroupByBenchmark.java | 6 +- .../druid/msq/exec/MSQComplexGroupByTest.java | 6 +- .../apache/druid/msq/test/MSQTestBase.java | 8 +- .../data/input/impl/DimensionSchema.java | 4 +- ...a => DefaultColumnFormatConfigModule.java} | 63 ++++++++-- .../guice/StringMultiValueHandlingModule.java | 77 ------------- .../nested/NestedDataComplexTypeSerde.java | 4 +- .../druid/frame/write/FrameWriterTest.java | 4 +- ... DefaultColumnFormatConfigModuleTest.java} | 56 ++++++++- .../StringMultiValueHandlingModuleTest.java | 109 ------------------ .../apache/druid/math/expr/FunctionTest.java | 4 +- .../druid/query/NestedDataTestUtils.java | 4 +- .../groupby/NestedDataGroupByQueryTest.java | 6 +- .../groupby/NestedGroupByArrayQueryTest.java | 6 +- ...lumnGroupByColumnSelectorStrategyTest.java | 4 +- .../query/scan/NestedDataScanQueryTest.java | 6 +- .../NestedDataTimeseriesQueryTest.java | 4 +- .../query/topn/NestedDataTopNQueryTest.java | 6 +- .../segment/AutoTypeColumnIndexerTest.java | 4 +- .../NestedDataColumnIndexerV4Test.java | 4 +- .../segment/data/IncrementalIndexTest.java | 4 +- .../ArrayContainsElementFilterTests.java | 4 +- .../druid/segment/filter/BaseFilterTest.java | 4 +- .../segment/filter/EqualityFilterTests.java | 4 +- .../segment/generator/SegmentGenerator.java | 4 +- .../IncrementalIndexIngestionTest.java | 4 +- .../IncrementalIndexMultiValueSpecTest.java | 4 +- .../IncrementalIndexStorageAdapterTest.java | 4 +- .../nested/NestedDataColumnSupplierTest.java | 4 +- .../NestedDataColumnSupplierV4Test.java | 4 +- .../NestedFieldColumnSelectorsTest.java | 6 +- .../ScalarDoubleColumnSupplierTest.java | 4 +- .../nested/ScalarLongColumnSupplierTest.java | 4 +- .../ScalarStringColumnSupplierTest.java | 4 +- .../nested/VariantColumnSupplierTest.java | 4 +- .../initialization/CoreInjectorBuilder.java | 8 +- .../org/apache/druid/cli/DumpSegmentTest.java | 8 +- .../sql/calcite/CalciteArraysQueryTest.java | 4 +- .../calcite/CalciteNestedDataQueryTest.java | 6 +- .../sql/calcite/util/CalciteTestBase.java | 1 - .../sql/calcite/util/SqlTestFramework.java | 4 +- 43 files changed, 196 insertions(+), 292 deletions(-) rename processing/src/main/java/org/apache/druid/guice/{NestedDataModule.java => DefaultColumnFormatConfigModule.java} (62%) delete mode 100644 processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java rename processing/src/test/java/org/apache/druid/guice/{NestedDataModuleTest.java => DefaultColumnFormatConfigModuleTest.java} (61%) delete mode 100644 processing/src/test/java/org/apache/druid/guice/StringMultiValueHandlingModuleTest.java diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/GroupByDeserializationBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/GroupByDeserializationBenchmark.java index c3ebbe5122f9..a1430e13dd5b 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/GroupByDeserializationBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/GroupByDeserializationBenchmark.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.jackson.AggregatorsModule; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Pair; @@ -68,7 +68,7 @@ public class GroupByDeserializationBenchmark static { NullHandling.initializeForTests(); - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); AggregatorsModule.registerComplexMetricsAndSerde(); } @@ -93,7 +93,7 @@ public class GroupByDeserializationBenchmark public void setup() throws JsonProcessingException { final ObjectMapper undecoratedMapper = TestHelper.makeJsonMapper(); - undecoratedMapper.registerModules(NestedDataModule.getJacksonModulesList()); + undecoratedMapper.registerModules(DefaultColumnFormatConfigModule.getJacksonModulesList()); undecoratedMapper.registerModule(new AggregatorsModule()); final Pair sqlQueryAndResultRow = sqlQueryAndResultRow( numDimensions, diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/frame/FrameChannelMergerBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/frame/FrameChannelMergerBenchmark.java index a864bcd3419f..f3292b35f54b 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/frame/FrameChannelMergerBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/frame/FrameChannelMergerBenchmark.java @@ -37,7 +37,7 @@ import org.apache.druid.frame.read.FrameReader; import org.apache.druid.frame.testutil.FrameSequenceBuilder; import org.apache.druid.frame.write.FrameWriters; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.NonnullPair; @@ -85,7 +85,7 @@ public class FrameChannelMergerBenchmark { static { NullHandling.initializeForTests(); - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); } private static final String KEY = "key"; diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlGroupByBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlGroupByBenchmark.java index 80b6647a0eee..453a69e20273 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlGroupByBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlGroupByBenchmark.java @@ -24,7 +24,7 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.data.input.impl.DimensionSchema; import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Sequence; @@ -89,7 +89,7 @@ public class SqlGroupByBenchmark static { NullHandling.initializeForTests(); ExpressionProcessing.initializeForTests(); - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); } private static final Logger log = new Logger(SqlGroupByBenchmark.class); @@ -331,7 +331,7 @@ public void setup() // Hacky and pollutes global namespace, but it is fine since benchmarks are run in isolation. Wasn't able // to work up a cleaner way of doing it by modifying the injector. - CalciteTests.getJsonMapper().registerModules(NestedDataModule.getJacksonModulesList()); + CalciteTests.getJsonMapper().registerModules(DefaultColumnFormatConfigModule.getJacksonModulesList()); final DruidSchemaCatalog rootSchema = CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER); diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java index efddab577e77..3e0240732871 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java @@ -25,7 +25,7 @@ import org.apache.druid.data.input.impl.JsonInputFormat; import org.apache.druid.data.input.impl.LocalInputSource; import org.apache.druid.data.input.impl.systemfield.SystemFields; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.msq.indexing.MSQSpec; import org.apache.druid.msq.indexing.MSQTuningConfig; @@ -64,7 +64,7 @@ public class MSQComplexGroupByTest extends MSQTestBase { static { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); } private String dataFileNameJsonString; @@ -109,7 +109,7 @@ public void setup() throws IOException dataFileSignature ); - objectMapper.registerModules(NestedDataModule.getJacksonModulesList()); + objectMapper.registerModules(DefaultColumnFormatConfigModule.getJacksonModulesList()); } @MethodSource("data") diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java index 2136d96d6d11..770d1fd833c7 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java @@ -48,6 +48,7 @@ import org.apache.druid.frame.channel.FrameChannelSequence; import org.apache.druid.frame.processor.Bouncer; import org.apache.druid.frame.testutil.FrameTestUtil; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.guice.DruidInjectorBuilder; import org.apache.druid.guice.DruidSecondaryModule; import org.apache.druid.guice.ExpressionModule; @@ -55,7 +56,6 @@ import org.apache.druid.guice.IndexingServiceTuningConfigModule; import org.apache.druid.guice.JoinableFactoryModule; import org.apache.druid.guice.JsonConfigProvider; -import org.apache.druid.guice.NestedDataModule; import org.apache.druid.guice.SegmentWranglerModule; import org.apache.druid.guice.StartupInjectorBuilder; import org.apache.druid.guice.annotations.EscalatedGlobal; @@ -357,8 +357,8 @@ public void configure(Binder binder) { // We want this module to bring InputSourceModule along for the ride. binder.install(new InputSourceModule()); - binder.install(new NestedDataModule()); - NestedDataModule.registerHandlersAndSerde(); + binder.install(new DefaultColumnFormatConfigModule()); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); SqlBindings.addOperatorConversion(binder, ExternalOperatorConversion.class); SqlBindings.addOperatorConversion(binder, HttpOperatorConversion.class); SqlBindings.addOperatorConversion(binder, InlineOperatorConversion.class); @@ -521,7 +521,7 @@ public String getFormatString() objectMapper = setupObjectMapper(injector); objectMapper.registerModules(new StorageConnectorModule().getJacksonModules()); objectMapper.registerModules(sqlModule.getJacksonModules()); - objectMapper.registerModules(NestedDataModule.getJacksonModulesList()); + objectMapper.registerModules(DefaultColumnFormatConfigModule.getJacksonModulesList()); doReturn(mock(Request.class)).when(brokerClient).makeRequest(any(), anyString()); diff --git a/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java b/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java index aa3bf593439b..08b5d2775a29 100644 --- a/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java +++ b/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java @@ -26,7 +26,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.annotation.JsonValue; import com.google.common.base.Strings; -import org.apache.druid.guice.StringMultiValueHandlingModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.guice.annotations.PublicApi; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.emitter.EmittingLogger; @@ -113,7 +113,7 @@ public static MultiValueHandling fromString(String name) public static MultiValueHandling ofDefault() { - return StringMultiValueHandlingModule.getConfiguredOrDefaultStringMvMode(); + return DefaultColumnFormatConfigModule.getStringMultiValueHandlingMode(); } } diff --git a/processing/src/main/java/org/apache/druid/guice/NestedDataModule.java b/processing/src/main/java/org/apache/druid/guice/DefaultColumnFormatConfigModule.java similarity index 62% rename from processing/src/main/java/org/apache/druid/guice/NestedDataModule.java rename to processing/src/main/java/org/apache/druid/guice/DefaultColumnFormatConfigModule.java index daaf4ff2f65a..3172dcb420dc 100644 --- a/processing/src/main/java/org/apache/druid/guice/NestedDataModule.java +++ b/processing/src/main/java/org/apache/druid/guice/DefaultColumnFormatConfigModule.java @@ -25,6 +25,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.inject.Binder; import com.google.inject.Provides; +import org.apache.druid.data.input.impl.DimensionSchema; import org.apache.druid.initialization.DruidModule; import org.apache.druid.segment.DefaultColumnFormatConfig; import org.apache.druid.segment.DimensionHandler; @@ -41,7 +42,7 @@ import java.util.Collections; import java.util.List; -public class NestedDataModule implements DruidModule +public class DefaultColumnFormatConfigModule implements DruidModule { @Override public List getJacksonModules() @@ -53,14 +54,17 @@ public List getJacksonModules() public void configure(Binder binder) { registerSerde(); - // binding our side effect class to the lifecycle causes registerHandler to be called on service start, allowing - // use of the config to get the system default format version - LifecycleModule.register(binder, SideEffectHandlerRegisterer.class); + // binding our side effect classes to the lifecycle causes the registerDimensionHandler and + // setStringMultiValueHandlingMode methods to be called on service start, allowing use of the config to get the + // system default format version and string multi value handling mode. We need to register them as two different side + // effect classes to make Guice initialization happy. + LifecycleModule.register(binder, SideEffectDimensionHandlerRegisterer.class); + LifecycleModule.register(binder, SideEffectMultiValueHandlingRegisterer.class); } @Provides @LazySingleton - public SideEffectHandlerRegisterer registerHandler(DefaultColumnFormatConfig formatsConfig) + public SideEffectDimensionHandlerRegisterer registerDimensionHandler(DefaultColumnFormatConfig formatsConfig) { if (formatsConfig.getNestedColumnFormatVersion() != null && formatsConfig.getNestedColumnFormatVersion() == 4) { DimensionHandlerUtils.registerDimensionHandlerProvider( @@ -73,7 +77,7 @@ public SideEffectHandlerRegisterer registerHandler(DefaultColumnFormatConfig for new NestedCommonFormatHandlerProvider() ); } - return new SideEffectHandlerRegisterer(); + return new SideEffectDimensionHandlerRegisterer(); } public static List getJacksonModulesList() @@ -126,13 +130,56 @@ public DimensionHandler get(Stri return new NestedDataColumnHandlerV4(dimensionName); } } + + /** + * The default value is set here so tests can just use it without any explicit initialization. + * In production, this default may be overridden if a value is configured via + * {@link #setStringMultiValueHandlingMode(DefaultColumnFormatConfig)}. + */ + private static DimensionSchema.MultiValueHandling STRING_MV_MODE = DimensionSchema.MultiValueHandling.SORTED_ARRAY; + + @Provides + @LazySingleton + public static SideEffectMultiValueHandlingRegisterer setStringMultiValueHandlingMode(DefaultColumnFormatConfig formatsConfig) + { + if (formatsConfig.getStringMultiValueHandlingMode() != null) { + STRING_MV_MODE = DimensionSchema.MultiValueHandling.fromString(formatsConfig.getStringMultiValueHandlingMode()); + } + return new SideEffectMultiValueHandlingRegisterer(); + } + + /** + * @return the configured string multi value handling mode from the system config if set; otherwise, returns + * the default. + */ + public static DimensionSchema.MultiValueHandling getStringMultiValueHandlingMode() + { + return STRING_MV_MODE; + } + /** * this is used as a vehicle to register the correct version of the system default nested column handler by side * effect with the help of binding to {@link org.apache.druid.java.util.common.lifecycle.Lifecycle} so that - * {@link #registerHandler(DefaultColumnFormatConfig)} can be called with the injected + * {@link #registerDimensionHandler(DefaultColumnFormatConfig)} can be called with the injected * {@link DefaultColumnFormatConfig}. */ - public static class SideEffectHandlerRegisterer + public static class SideEffectDimensionHandlerRegisterer + { + // nothing to see here + } + + /** + * This is used as a vehicle to register the correct multi value handling mode by side effect with the help of + * binding to {@link org.apache.druid.java.util.common.lifecycle.Lifecycle} so that + * {@link #setStringMultiValueHandlingMode(DefaultColumnFormatConfig)} can be called with the injected + * {@link DefaultColumnFormatConfig}. + * + *

+ * Similar purpose as {@link SideEffectDimensionHandlerRegisterer}, but we need separate static classes to make + * Guice initialization happy. + *

+ */ + public static class SideEffectMultiValueHandlingRegisterer { // nothing to see here } diff --git a/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java b/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java deleted file mode 100644 index 4e33e0d460fa..000000000000 --- a/processing/src/main/java/org/apache/druid/guice/StringMultiValueHandlingModule.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.guice; - -import com.google.inject.Binder; -import com.google.inject.Provides; -import org.apache.druid.data.input.impl.DimensionSchema; -import org.apache.druid.initialization.DruidModule; -import org.apache.druid.segment.DefaultColumnFormatConfig; - -/** - * Module to determine the default mode of string multi value handling. - */ -public class StringMultiValueHandlingModule implements DruidModule -{ - @Override - public void configure(Binder binder) - { - // binding our side effect class to the lifecycle causes setStringMultiValueHandlingMode to be called on service - // start, allowing use of the config to get the system default multi value handling mode. - LifecycleModule.register(binder, SideEffectHandlerRegisterer.class); - } - - /** - * The default value is set here so tests can just use it without any explicit initialization. - * In production, this default may be overridden if a value is configured via - * {@link #setStringMvMode(DefaultColumnFormatConfig)}. - */ - private static DimensionSchema.MultiValueHandling STRING_MV_MODE = DimensionSchema.MultiValueHandling.SORTED_ARRAY; - - /** - * @return the configured string multi value handling mode from the system config if set; otherwise, returns - * the default. - */ - public static DimensionSchema.MultiValueHandling getConfiguredOrDefaultStringMvMode() - { - return STRING_MV_MODE; - } - - @Provides - @LazySingleton - public static SideEffectHandlerRegisterer setStringMvMode(DefaultColumnFormatConfig formatsConfig) - { - if (formatsConfig.getStringMultiValueHandlingMode() != null) { - STRING_MV_MODE = DimensionSchema.MultiValueHandling.fromString(formatsConfig.getStringMultiValueHandlingMode()); - } - return new SideEffectHandlerRegisterer(); - } - - /** - * This is used as a vehicle to register the correct version of the system default string mvd mode by side - * effect with the help of binding to {@link org.apache.druid.java.util.common.lifecycle.Lifecycle} so that - * {@link #setStringMvMode(DefaultColumnFormatConfig)} can be called with the injected - * {@link DefaultColumnFormatConfig}. - */ - public static class SideEffectHandlerRegisterer - { - // nothing to see here - } -} diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataComplexTypeSerde.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataComplexTypeSerde.java index 4d1bb347f218..c22e76fc00b6 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataComplexTypeSerde.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataComplexTypeSerde.java @@ -25,7 +25,7 @@ import com.fasterxml.jackson.dataformat.smile.SmileGenerator; import it.unimi.dsi.fastutil.Hash; import org.apache.druid.data.input.impl.DimensionSchema; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.guava.Comparators; @@ -62,7 +62,7 @@ public class NestedDataComplexTypeSerde extends ComplexMetricSerde smileFactory.delegateToTextual(true); final ObjectMapper mapper = new DefaultObjectMapper(smileFactory, null); mapper.getFactory().setCodec(mapper); - mapper.registerModules(NestedDataModule.getJacksonModulesList()); + mapper.registerModules(DefaultColumnFormatConfigModule.getJacksonModulesList()); OBJECT_MAPPER = mapper; } diff --git a/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java b/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java index cde617a395cc..d2cc9de10f76 100644 --- a/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java +++ b/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java @@ -37,7 +37,7 @@ import org.apache.druid.frame.segment.FrameSegment; import org.apache.druid.frame.segment.FrameStorageAdapter; import org.apache.druid.frame.testutil.FrameTestUtil; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.RE; @@ -92,7 +92,7 @@ public class FrameWriterTest extends InitializedNullHandlingTest static { ComplexMetrics.registerSerde(HyperUniquesSerde.TYPE_NAME, new HyperUniquesSerde()); - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); } private static final int DEFAULT_ALLOCATOR_CAPACITY = 1_000_000; diff --git a/processing/src/test/java/org/apache/druid/guice/NestedDataModuleTest.java b/processing/src/test/java/org/apache/druid/guice/DefaultColumnFormatConfigModuleTest.java similarity index 61% rename from processing/src/test/java/org/apache/druid/guice/NestedDataModuleTest.java rename to processing/src/test/java/org/apache/druid/guice/DefaultColumnFormatConfigModuleTest.java index 59b5b2df4158..b310b37977e3 100644 --- a/processing/src/test/java/org/apache/druid/guice/NestedDataModuleTest.java +++ b/processing/src/test/java/org/apache/druid/guice/DefaultColumnFormatConfigModuleTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Injector; +import org.apache.druid.data.input.impl.DimensionSchema; import org.apache.druid.segment.DefaultColumnFormatConfig; import org.apache.druid.segment.DimensionHandlerProvider; import org.apache.druid.segment.DimensionHandlerUtils; @@ -35,7 +36,7 @@ import javax.annotation.Nullable; import java.util.Properties; -public class NestedDataModuleTest +public class DefaultColumnFormatConfigModuleTest { @Nullable private static DimensionHandlerProvider DEFAULT_HANDLER_PROVIDER; @@ -69,12 +70,18 @@ public void testDefaults() Injector gadget = makeInjector(props); // side effects - gadget.getInstance(NestedDataModule.SideEffectHandlerRegisterer.class); + gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectDimensionHandlerRegisterer.class); + gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectMultiValueHandlingRegisterer.class); DimensionHandlerProvider provider = DimensionHandlerUtils.DIMENSION_HANDLER_PROVIDERS.get( NestedDataComplexTypeSerde.TYPE_NAME ); Assert.assertTrue(provider.get("test") instanceof NestedCommonFormatColumnHandler); + + Assert.assertEquals( + DimensionSchema.MultiValueHandling.SORTED_ARRAY, + DefaultColumnFormatConfigModule.getStringMultiValueHandlingMode() + ); } @Test @@ -82,16 +89,55 @@ public void testOverride() { DimensionHandlerUtils.DIMENSION_HANDLER_PROVIDERS.remove(NestedDataComplexTypeSerde.TYPE_NAME); Properties props = new Properties(); - props.put("druid.indexing.formats.nestedColumnFormatVersion", "4"); + props.setProperty("druid.indexing.formats.nestedColumnFormatVersion", "4"); + props.setProperty("druid.indexing.formats.stringMultiValueHandlingMode", "sorted_array"); Injector gadget = makeInjector(props); // side effects - gadget.getInstance(NestedDataModule.SideEffectHandlerRegisterer.class); + gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectDimensionHandlerRegisterer.class); + gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectMultiValueHandlingRegisterer.class); DimensionHandlerProvider provider = DimensionHandlerUtils.DIMENSION_HANDLER_PROVIDERS.get( NestedDataComplexTypeSerde.TYPE_NAME ); Assert.assertTrue(provider.get("test") instanceof NestedDataColumnHandlerV4); + + Assert.assertEquals( + DimensionSchema.MultiValueHandling.SORTED_ARRAY, + DefaultColumnFormatConfigModule.getStringMultiValueHandlingMode() + ); + } + + @Test + public void testOverrideMultiValueHandlingModeCaseInsensitive() + { + final Properties props = new Properties(); + props.setProperty("druid.indexing.formats.stringMultiValueHandlingMode", "ARRAY"); + final Injector gadget = makeInjector(props); + + gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectMultiValueHandlingRegisterer.class); + + Assert.assertEquals( + DimensionSchema.MultiValueHandling.ARRAY, + DefaultColumnFormatConfigModule.getStringMultiValueHandlingMode() + ); + } + + @Test + public void testInvalidMultiValueHandlingMode() + { + final Properties props = new Properties(); + props.setProperty("druid.indexing.formats.stringMultiValueHandlingMode", "boo"); + final Injector gadget = makeInjector(props); + + final Exception exception = Assert.assertThrows( + Exception.class, + () -> gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectMultiValueHandlingRegisterer.class) + ); + Assert.assertTrue(exception.getMessage().contains( + "Invalid value[boo] specified for 'druid.indexing.formats.stringMultiValueHandlingMode'." + + " Supported values are [[SORTED_ARRAY, SORTED_SET, ARRAY]]." + )); } private Injector makeInjector(Properties props) @@ -104,7 +150,7 @@ private Injector makeInjector(Properties props) binder -> { JsonConfigProvider.bind(binder, "druid.indexing.formats", DefaultColumnFormatConfig.class); }, - new NestedDataModule() + new DefaultColumnFormatConfigModule() ) ); diff --git a/processing/src/test/java/org/apache/druid/guice/StringMultiValueHandlingModuleTest.java b/processing/src/test/java/org/apache/druid/guice/StringMultiValueHandlingModuleTest.java deleted file mode 100644 index 5cfd2ee7eef8..000000000000 --- a/processing/src/test/java/org/apache/druid/guice/StringMultiValueHandlingModuleTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.guice; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Injector; -import org.apache.druid.data.input.impl.DimensionSchema; -import org.apache.druid.segment.DefaultColumnFormatConfig; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Properties; - -public class StringMultiValueHandlingModuleTest -{ - @Test - public void testDefaultMode() - { - final Properties props = new Properties(); - final Injector gadget = makeInjector(props); - - gadget.getInstance(StringMultiValueHandlingModule.SideEffectHandlerRegisterer.class); - - Assert.assertEquals( - DimensionSchema.MultiValueHandling.SORTED_ARRAY, - StringMultiValueHandlingModule.getConfiguredOrDefaultStringMvMode() - ); - } - - @Test - public void testOverrideMode() - { - final Properties props = new Properties(); - props.setProperty("druid.indexing.formats.stringMultiValueHandlingMode", "sorted_array"); - final Injector gadget = makeInjector(props); - - gadget.getInstance(StringMultiValueHandlingModule.SideEffectHandlerRegisterer.class); - - Assert.assertEquals( - DimensionSchema.MultiValueHandling.SORTED_ARRAY, - StringMultiValueHandlingModule.getConfiguredOrDefaultStringMvMode() - ); - } - - @Test - public void testOverrideCaseInsensitiveMode() - { - final Properties props = new Properties(); - props.setProperty("druid.indexing.formats.stringMultiValueHandlingMode", "ARRAY"); - final Injector gadget = makeInjector(props); - - gadget.getInstance(StringMultiValueHandlingModule.SideEffectHandlerRegisterer.class); - - Assert.assertEquals( - DimensionSchema.MultiValueHandling.ARRAY, - StringMultiValueHandlingModule.getConfiguredOrDefaultStringMvMode() - ); - } - - @Test - public void testInvalidMode() - { - final Properties props = new Properties(); - props.setProperty("druid.indexing.formats.stringMultiValueHandlingMode", "boo"); - final Injector gadget = makeInjector(props); - - final Exception exception = Assert.assertThrows( - Exception.class, - () -> gadget.getInstance(StringMultiValueHandlingModule.SideEffectHandlerRegisterer.class) - ); - Assert.assertTrue(exception.getMessage().contains( - "Invalid value[boo] specified for 'druid.indexing.formats.stringMultiValueHandlingMode'." - + " Supported values are [[SORTED_ARRAY, SORTED_SET, ARRAY]]." - )); - } - - private Injector makeInjector(final Properties props) - { - final StartupInjectorBuilder bob = new StartupInjectorBuilder().forTests().withProperties(props); - - bob.addAll( - ImmutableList.of( - binder -> { - JsonConfigProvider.bind(binder, "druid.indexing.formats", DefaultColumnFormatConfig.class); - }, - new StringMultiValueHandlingModule() - ) - ); - - return bob.build(); - } -} diff --git a/processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java b/processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java index 56b3583031f0..05e7d543e600 100644 --- a/processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java +++ b/processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableSet; import org.apache.druid.common.config.NullHandling; import org.apache.druid.error.DruidException; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.StringUtils; @@ -66,7 +66,7 @@ public static void setupClass() TypeStrategiesTest.NULLABLE_TEST_PAIR_TYPE.getComplexTypeName(), new TypeStrategiesTest.NullableLongPairTypeStrategy() ); - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); } @Before diff --git a/processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java b/processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java index 1ff564106c58..c747f265b391 100644 --- a/processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java +++ b/processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java @@ -31,7 +31,7 @@ import org.apache.druid.data.input.impl.JsonInputFormat; import org.apache.druid.data.input.impl.LocalInputSource; import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.Granularity; @@ -166,7 +166,7 @@ public class NestedDataTestUtils static { JSON_MAPPER = TestHelper.makeJsonMapper(); - JSON_MAPPER.registerModules(NestedDataModule.getJacksonModulesList()); + JSON_MAPPER.registerModules(DefaultColumnFormatConfigModule.getJacksonModulesList()); } public static List createSimpleSegmentsTsv( diff --git a/processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java b/processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java index 77f86bdc4c50..5ec6d6e2cb29 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.io.Closer; @@ -82,10 +82,10 @@ public NestedDataGroupByQueryTest( String vectorize ) { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); this.vectorize = QueryContexts.Vectorize.fromString(vectorize); this.helper = AggregationTestHelper.createGroupByQueryAggregationTestHelper( - NestedDataModule.getJacksonModulesList(), + DefaultColumnFormatConfigModule.getJacksonModulesList(), config, tempFolder ); diff --git a/processing/src/test/java/org/apache/druid/query/groupby/NestedGroupByArrayQueryTest.java b/processing/src/test/java/org/apache/druid/query/groupby/NestedGroupByArrayQueryTest.java index d305cb95444a..f8344a7f5d6e 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/NestedGroupByArrayQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/NestedGroupByArrayQueryTest.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.io.Closer; @@ -74,10 +74,10 @@ public NestedGroupByArrayQueryTest( String vectorize ) { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); this.vectorize = QueryContexts.Vectorize.fromString(vectorize); this.helper = AggregationTestHelper.createGroupByQueryAggregationTestHelper( - NestedDataModule.getJacksonModulesList(), + DefaultColumnFormatConfigModule.getJacksonModulesList(), config, tempFolder ); diff --git a/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/column/NestedColumnGroupByColumnSelectorStrategyTest.java b/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/column/NestedColumnGroupByColumnSelectorStrategyTest.java index a35432cc80e3..dda7e60eb34a 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/column/NestedColumnGroupByColumnSelectorStrategyTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/column/NestedColumnGroupByColumnSelectorStrategyTest.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.query.IterableRowsCursorHelper; import org.apache.druid.query.groupby.ResultRow; import org.apache.druid.query.groupby.epinephelinae.GroupByColumnSelectorStrategyFactory; @@ -44,7 +44,7 @@ public class NestedColumnGroupByColumnSelectorStrategyTest extends InitializedNullHandlingTest { static { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); } private static final GroupByColumnSelectorStrategyFactory STRATEGY_FACTORY = new GroupByColumnSelectorStrategyFactory(); diff --git a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java index 64c68c943f52..b67c16c5fad0 100644 --- a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.druid.common.config.NullHandling; import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Sequence; @@ -78,8 +78,8 @@ public void teardown() throws IOException public NestedDataScanQueryTest() { - NestedDataModule.registerHandlersAndSerde(); - List mods = NestedDataModule.getJacksonModulesList(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + List mods = DefaultColumnFormatConfigModule.getJacksonModulesList(); this.helper = AggregationTestHelper.createScanQueryAggregationTestHelper(mods, tempFolder); this.closer = Closer.create(); } diff --git a/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java b/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java index 815a6ed9951f..dbf326e0cc8f 100644 --- a/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.io.Closer; @@ -98,7 +98,7 @@ public NestedDataTimeseriesQueryTest( ) { this.helper = AggregationTestHelper.createTimeseriesQueryAggregationTestHelper( - NestedDataModule.getJacksonModulesList(), + DefaultColumnFormatConfigModule.getJacksonModulesList(), tempFolder ); this.segmentsGenerator = segmentsGenerator; diff --git a/processing/src/test/java/org/apache/druid/query/topn/NestedDataTopNQueryTest.java b/processing/src/test/java/org/apache/druid/query/topn/NestedDataTopNQueryTest.java index 392df70e1d7c..6a092bcb926d 100644 --- a/processing/src/test/java/org/apache/druid/query/topn/NestedDataTopNQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/topn/NestedDataTopNQueryTest.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; @@ -72,9 +72,9 @@ public NestedDataTopNQueryTest( BiFunction> segmentGenerator ) { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); this.helper = AggregationTestHelper.createTopNQueryAggregationTestHelper( - NestedDataModule.getJacksonModulesList(), + DefaultColumnFormatConfigModule.getJacksonModulesList(), tempFolder ); this.segmentsGenerator = segmentGenerator; diff --git a/processing/src/test/java/org/apache/druid/segment/AutoTypeColumnIndexerTest.java b/processing/src/test/java/org/apache/druid/segment/AutoTypeColumnIndexerTest.java index e397267907e6..d4db932822fc 100644 --- a/processing/src/test/java/org/apache/druid/segment/AutoTypeColumnIndexerTest.java +++ b/processing/src/test/java/org/apache/druid/segment/AutoTypeColumnIndexerTest.java @@ -25,7 +25,7 @@ import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Sequence; @@ -62,7 +62,7 @@ public class AutoTypeColumnIndexerTest extends InitializedNullHandlingTest @BeforeClass public static void setup() { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); } @Test diff --git a/processing/src/test/java/org/apache/druid/segment/NestedDataColumnIndexerV4Test.java b/processing/src/test/java/org/apache/druid/segment/NestedDataColumnIndexerV4Test.java index e670ae10003d..9cf1c9a25b8a 100644 --- a/processing/src/test/java/org/apache/druid/segment/NestedDataColumnIndexerV4Test.java +++ b/processing/src/test/java/org/apache/druid/segment/NestedDataColumnIndexerV4Test.java @@ -25,7 +25,7 @@ import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Sequence; @@ -61,7 +61,7 @@ public class NestedDataColumnIndexerV4Test extends InitializedNullHandlingTest @BeforeClass public static void setup() { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); } @Test diff --git a/processing/src/test/java/org/apache/druid/segment/data/IncrementalIndexTest.java b/processing/src/test/java/org/apache/druid/segment/data/IncrementalIndexTest.java index 4a104510f8fd..5e6cdb6e1696 100644 --- a/processing/src/test/java/org/apache/druid/segment/data/IncrementalIndexTest.java +++ b/processing/src/test/java/org/apache/druid/segment/data/IncrementalIndexTest.java @@ -33,7 +33,7 @@ import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.Row; import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; @@ -104,7 +104,7 @@ public class IncrementalIndexTest extends InitializedNullHandlingTest public IncrementalIndexTest(String indexType, String mode, boolean isPreserveExistingMetrics) throws JsonProcessingException { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); this.isPreserveExistingMetrics = isPreserveExistingMetrics; indexCreator = closer.closeLater(new IncrementalIndexCreator(indexType, (builder, args) -> builder .setSimpleTestingIndexSchema("rollup".equals(mode), isPreserveExistingMetrics, (AggregatorFactory[]) args[0]) diff --git a/processing/src/test/java/org/apache/druid/segment/filter/ArrayContainsElementFilterTests.java b/processing/src/test/java/org/apache/druid/segment/filter/ArrayContainsElementFilterTests.java index 2418941bf73e..de78465e66fc 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/ArrayContainsElementFilterTests.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/ArrayContainsElementFilterTests.java @@ -27,7 +27,7 @@ import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.common.config.NullHandling; import org.apache.druid.error.DruidException; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.filter.ArrayContainsElementFilter; @@ -1071,7 +1071,7 @@ public void testGetCacheKey() Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); f1 = new ArrayContainsElementFilter( "x", ColumnType.NESTED_DATA, diff --git a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java index d9eb693ab76d..365de176a694 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java @@ -41,7 +41,7 @@ import org.apache.druid.frame.FrameType; import org.apache.druid.frame.segment.FrameSegment; import org.apache.druid.frame.segment.FrameStorageAdapter; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; @@ -445,7 +445,7 @@ public BaseFilterTest( @Before public void setUp() throws Exception { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); String className = getClass().getName(); Map> adaptersForClass = adapterCache.get().get(className); if (adaptersForClass == null) { diff --git a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java index fd87969a042d..0a96c2351630 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java @@ -30,7 +30,7 @@ import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.common.config.NullHandling; import org.apache.druid.error.DruidException; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; import org.apache.druid.math.expr.ExprEval; @@ -1760,7 +1760,7 @@ public void testGetCacheKey() Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); f1 = new EqualityFilter("x", ColumnType.NESTED_DATA, ImmutableMap.of("x", ImmutableList.of(1, 2, 3)), null); f1_2 = new EqualityFilter("x", ColumnType.NESTED_DATA, ImmutableMap.of("x", ImmutableList.of(1, 2, 3)), null); f2 = new EqualityFilter("x", ColumnType.NESTED_DATA, ImmutableMap.of("x", ImmutableList.of(1, 2, 3, 4)), null); diff --git a/processing/src/test/java/org/apache/druid/segment/generator/SegmentGenerator.java b/processing/src/test/java/org/apache/druid/segment/generator/SegmentGenerator.java index 3c7ff2389c66..537039f9a29f 100644 --- a/processing/src/test/java/org/apache/druid/segment/generator/SegmentGenerator.java +++ b/processing/src/test/java/org/apache/druid/segment/generator/SegmentGenerator.java @@ -26,7 +26,7 @@ import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.MapInputRowParser; import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; @@ -140,7 +140,7 @@ public QueryableIndex generate( { // In case we need to generate hyperUniques or json ComplexMetrics.registerSerde(HyperUniquesSerde.TYPE_NAME, new HyperUniquesSerde()); - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); final String dataHash = Hashing.sha256() .newHasher() diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexIngestionTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexIngestionTest.java index 4ef0a0c69d9c..931f72b95212 100644 --- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexIngestionTest.java +++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexIngestionTest.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.ImmutableMap; import org.apache.druid.data.input.MapBasedInputRow; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.LongMaxAggregator; @@ -50,7 +50,7 @@ public class IncrementalIndexIngestionTest extends InitializedNullHandlingTest public IncrementalIndexIngestionTest(String indexType) throws JsonProcessingException { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); indexCreator = closer.closeLater(new IncrementalIndexCreator(indexType, (builder, args) -> builder .setIndexSchema((IncrementalIndexSchema) args[0]) .setMaxRowCount(MAX_ROWS) diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java index b980c0251b6a..51457cca268f 100644 --- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java +++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java @@ -27,7 +27,7 @@ import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.CloserRule; @@ -56,7 +56,7 @@ public class IncrementalIndexMultiValueSpecTest extends InitializedNullHandlingT public IncrementalIndexMultiValueSpecTest(String indexType) throws JsonProcessingException { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); indexCreator = closer.closeLater(new IncrementalIndexCreator(indexType, (builder, args) -> builder .setIndexSchema((IncrementalIndexSchema) args[0]) .setMaxRowCount(10_000) diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java index 0d0f979f715b..e5b850627557 100644 --- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java @@ -27,7 +27,7 @@ import org.apache.druid.collections.ResourceHolder; import org.apache.druid.common.config.NullHandling; import org.apache.druid.data.input.MapBasedInputRow; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; @@ -97,7 +97,7 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT public IncrementalIndexStorageAdapterTest(String indexType) throws JsonProcessingException { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); indexCreator = closer.closeLater(new IncrementalIndexCreator(indexType, (builder, args) -> builder .setSimpleTestingIndexSchema(new CountAggregatorFactory("cnt")) .setMaxRowCount(1_000) diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java index cd85768e5acb..ed2996731777 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java @@ -29,7 +29,7 @@ import org.apache.druid.collections.bitmap.ImmutableBitmap; import org.apache.druid.collections.bitmap.WrappedRoaringBitmap; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.io.smoosh.FileSmoosher; @@ -149,7 +149,7 @@ public class NestedDataColumnSupplierTest extends InitializedNullHandlingTest @BeforeClass public static void staticSetup() { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); } @Before diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java index bcc40b333c42..a53ee99f8d17 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java @@ -28,7 +28,7 @@ import com.google.common.util.concurrent.MoreExecutors; import org.apache.druid.collections.bitmap.RoaringBitmapFactory; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.io.smoosh.FileSmoosher; @@ -141,7 +141,7 @@ public class NestedDataColumnSupplierV4Test extends InitializedNullHandlingTest @BeforeClass public static void staticSetup() { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); } @Before diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnSelectorsTest.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnSelectorsTest.java index 86455f12c6e8..e99d14522ec7 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnSelectorsTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnSelectorsTest.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.databind.Module; import com.google.common.collect.ImmutableList; import org.apache.druid.error.DruidException; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Sequence; @@ -81,8 +81,8 @@ public class NestedFieldColumnSelectorsTest extends InitializedNullHandlingTest public NestedFieldColumnSelectorsTest() { - NestedDataModule.registerHandlersAndSerde(); - List mods = NestedDataModule.getJacksonModulesList(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + List mods = DefaultColumnFormatConfigModule.getJacksonModulesList(); this.helper = AggregationTestHelper.createScanQueryAggregationTestHelper( mods, tempFolder diff --git a/processing/src/test/java/org/apache/druid/segment/nested/ScalarDoubleColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/ScalarDoubleColumnSupplierTest.java index 5d76e18f94c1..ca30d3fdb12c 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/ScalarDoubleColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/ScalarDoubleColumnSupplierTest.java @@ -26,7 +26,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.io.smoosh.FileSmoosher; @@ -105,7 +105,7 @@ public class ScalarDoubleColumnSupplierTest extends InitializedNullHandlingTest @BeforeClass public static void staticSetup() { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); } @Before diff --git a/processing/src/test/java/org/apache/druid/segment/nested/ScalarLongColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/ScalarLongColumnSupplierTest.java index 4661d72cdbb7..902aeaae2276 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/ScalarLongColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/ScalarLongColumnSupplierTest.java @@ -26,7 +26,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.io.smoosh.FileSmoosher; @@ -105,7 +105,7 @@ public class ScalarLongColumnSupplierTest extends InitializedNullHandlingTest @BeforeClass public static void staticSetup() { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); } @Before diff --git a/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java index 5f0c1d020c83..19b9f0a13919 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java @@ -26,7 +26,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.io.smoosh.FileSmoosher; @@ -105,7 +105,7 @@ public class ScalarStringColumnSupplierTest extends InitializedNullHandlingTest @BeforeClass public static void staticSetup() { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); } @Before diff --git a/processing/src/test/java/org/apache/druid/segment/nested/VariantColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/VariantColumnSupplierTest.java index 6ea4509bef7a..4b572a4236cb 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/VariantColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/VariantColumnSupplierTest.java @@ -24,7 +24,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.io.smoosh.FileSmoosher; @@ -167,7 +167,7 @@ public class VariantColumnSupplierTest extends InitializedNullHandlingTest @BeforeClass public static void staticSetup() { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); } @Parameterized.Parameters(name = "data = {0}") diff --git a/server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java b/server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java index 05b2f90e3e76..d4989d05f562 100644 --- a/server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java +++ b/server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java @@ -25,6 +25,7 @@ import org.apache.druid.discovery.NodeRole; import org.apache.druid.guice.AnnouncerModule; import org.apache.druid.guice.CoordinatorDiscoveryModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.guice.DruidInjectorBuilder; import org.apache.druid.guice.DruidSecondaryModule; import org.apache.druid.guice.ExpressionModule; @@ -34,12 +35,10 @@ import org.apache.druid.guice.LifecycleModule; import org.apache.druid.guice.LocalDataStorageDruidModule; import org.apache.druid.guice.MetadataConfigModule; -import org.apache.druid.guice.NestedDataModule; import org.apache.druid.guice.ServerModule; import org.apache.druid.guice.ServerViewModule; import org.apache.druid.guice.StartupLoggingModule; import org.apache.druid.guice.StorageNodeModule; -import org.apache.druid.guice.StringMultiValueHandlingModule; import org.apache.druid.guice.annotations.Client; import org.apache.druid.guice.annotations.EscalatedClient; import org.apache.druid.guice.http.HttpClientModule; @@ -114,7 +113,7 @@ public CoreInjectorBuilder forServer() new StorageNodeModule(), new JettyServerModule(), new ExpressionModule(), - new NestedDataModule(), + new DefaultColumnFormatConfigModule(), new DiscoveryModule(), new ServerViewModule(), new MetadataConfigModule(), @@ -132,8 +131,7 @@ public CoreInjectorBuilder forServer() new StartupLoggingModule(), new ExternalStorageAccessSecurityModule(), new ServiceClientModule(), - new StorageConnectorModule(), - new StringMultiValueHandlingModule() + new StorageConnectorModule() ); return this; } diff --git a/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java b/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java index 83ca668acf93..21d5a555d36e 100644 --- a/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java +++ b/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java @@ -31,7 +31,7 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.guice.StartupInjectorBuilder; import org.apache.druid.guice.annotations.Json; import org.apache.druid.initialization.ServerInjectorBuilder; @@ -84,7 +84,7 @@ public class DumpSegmentTest extends InitializedNullHandlingTest public DumpSegmentTest() { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); this.closer = Closer.create(); } @@ -156,7 +156,7 @@ public void testDumpNestedColumn() throws Exception { Injector injector = Mockito.mock(Injector.class); ObjectMapper mapper = TestHelper.makeJsonMapper(); - mapper.registerModules(NestedDataModule.getJacksonModulesList()); + mapper.registerModules(DefaultColumnFormatConfigModule.getJacksonModulesList()); mapper.setInjectableValues( new InjectableValues.Std() .addValue(ExprMacroTable.class.getName(), TestExprMacroTable.INSTANCE) @@ -196,7 +196,7 @@ public void testDumpNestedColumnPath() throws Exception { Injector injector = Mockito.mock(Injector.class); ObjectMapper mapper = TestHelper.makeJsonMapper(); - mapper.registerModules(NestedDataModule.getJacksonModulesList()); + mapper.registerModules(DefaultColumnFormatConfigModule.getJacksonModulesList()); mapper.setInjectableValues( new InjectableValues.Std() .addValue(ExprMacroTable.class.getName(), TestExprMacroTable.INSTANCE) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index 2fe1919f6a16..12cfb78b6489 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -25,7 +25,7 @@ import org.apache.calcite.avatica.SqlType; import org.apache.druid.common.config.NullHandling; import org.apache.druid.guice.DruidInjectorBuilder; -import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.java.util.common.HumanReadableBytes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; @@ -130,7 +130,7 @@ public ArraysComponentSupplier(TempDirProducer tempFolderProducer) public void configureGuice(DruidInjectorBuilder builder) { super.configureGuice(builder); - builder.addModule(new NestedDataModule()); + builder.addModule(new DefaultColumnFormatConfigModule()); } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java index 9e6e5da2bb0d..6d769dd6faf8 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java @@ -34,8 +34,8 @@ import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.error.DruidException; import org.apache.druid.error.DruidExceptionMatcher; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.guice.DruidInjectorBuilder; -import org.apache.druid.guice.NestedDataModule; import org.apache.druid.java.util.common.HumanReadableBytes; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.math.expr.ExprMacroTable; @@ -196,7 +196,7 @@ public NestedComponentSupplier(TempDirProducer tempFolderProducer) public void configureGuice(DruidInjectorBuilder builder) { super.configureGuice(builder); - builder.addModule(new NestedDataModule()); + builder.addModule(new DefaultColumnFormatConfigModule()); } @SuppressWarnings("resource") @@ -207,7 +207,7 @@ public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker( final Injector injector ) { - NestedDataModule.registerHandlersAndSerde(); + DefaultColumnFormatConfigModule.registerHandlersAndSerde(); final QueryableIndex index = IndexBuilder.create() .tmpDir(tempDirProducer.newTempFolder()) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTestBase.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTestBase.java index babd9fe60afc..9a69c62dc734 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTestBase.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTestBase.java @@ -21,7 +21,6 @@ import com.google.common.collect.ImmutableList; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.StringMultiValueHandlingModule; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.math.expr.ExpressionProcessing; import org.apache.druid.segment.column.ColumnType; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index f3f8bacbb012..8eb9fb9b04e4 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -26,12 +26,12 @@ import com.google.inject.Injector; import com.google.inject.Module; import com.google.inject.Provides; +import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.guice.DruidInjectorBuilder; import org.apache.druid.guice.ExpressionModule; import org.apache.druid.guice.LazySingleton; import org.apache.druid.guice.SegmentWranglerModule; import org.apache.druid.guice.StartupInjectorBuilder; -import org.apache.druid.guice.StringMultiValueHandlingModule; import org.apache.druid.initialization.CoreInjectorBuilder; import org.apache.druid.initialization.DruidModule; import org.apache.druid.initialization.ServiceInjectorBuilder; @@ -621,7 +621,7 @@ private SqlTestFramework(Builder builder) .addModule(new SqlAggregationModule()) .addModule(new ExpressionModule()) .addModule(new TestSetupModule(builder)) - .addModule(new StringMultiValueHandlingModule()); + .addModule(new DefaultColumnFormatConfigModule()); builder.componentSupplier.configureGuice(injectorBuilder); From e14cbd89ee51a3cb18701f8309c567a4107a6705 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 1 Aug 2024 17:06:52 -0700 Subject: [PATCH 10/14] One side effect to rule 'em all. --- .../DefaultColumnFormatConfigModule.java | 85 +++++++------------ .../DefaultColumnFormatConfigModuleTest.java | 10 +-- 2 files changed, 35 insertions(+), 60 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/guice/DefaultColumnFormatConfigModule.java b/processing/src/main/java/org/apache/druid/guice/DefaultColumnFormatConfigModule.java index 3172dcb420dc..7cb98388b931 100644 --- a/processing/src/main/java/org/apache/druid/guice/DefaultColumnFormatConfigModule.java +++ b/processing/src/main/java/org/apache/druid/guice/DefaultColumnFormatConfigModule.java @@ -44,6 +44,13 @@ public class DefaultColumnFormatConfigModule implements DruidModule { + /** + * Initialized with a default value so tests can just get it via {@link #getStringMultiValueHandlingMode} without any + * explicit initialization. In production, this default may be overridden if a value is configured via + * {@link #initDimensionHandlerAndMvHandlingMode(DefaultColumnFormatConfig)}. + */ + private static DimensionSchema.MultiValueHandling STRING_MV_MODE = DimensionSchema.MultiValueHandling.SORTED_ARRAY; + @Override public List getJacksonModules() { @@ -54,17 +61,15 @@ public List getJacksonModules() public void configure(Binder binder) { registerSerde(); - // binding our side effect classes to the lifecycle causes the registerDimensionHandler and - // setStringMultiValueHandlingMode methods to be called on service start, allowing use of the config to get the - // system default format version and string multi value handling mode. We need to register them as two different side - // effect classes to make Guice initialization happy. - LifecycleModule.register(binder, SideEffectDimensionHandlerRegisterer.class); - LifecycleModule.register(binder, SideEffectMultiValueHandlingRegisterer.class); + // binding our side effect classes to the lifecycle causes the initDimensionHandlerAndMvHandlingMode to be + // called on service start, allowing use of the config to get the system default format version and string multi + // value handling mode. + LifecycleModule.register(binder, SideEffectRegisterer.class); } @Provides @LazySingleton - public SideEffectDimensionHandlerRegisterer registerDimensionHandler(DefaultColumnFormatConfig formatsConfig) + public static SideEffectRegisterer initDimensionHandlerAndMvHandlingMode(DefaultColumnFormatConfig formatsConfig) { if (formatsConfig.getNestedColumnFormatVersion() != null && formatsConfig.getNestedColumnFormatVersion() == 4) { DimensionHandlerUtils.registerDimensionHandlerProvider( @@ -77,7 +82,20 @@ public SideEffectDimensionHandlerRegisterer registerDimensionHandler(DefaultColu new NestedCommonFormatHandlerProvider() ); } - return new SideEffectDimensionHandlerRegisterer(); + + if (formatsConfig.getStringMultiValueHandlingMode() != null) { + STRING_MV_MODE = DimensionSchema.MultiValueHandling.fromString(formatsConfig.getStringMultiValueHandlingMode()); + } + return new SideEffectRegisterer(); + } + + /** + * @return the configured string multi value handling mode from the system config if set; otherwise, returns + * the default. + */ + public static DimensionSchema.MultiValueHandling getStringMultiValueHandlingMode() + { + return STRING_MV_MODE; } public static List getJacksonModulesList() @@ -132,54 +150,13 @@ public DimensionHandler get(Stri } /** - * The default value is set here so tests can just use it without any explicit initialization. - * In production, this default may be overridden if a value is configured via - * {@link #setStringMultiValueHandlingMode(DefaultColumnFormatConfig)}. - */ - private static DimensionSchema.MultiValueHandling STRING_MV_MODE = DimensionSchema.MultiValueHandling.SORTED_ARRAY; - - @Provides - @LazySingleton - public static SideEffectMultiValueHandlingRegisterer setStringMultiValueHandlingMode(DefaultColumnFormatConfig formatsConfig) - { - if (formatsConfig.getStringMultiValueHandlingMode() != null) { - STRING_MV_MODE = DimensionSchema.MultiValueHandling.fromString(formatsConfig.getStringMultiValueHandlingMode()); - } - return new SideEffectMultiValueHandlingRegisterer(); - } - - /** - * @return the configured string multi value handling mode from the system config if set; otherwise, returns - * the default. - */ - public static DimensionSchema.MultiValueHandling getStringMultiValueHandlingMode() - { - return STRING_MV_MODE; - } - - /** - * this is used as a vehicle to register the correct version of the system default nested column handler by side - * effect with the help of binding to {@link org.apache.druid.java.util.common.lifecycle.Lifecycle} so that - * {@link #registerDimensionHandler(DefaultColumnFormatConfig)} can be called with the injected + * this is used as a vehicle to register the correct version of the system default nested column handler and multi + * value handling mode by side effect with the help of binding to + * {@link org.apache.druid.java.util.common.lifecycle.Lifecycle} so that + * {@link #initDimensionHandlerAndMvHandlingMode(DefaultColumnFormatConfig)} can be called with the injected * {@link DefaultColumnFormatConfig}. */ - public static class SideEffectDimensionHandlerRegisterer - { - // nothing to see here - } - - /** - * This is used as a vehicle to register the correct multi value handling mode by side effect with the help of - * binding to {@link org.apache.druid.java.util.common.lifecycle.Lifecycle} so that - * {@link #setStringMultiValueHandlingMode(DefaultColumnFormatConfig)} can be called with the injected - * {@link DefaultColumnFormatConfig}. - * - *

- * Similar purpose as {@link SideEffectDimensionHandlerRegisterer}, but we need separate static classes to make - * Guice initialization happy. - *

- */ - public static class SideEffectMultiValueHandlingRegisterer + public static class SideEffectRegisterer { // nothing to see here } diff --git a/processing/src/test/java/org/apache/druid/guice/DefaultColumnFormatConfigModuleTest.java b/processing/src/test/java/org/apache/druid/guice/DefaultColumnFormatConfigModuleTest.java index b310b37977e3..cce906736a50 100644 --- a/processing/src/test/java/org/apache/druid/guice/DefaultColumnFormatConfigModuleTest.java +++ b/processing/src/test/java/org/apache/druid/guice/DefaultColumnFormatConfigModuleTest.java @@ -70,8 +70,7 @@ public void testDefaults() Injector gadget = makeInjector(props); // side effects - gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectDimensionHandlerRegisterer.class); - gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectMultiValueHandlingRegisterer.class); + gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectRegisterer.class); DimensionHandlerProvider provider = DimensionHandlerUtils.DIMENSION_HANDLER_PROVIDERS.get( NestedDataComplexTypeSerde.TYPE_NAME @@ -94,8 +93,7 @@ public void testOverride() Injector gadget = makeInjector(props); // side effects - gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectDimensionHandlerRegisterer.class); - gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectMultiValueHandlingRegisterer.class); + gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectRegisterer.class); DimensionHandlerProvider provider = DimensionHandlerUtils.DIMENSION_HANDLER_PROVIDERS.get( NestedDataComplexTypeSerde.TYPE_NAME @@ -115,7 +113,7 @@ public void testOverrideMultiValueHandlingModeCaseInsensitive() props.setProperty("druid.indexing.formats.stringMultiValueHandlingMode", "ARRAY"); final Injector gadget = makeInjector(props); - gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectMultiValueHandlingRegisterer.class); + gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectRegisterer.class); Assert.assertEquals( DimensionSchema.MultiValueHandling.ARRAY, @@ -132,7 +130,7 @@ public void testInvalidMultiValueHandlingMode() final Exception exception = Assert.assertThrows( Exception.class, - () -> gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectMultiValueHandlingRegisterer.class) + () -> gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectRegisterer.class) ); Assert.assertTrue(exception.getMessage().contains( "Invalid value[boo] specified for 'druid.indexing.formats.stringMultiValueHandlingMode'." From c13b448634dd0772cb428fe71d9b4d0bf90972a0 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 1 Aug 2024 17:20:43 -0700 Subject: [PATCH 11/14] fix checkstyle error --- .../org/apache/druid/sql/calcite/CalciteArraysQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index 12cfb78b6489..9ad04e0ba497 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -24,8 +24,8 @@ import com.google.common.collect.ImmutableSet; import org.apache.calcite.avatica.SqlType; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.DruidInjectorBuilder; import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.DruidInjectorBuilder; import org.apache.druid.java.util.common.HumanReadableBytes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; From ab05d0649db3a84da9f5b17f4e07b23d3d9989c2 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 1 Aug 2024 17:48:53 -0700 Subject: [PATCH 12/14] remove extraneous module init & checkstyle --- .../org/apache/druid/sql/calcite/util/SqlTestFramework.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 8eb9fb9b04e4..5d710600a11c 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -26,7 +26,6 @@ import com.google.inject.Injector; import com.google.inject.Module; import com.google.inject.Provides; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.guice.DruidInjectorBuilder; import org.apache.druid.guice.ExpressionModule; import org.apache.druid.guice.LazySingleton; @@ -620,9 +619,7 @@ private SqlTestFramework(Builder builder) .addModule(new SegmentWranglerModule()) .addModule(new SqlAggregationModule()) .addModule(new ExpressionModule()) - .addModule(new TestSetupModule(builder)) - .addModule(new DefaultColumnFormatConfigModule()); - + .addModule(new TestSetupModule(builder)); builder.componentSupplier.configureGuice(injectorBuilder); ServiceInjectorBuilder serviceInjector = new ServiceInjectorBuilder(injectorBuilder); From b2f3d3ef2e2b5ae457f7c64433b1fb11598ba714 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Thu, 1 Aug 2024 18:11:15 -0700 Subject: [PATCH 13/14] DefaultColumnFormatConfigModule -> BuiltInTypesModule and rename test DefaultColumnFormatConfigModuleTest -> BuiltInTypesModuleTest --- .../GroupByDeserializationBenchmark.java | 6 +++--- .../frame/FrameChannelMergerBenchmark.java | 4 ++-- .../benchmark/query/SqlGroupByBenchmark.java | 6 +++--- .../druid/msq/exec/MSQComplexGroupByTest.java | 6 +++--- .../org/apache/druid/msq/test/MSQTestBase.java | 8 ++++---- .../druid/data/input/impl/DimensionSchema.java | 4 ++-- ...nfigModule.java => BuiltInTypesModule.java} | 2 +- .../nested/NestedDataComplexTypeSerde.java | 4 ++-- .../druid/frame/write/FrameWriterTest.java | 4 ++-- ...leTest.java => BuiltInTypesModuleTest.java} | 18 +++++++++--------- .../apache/druid/math/expr/FunctionTest.java | 4 ++-- .../druid/query/NestedDataTestUtils.java | 4 ++-- .../groupby/NestedDataGroupByQueryTest.java | 6 +++--- .../groupby/NestedGroupByArrayQueryTest.java | 6 +++--- ...olumnGroupByColumnSelectorStrategyTest.java | 4 ++-- .../query/scan/NestedDataScanQueryTest.java | 6 +++--- .../NestedDataTimeseriesQueryTest.java | 4 ++-- .../query/topn/NestedDataTopNQueryTest.java | 6 +++--- .../segment/AutoTypeColumnIndexerTest.java | 4 ++-- .../segment/NestedDataColumnIndexerV4Test.java | 4 ++-- .../segment/data/IncrementalIndexTest.java | 4 ++-- .../ArrayContainsElementFilterTests.java | 4 ++-- .../druid/segment/filter/BaseFilterTest.java | 4 ++-- .../segment/filter/EqualityFilterTests.java | 4 ++-- .../segment/generator/SegmentGenerator.java | 4 ++-- .../IncrementalIndexIngestionTest.java | 4 ++-- .../IncrementalIndexMultiValueSpecTest.java | 4 ++-- .../IncrementalIndexStorageAdapterTest.java | 4 ++-- .../nested/NestedDataColumnSupplierTest.java | 4 ++-- .../nested/NestedDataColumnSupplierV4Test.java | 4 ++-- .../nested/NestedFieldColumnSelectorsTest.java | 6 +++--- .../nested/ScalarDoubleColumnSupplierTest.java | 4 ++-- .../nested/ScalarLongColumnSupplierTest.java | 4 ++-- .../nested/ScalarStringColumnSupplierTest.java | 4 ++-- .../nested/VariantColumnSupplierTest.java | 4 ++-- .../initialization/CoreInjectorBuilder.java | 4 ++-- .../org/apache/druid/cli/DumpSegmentTest.java | 8 ++++---- .../sql/calcite/CalciteArraysQueryTest.java | 4 ++-- .../calcite/CalciteNestedDataQueryTest.java | 6 +++--- 39 files changed, 97 insertions(+), 97 deletions(-) rename processing/src/main/java/org/apache/druid/guice/{DefaultColumnFormatConfigModule.java => BuiltInTypesModule.java} (98%) rename processing/src/test/java/org/apache/druid/guice/{DefaultColumnFormatConfigModuleTest.java => BuiltInTypesModuleTest.java} (88%) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/GroupByDeserializationBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/GroupByDeserializationBenchmark.java index a1430e13dd5b..ee1ec34dc3b0 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/GroupByDeserializationBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/GroupByDeserializationBenchmark.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.jackson.AggregatorsModule; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Pair; @@ -68,7 +68,7 @@ public class GroupByDeserializationBenchmark static { NullHandling.initializeForTests(); - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); AggregatorsModule.registerComplexMetricsAndSerde(); } @@ -93,7 +93,7 @@ public class GroupByDeserializationBenchmark public void setup() throws JsonProcessingException { final ObjectMapper undecoratedMapper = TestHelper.makeJsonMapper(); - undecoratedMapper.registerModules(DefaultColumnFormatConfigModule.getJacksonModulesList()); + undecoratedMapper.registerModules(BuiltInTypesModule.getJacksonModulesList()); undecoratedMapper.registerModule(new AggregatorsModule()); final Pair sqlQueryAndResultRow = sqlQueryAndResultRow( numDimensions, diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/frame/FrameChannelMergerBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/frame/FrameChannelMergerBenchmark.java index f3292b35f54b..a8b90e56eea1 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/frame/FrameChannelMergerBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/frame/FrameChannelMergerBenchmark.java @@ -37,7 +37,7 @@ import org.apache.druid.frame.read.FrameReader; import org.apache.druid.frame.testutil.FrameSequenceBuilder; import org.apache.druid.frame.write.FrameWriters; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.NonnullPair; @@ -85,7 +85,7 @@ public class FrameChannelMergerBenchmark { static { NullHandling.initializeForTests(); - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); } private static final String KEY = "key"; diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlGroupByBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlGroupByBenchmark.java index 453a69e20273..bf521b83163b 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlGroupByBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlGroupByBenchmark.java @@ -24,7 +24,7 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.data.input.impl.DimensionSchema; import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Sequence; @@ -89,7 +89,7 @@ public class SqlGroupByBenchmark static { NullHandling.initializeForTests(); ExpressionProcessing.initializeForTests(); - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); } private static final Logger log = new Logger(SqlGroupByBenchmark.class); @@ -331,7 +331,7 @@ public void setup() // Hacky and pollutes global namespace, but it is fine since benchmarks are run in isolation. Wasn't able // to work up a cleaner way of doing it by modifying the injector. - CalciteTests.getJsonMapper().registerModules(DefaultColumnFormatConfigModule.getJacksonModulesList()); + CalciteTests.getJsonMapper().registerModules(BuiltInTypesModule.getJacksonModulesList()); final DruidSchemaCatalog rootSchema = CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER); diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java index 3e0240732871..8968d1fbc0ff 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQComplexGroupByTest.java @@ -25,7 +25,7 @@ import org.apache.druid.data.input.impl.JsonInputFormat; import org.apache.druid.data.input.impl.LocalInputSource; import org.apache.druid.data.input.impl.systemfield.SystemFields; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.msq.indexing.MSQSpec; import org.apache.druid.msq.indexing.MSQTuningConfig; @@ -64,7 +64,7 @@ public class MSQComplexGroupByTest extends MSQTestBase { static { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); } private String dataFileNameJsonString; @@ -109,7 +109,7 @@ public void setup() throws IOException dataFileSignature ); - objectMapper.registerModules(DefaultColumnFormatConfigModule.getJacksonModulesList()); + objectMapper.registerModules(BuiltInTypesModule.getJacksonModulesList()); } @MethodSource("data") diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java index 770d1fd833c7..be65d558f8d3 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java @@ -48,7 +48,7 @@ import org.apache.druid.frame.channel.FrameChannelSequence; import org.apache.druid.frame.processor.Bouncer; import org.apache.druid.frame.testutil.FrameTestUtil; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.guice.DruidInjectorBuilder; import org.apache.druid.guice.DruidSecondaryModule; import org.apache.druid.guice.ExpressionModule; @@ -357,8 +357,8 @@ public void configure(Binder binder) { // We want this module to bring InputSourceModule along for the ride. binder.install(new InputSourceModule()); - binder.install(new DefaultColumnFormatConfigModule()); - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + binder.install(new BuiltInTypesModule()); + BuiltInTypesModule.registerHandlersAndSerde(); SqlBindings.addOperatorConversion(binder, ExternalOperatorConversion.class); SqlBindings.addOperatorConversion(binder, HttpOperatorConversion.class); SqlBindings.addOperatorConversion(binder, InlineOperatorConversion.class); @@ -521,7 +521,7 @@ public String getFormatString() objectMapper = setupObjectMapper(injector); objectMapper.registerModules(new StorageConnectorModule().getJacksonModules()); objectMapper.registerModules(sqlModule.getJacksonModules()); - objectMapper.registerModules(DefaultColumnFormatConfigModule.getJacksonModulesList()); + objectMapper.registerModules(BuiltInTypesModule.getJacksonModulesList()); doReturn(mock(Request.class)).when(brokerClient).makeRequest(any(), anyString()); diff --git a/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java b/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java index 08b5d2775a29..b5933e722d8c 100644 --- a/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java +++ b/processing/src/main/java/org/apache/druid/data/input/impl/DimensionSchema.java @@ -26,7 +26,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.annotation.JsonValue; import com.google.common.base.Strings; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.guice.annotations.PublicApi; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.emitter.EmittingLogger; @@ -113,7 +113,7 @@ public static MultiValueHandling fromString(String name) public static MultiValueHandling ofDefault() { - return DefaultColumnFormatConfigModule.getStringMultiValueHandlingMode(); + return BuiltInTypesModule.getStringMultiValueHandlingMode(); } } diff --git a/processing/src/main/java/org/apache/druid/guice/DefaultColumnFormatConfigModule.java b/processing/src/main/java/org/apache/druid/guice/BuiltInTypesModule.java similarity index 98% rename from processing/src/main/java/org/apache/druid/guice/DefaultColumnFormatConfigModule.java rename to processing/src/main/java/org/apache/druid/guice/BuiltInTypesModule.java index 7cb98388b931..36e5cd6f0205 100644 --- a/processing/src/main/java/org/apache/druid/guice/DefaultColumnFormatConfigModule.java +++ b/processing/src/main/java/org/apache/druid/guice/BuiltInTypesModule.java @@ -42,7 +42,7 @@ import java.util.Collections; import java.util.List; -public class DefaultColumnFormatConfigModule implements DruidModule +public class BuiltInTypesModule implements DruidModule { /** * Initialized with a default value so tests can just get it via {@link #getStringMultiValueHandlingMode} without any diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataComplexTypeSerde.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataComplexTypeSerde.java index c22e76fc00b6..56a66c64c066 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataComplexTypeSerde.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataComplexTypeSerde.java @@ -25,7 +25,7 @@ import com.fasterxml.jackson.dataformat.smile.SmileGenerator; import it.unimi.dsi.fastutil.Hash; import org.apache.druid.data.input.impl.DimensionSchema; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.guava.Comparators; @@ -62,7 +62,7 @@ public class NestedDataComplexTypeSerde extends ComplexMetricSerde smileFactory.delegateToTextual(true); final ObjectMapper mapper = new DefaultObjectMapper(smileFactory, null); mapper.getFactory().setCodec(mapper); - mapper.registerModules(DefaultColumnFormatConfigModule.getJacksonModulesList()); + mapper.registerModules(BuiltInTypesModule.getJacksonModulesList()); OBJECT_MAPPER = mapper; } diff --git a/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java b/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java index d2cc9de10f76..c45ffa697350 100644 --- a/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java +++ b/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java @@ -37,7 +37,7 @@ import org.apache.druid.frame.segment.FrameSegment; import org.apache.druid.frame.segment.FrameStorageAdapter; import org.apache.druid.frame.testutil.FrameTestUtil; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.RE; @@ -92,7 +92,7 @@ public class FrameWriterTest extends InitializedNullHandlingTest static { ComplexMetrics.registerSerde(HyperUniquesSerde.TYPE_NAME, new HyperUniquesSerde()); - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); } private static final int DEFAULT_ALLOCATOR_CAPACITY = 1_000_000; diff --git a/processing/src/test/java/org/apache/druid/guice/DefaultColumnFormatConfigModuleTest.java b/processing/src/test/java/org/apache/druid/guice/BuiltInTypesModuleTest.java similarity index 88% rename from processing/src/test/java/org/apache/druid/guice/DefaultColumnFormatConfigModuleTest.java rename to processing/src/test/java/org/apache/druid/guice/BuiltInTypesModuleTest.java index cce906736a50..23a2d18f39e4 100644 --- a/processing/src/test/java/org/apache/druid/guice/DefaultColumnFormatConfigModuleTest.java +++ b/processing/src/test/java/org/apache/druid/guice/BuiltInTypesModuleTest.java @@ -36,7 +36,7 @@ import javax.annotation.Nullable; import java.util.Properties; -public class DefaultColumnFormatConfigModuleTest +public class BuiltInTypesModuleTest { @Nullable private static DimensionHandlerProvider DEFAULT_HANDLER_PROVIDER; @@ -70,7 +70,7 @@ public void testDefaults() Injector gadget = makeInjector(props); // side effects - gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectRegisterer.class); + gadget.getInstance(BuiltInTypesModule.SideEffectRegisterer.class); DimensionHandlerProvider provider = DimensionHandlerUtils.DIMENSION_HANDLER_PROVIDERS.get( NestedDataComplexTypeSerde.TYPE_NAME @@ -79,7 +79,7 @@ public void testDefaults() Assert.assertEquals( DimensionSchema.MultiValueHandling.SORTED_ARRAY, - DefaultColumnFormatConfigModule.getStringMultiValueHandlingMode() + BuiltInTypesModule.getStringMultiValueHandlingMode() ); } @@ -93,7 +93,7 @@ public void testOverride() Injector gadget = makeInjector(props); // side effects - gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectRegisterer.class); + gadget.getInstance(BuiltInTypesModule.SideEffectRegisterer.class); DimensionHandlerProvider provider = DimensionHandlerUtils.DIMENSION_HANDLER_PROVIDERS.get( NestedDataComplexTypeSerde.TYPE_NAME @@ -102,7 +102,7 @@ public void testOverride() Assert.assertEquals( DimensionSchema.MultiValueHandling.SORTED_ARRAY, - DefaultColumnFormatConfigModule.getStringMultiValueHandlingMode() + BuiltInTypesModule.getStringMultiValueHandlingMode() ); } @@ -113,11 +113,11 @@ public void testOverrideMultiValueHandlingModeCaseInsensitive() props.setProperty("druid.indexing.formats.stringMultiValueHandlingMode", "ARRAY"); final Injector gadget = makeInjector(props); - gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectRegisterer.class); + gadget.getInstance(BuiltInTypesModule.SideEffectRegisterer.class); Assert.assertEquals( DimensionSchema.MultiValueHandling.ARRAY, - DefaultColumnFormatConfigModule.getStringMultiValueHandlingMode() + BuiltInTypesModule.getStringMultiValueHandlingMode() ); } @@ -130,7 +130,7 @@ public void testInvalidMultiValueHandlingMode() final Exception exception = Assert.assertThrows( Exception.class, - () -> gadget.getInstance(DefaultColumnFormatConfigModule.SideEffectRegisterer.class) + () -> gadget.getInstance(BuiltInTypesModule.SideEffectRegisterer.class) ); Assert.assertTrue(exception.getMessage().contains( "Invalid value[boo] specified for 'druid.indexing.formats.stringMultiValueHandlingMode'." @@ -148,7 +148,7 @@ private Injector makeInjector(Properties props) binder -> { JsonConfigProvider.bind(binder, "druid.indexing.formats", DefaultColumnFormatConfig.class); }, - new DefaultColumnFormatConfigModule() + new BuiltInTypesModule() ) ); diff --git a/processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java b/processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java index 05e7d543e600..521c00f2f171 100644 --- a/processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java +++ b/processing/src/test/java/org/apache/druid/math/expr/FunctionTest.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableSet; import org.apache.druid.common.config.NullHandling; import org.apache.druid.error.DruidException; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.StringUtils; @@ -66,7 +66,7 @@ public static void setupClass() TypeStrategiesTest.NULLABLE_TEST_PAIR_TYPE.getComplexTypeName(), new TypeStrategiesTest.NullableLongPairTypeStrategy() ); - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); } @Before diff --git a/processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java b/processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java index c747f265b391..23215b95d1e4 100644 --- a/processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java +++ b/processing/src/test/java/org/apache/druid/query/NestedDataTestUtils.java @@ -31,7 +31,7 @@ import org.apache.druid.data.input.impl.JsonInputFormat; import org.apache.druid.data.input.impl.LocalInputSource; import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.Granularity; @@ -166,7 +166,7 @@ public class NestedDataTestUtils static { JSON_MAPPER = TestHelper.makeJsonMapper(); - JSON_MAPPER.registerModules(DefaultColumnFormatConfigModule.getJacksonModulesList()); + JSON_MAPPER.registerModules(BuiltInTypesModule.getJacksonModulesList()); } public static List createSimpleSegmentsTsv( diff --git a/processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java b/processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java index 5ec6d6e2cb29..6c0ff3b49511 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.io.Closer; @@ -82,10 +82,10 @@ public NestedDataGroupByQueryTest( String vectorize ) { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); this.vectorize = QueryContexts.Vectorize.fromString(vectorize); this.helper = AggregationTestHelper.createGroupByQueryAggregationTestHelper( - DefaultColumnFormatConfigModule.getJacksonModulesList(), + BuiltInTypesModule.getJacksonModulesList(), config, tempFolder ); diff --git a/processing/src/test/java/org/apache/druid/query/groupby/NestedGroupByArrayQueryTest.java b/processing/src/test/java/org/apache/druid/query/groupby/NestedGroupByArrayQueryTest.java index f8344a7f5d6e..f0e581291d36 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/NestedGroupByArrayQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/NestedGroupByArrayQueryTest.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.io.Closer; @@ -74,10 +74,10 @@ public NestedGroupByArrayQueryTest( String vectorize ) { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); this.vectorize = QueryContexts.Vectorize.fromString(vectorize); this.helper = AggregationTestHelper.createGroupByQueryAggregationTestHelper( - DefaultColumnFormatConfigModule.getJacksonModulesList(), + BuiltInTypesModule.getJacksonModulesList(), config, tempFolder ); diff --git a/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/column/NestedColumnGroupByColumnSelectorStrategyTest.java b/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/column/NestedColumnGroupByColumnSelectorStrategyTest.java index dda7e60eb34a..bdce3d693e88 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/column/NestedColumnGroupByColumnSelectorStrategyTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/column/NestedColumnGroupByColumnSelectorStrategyTest.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.query.IterableRowsCursorHelper; import org.apache.druid.query.groupby.ResultRow; import org.apache.druid.query.groupby.epinephelinae.GroupByColumnSelectorStrategyFactory; @@ -44,7 +44,7 @@ public class NestedColumnGroupByColumnSelectorStrategyTest extends InitializedNullHandlingTest { static { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); } private static final GroupByColumnSelectorStrategyFactory STRATEGY_FACTORY = new GroupByColumnSelectorStrategyFactory(); diff --git a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java index b67c16c5fad0..80fa1fa0832c 100644 --- a/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/scan/NestedDataScanQueryTest.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.druid.common.config.NullHandling; import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Sequence; @@ -78,8 +78,8 @@ public void teardown() throws IOException public NestedDataScanQueryTest() { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); - List mods = DefaultColumnFormatConfigModule.getJacksonModulesList(); + BuiltInTypesModule.registerHandlersAndSerde(); + List mods = BuiltInTypesModule.getJacksonModulesList(); this.helper = AggregationTestHelper.createScanQueryAggregationTestHelper(mods, tempFolder); this.closer = Closer.create(); } diff --git a/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java b/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java index dbf326e0cc8f..f555d81cefe5 100644 --- a/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/timeseries/NestedDataTimeseriesQueryTest.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.io.Closer; @@ -98,7 +98,7 @@ public NestedDataTimeseriesQueryTest( ) { this.helper = AggregationTestHelper.createTimeseriesQueryAggregationTestHelper( - DefaultColumnFormatConfigModule.getJacksonModulesList(), + BuiltInTypesModule.getJacksonModulesList(), tempFolder ); this.segmentsGenerator = segmentsGenerator; diff --git a/processing/src/test/java/org/apache/druid/query/topn/NestedDataTopNQueryTest.java b/processing/src/test/java/org/apache/druid/query/topn/NestedDataTopNQueryTest.java index 6a092bcb926d..cb3b5fbfc9a9 100644 --- a/processing/src/test/java/org/apache/druid/query/topn/NestedDataTopNQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/topn/NestedDataTopNQueryTest.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableList; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; @@ -72,9 +72,9 @@ public NestedDataTopNQueryTest( BiFunction> segmentGenerator ) { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); this.helper = AggregationTestHelper.createTopNQueryAggregationTestHelper( - DefaultColumnFormatConfigModule.getJacksonModulesList(), + BuiltInTypesModule.getJacksonModulesList(), tempFolder ); this.segmentsGenerator = segmentGenerator; diff --git a/processing/src/test/java/org/apache/druid/segment/AutoTypeColumnIndexerTest.java b/processing/src/test/java/org/apache/druid/segment/AutoTypeColumnIndexerTest.java index d4db932822fc..73a2b6db50a4 100644 --- a/processing/src/test/java/org/apache/druid/segment/AutoTypeColumnIndexerTest.java +++ b/processing/src/test/java/org/apache/druid/segment/AutoTypeColumnIndexerTest.java @@ -25,7 +25,7 @@ import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Sequence; @@ -62,7 +62,7 @@ public class AutoTypeColumnIndexerTest extends InitializedNullHandlingTest @BeforeClass public static void setup() { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); } @Test diff --git a/processing/src/test/java/org/apache/druid/segment/NestedDataColumnIndexerV4Test.java b/processing/src/test/java/org/apache/druid/segment/NestedDataColumnIndexerV4Test.java index 9cf1c9a25b8a..359d2de58cb7 100644 --- a/processing/src/test/java/org/apache/druid/segment/NestedDataColumnIndexerV4Test.java +++ b/processing/src/test/java/org/apache/druid/segment/NestedDataColumnIndexerV4Test.java @@ -25,7 +25,7 @@ import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Sequence; @@ -61,7 +61,7 @@ public class NestedDataColumnIndexerV4Test extends InitializedNullHandlingTest @BeforeClass public static void setup() { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); } @Test diff --git a/processing/src/test/java/org/apache/druid/segment/data/IncrementalIndexTest.java b/processing/src/test/java/org/apache/druid/segment/data/IncrementalIndexTest.java index 5e6cdb6e1696..3143d2f7be74 100644 --- a/processing/src/test/java/org/apache/druid/segment/data/IncrementalIndexTest.java +++ b/processing/src/test/java/org/apache/druid/segment/data/IncrementalIndexTest.java @@ -33,7 +33,7 @@ import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.Row; import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; @@ -104,7 +104,7 @@ public class IncrementalIndexTest extends InitializedNullHandlingTest public IncrementalIndexTest(String indexType, String mode, boolean isPreserveExistingMetrics) throws JsonProcessingException { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); this.isPreserveExistingMetrics = isPreserveExistingMetrics; indexCreator = closer.closeLater(new IncrementalIndexCreator(indexType, (builder, args) -> builder .setSimpleTestingIndexSchema("rollup".equals(mode), isPreserveExistingMetrics, (AggregatorFactory[]) args[0]) diff --git a/processing/src/test/java/org/apache/druid/segment/filter/ArrayContainsElementFilterTests.java b/processing/src/test/java/org/apache/druid/segment/filter/ArrayContainsElementFilterTests.java index de78465e66fc..f45885790dc3 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/ArrayContainsElementFilterTests.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/ArrayContainsElementFilterTests.java @@ -27,7 +27,7 @@ import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.common.config.NullHandling; import org.apache.druid.error.DruidException; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; import org.apache.druid.query.filter.ArrayContainsElementFilter; @@ -1071,7 +1071,7 @@ public void testGetCacheKey() Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); f1 = new ArrayContainsElementFilter( "x", ColumnType.NESTED_DATA, diff --git a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java index 365de176a694..179e4e96d10c 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/BaseFilterTest.java @@ -41,7 +41,7 @@ import org.apache.druid.frame.FrameType; import org.apache.druid.frame.segment.FrameSegment; import org.apache.druid.frame.segment.FrameStorageAdapter; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; @@ -445,7 +445,7 @@ public BaseFilterTest( @Before public void setUp() throws Exception { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); String className = getClass().getName(); Map> adaptersForClass = adapterCache.get().get(className); if (adaptersForClass == null) { diff --git a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java index 0a96c2351630..97be448e61ab 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java @@ -30,7 +30,7 @@ import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.common.config.NullHandling; import org.apache.druid.error.DruidException; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; import org.apache.druid.math.expr.ExprEval; @@ -1760,7 +1760,7 @@ public void testGetCacheKey() Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); f1 = new EqualityFilter("x", ColumnType.NESTED_DATA, ImmutableMap.of("x", ImmutableList.of(1, 2, 3)), null); f1_2 = new EqualityFilter("x", ColumnType.NESTED_DATA, ImmutableMap.of("x", ImmutableList.of(1, 2, 3)), null); f2 = new EqualityFilter("x", ColumnType.NESTED_DATA, ImmutableMap.of("x", ImmutableList.of(1, 2, 3, 4)), null); diff --git a/processing/src/test/java/org/apache/druid/segment/generator/SegmentGenerator.java b/processing/src/test/java/org/apache/druid/segment/generator/SegmentGenerator.java index 537039f9a29f..5afb9fa04123 100644 --- a/processing/src/test/java/org/apache/druid/segment/generator/SegmentGenerator.java +++ b/processing/src/test/java/org/apache/druid/segment/generator/SegmentGenerator.java @@ -26,7 +26,7 @@ import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.MapInputRowParser; import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; @@ -140,7 +140,7 @@ public QueryableIndex generate( { // In case we need to generate hyperUniques or json ComplexMetrics.registerSerde(HyperUniquesSerde.TYPE_NAME, new HyperUniquesSerde()); - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); final String dataHash = Hashing.sha256() .newHasher() diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexIngestionTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexIngestionTest.java index 931f72b95212..77e0470c5486 100644 --- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexIngestionTest.java +++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexIngestionTest.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.ImmutableMap; import org.apache.druid.data.input.MapBasedInputRow; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.LongMaxAggregator; @@ -50,7 +50,7 @@ public class IncrementalIndexIngestionTest extends InitializedNullHandlingTest public IncrementalIndexIngestionTest(String indexType) throws JsonProcessingException { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); indexCreator = closer.closeLater(new IncrementalIndexCreator(indexType, (builder, args) -> builder .setIndexSchema((IncrementalIndexSchema) args[0]) .setMaxRowCount(MAX_ROWS) diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java index 51457cca268f..80c8207ed605 100644 --- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java +++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexMultiValueSpecTest.java @@ -27,7 +27,7 @@ import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.segment.CloserRule; @@ -56,7 +56,7 @@ public class IncrementalIndexMultiValueSpecTest extends InitializedNullHandlingT public IncrementalIndexMultiValueSpecTest(String indexType) throws JsonProcessingException { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); indexCreator = closer.closeLater(new IncrementalIndexCreator(indexType, (builder, args) -> builder .setIndexSchema((IncrementalIndexSchema) args[0]) .setMaxRowCount(10_000) diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java index e5b850627557..09f78e066fb2 100644 --- a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java @@ -27,7 +27,7 @@ import org.apache.druid.collections.ResourceHolder; import org.apache.druid.common.config.NullHandling; import org.apache.druid.data.input.MapBasedInputRow; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; @@ -97,7 +97,7 @@ public class IncrementalIndexStorageAdapterTest extends InitializedNullHandlingT public IncrementalIndexStorageAdapterTest(String indexType) throws JsonProcessingException { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); indexCreator = closer.closeLater(new IncrementalIndexCreator(indexType, (builder, args) -> builder .setSimpleTestingIndexSchema(new CountAggregatorFactory("cnt")) .setMaxRowCount(1_000) diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java index ed2996731777..1ebd1731c331 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java @@ -29,7 +29,7 @@ import org.apache.druid.collections.bitmap.ImmutableBitmap; import org.apache.druid.collections.bitmap.WrappedRoaringBitmap; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.io.smoosh.FileSmoosher; @@ -149,7 +149,7 @@ public class NestedDataColumnSupplierTest extends InitializedNullHandlingTest @BeforeClass public static void staticSetup() { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); } @Before diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java index a53ee99f8d17..c7b1facec56e 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java @@ -28,7 +28,7 @@ import com.google.common.util.concurrent.MoreExecutors; import org.apache.druid.collections.bitmap.RoaringBitmapFactory; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.io.smoosh.FileSmoosher; @@ -141,7 +141,7 @@ public class NestedDataColumnSupplierV4Test extends InitializedNullHandlingTest @BeforeClass public static void staticSetup() { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); } @Before diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnSelectorsTest.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnSelectorsTest.java index e99d14522ec7..e46b7893a0de 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnSelectorsTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnSelectorsTest.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.databind.Module; import com.google.common.collect.ImmutableList; import org.apache.druid.error.DruidException; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.guava.Sequence; @@ -81,8 +81,8 @@ public class NestedFieldColumnSelectorsTest extends InitializedNullHandlingTest public NestedFieldColumnSelectorsTest() { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); - List mods = DefaultColumnFormatConfigModule.getJacksonModulesList(); + BuiltInTypesModule.registerHandlersAndSerde(); + List mods = BuiltInTypesModule.getJacksonModulesList(); this.helper = AggregationTestHelper.createScanQueryAggregationTestHelper( mods, tempFolder diff --git a/processing/src/test/java/org/apache/druid/segment/nested/ScalarDoubleColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/ScalarDoubleColumnSupplierTest.java index ca30d3fdb12c..3f237b9396f1 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/ScalarDoubleColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/ScalarDoubleColumnSupplierTest.java @@ -26,7 +26,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.io.smoosh.FileSmoosher; @@ -105,7 +105,7 @@ public class ScalarDoubleColumnSupplierTest extends InitializedNullHandlingTest @BeforeClass public static void staticSetup() { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); } @Before diff --git a/processing/src/test/java/org/apache/druid/segment/nested/ScalarLongColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/ScalarLongColumnSupplierTest.java index 902aeaae2276..5fd8ddd299ce 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/ScalarLongColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/ScalarLongColumnSupplierTest.java @@ -26,7 +26,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.io.smoosh.FileSmoosher; @@ -105,7 +105,7 @@ public class ScalarLongColumnSupplierTest extends InitializedNullHandlingTest @BeforeClass public static void staticSetup() { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); } @Before diff --git a/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java index 19b9f0a13919..ee7ef05149ea 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java @@ -26,7 +26,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.io.smoosh.FileSmoosher; @@ -105,7 +105,7 @@ public class ScalarStringColumnSupplierTest extends InitializedNullHandlingTest @BeforeClass public static void staticSetup() { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); } @Before diff --git a/processing/src/test/java/org/apache/druid/segment/nested/VariantColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/VariantColumnSupplierTest.java index 4b572a4236cb..4a9423b87bc1 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/VariantColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/VariantColumnSupplierTest.java @@ -24,7 +24,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.io.smoosh.FileSmoosher; @@ -167,7 +167,7 @@ public class VariantColumnSupplierTest extends InitializedNullHandlingTest @BeforeClass public static void staticSetup() { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); } @Parameterized.Parameters(name = "data = {0}") diff --git a/server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java b/server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java index d4989d05f562..902d2855a0a5 100644 --- a/server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java +++ b/server/src/main/java/org/apache/druid/initialization/CoreInjectorBuilder.java @@ -24,8 +24,8 @@ import org.apache.druid.curator.discovery.DiscoveryModule; import org.apache.druid.discovery.NodeRole; import org.apache.druid.guice.AnnouncerModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.guice.CoordinatorDiscoveryModule; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; import org.apache.druid.guice.DruidInjectorBuilder; import org.apache.druid.guice.DruidSecondaryModule; import org.apache.druid.guice.ExpressionModule; @@ -113,7 +113,7 @@ public CoreInjectorBuilder forServer() new StorageNodeModule(), new JettyServerModule(), new ExpressionModule(), - new DefaultColumnFormatConfigModule(), + new BuiltInTypesModule(), new DiscoveryModule(), new ServerViewModule(), new MetadataConfigModule(), diff --git a/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java b/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java index 21d5a555d36e..75c8bb6b143f 100644 --- a/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java +++ b/services/src/test/java/org/apache/druid/cli/DumpSegmentTest.java @@ -31,7 +31,7 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.TimestampSpec; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.guice.StartupInjectorBuilder; import org.apache.druid.guice.annotations.Json; import org.apache.druid.initialization.ServerInjectorBuilder; @@ -84,7 +84,7 @@ public class DumpSegmentTest extends InitializedNullHandlingTest public DumpSegmentTest() { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); this.closer = Closer.create(); } @@ -156,7 +156,7 @@ public void testDumpNestedColumn() throws Exception { Injector injector = Mockito.mock(Injector.class); ObjectMapper mapper = TestHelper.makeJsonMapper(); - mapper.registerModules(DefaultColumnFormatConfigModule.getJacksonModulesList()); + mapper.registerModules(BuiltInTypesModule.getJacksonModulesList()); mapper.setInjectableValues( new InjectableValues.Std() .addValue(ExprMacroTable.class.getName(), TestExprMacroTable.INSTANCE) @@ -196,7 +196,7 @@ public void testDumpNestedColumnPath() throws Exception { Injector injector = Mockito.mock(Injector.class); ObjectMapper mapper = TestHelper.makeJsonMapper(); - mapper.registerModules(DefaultColumnFormatConfigModule.getJacksonModulesList()); + mapper.registerModules(BuiltInTypesModule.getJacksonModulesList()); mapper.setInjectableValues( new InjectableValues.Std() .addValue(ExprMacroTable.class.getName(), TestExprMacroTable.INSTANCE) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index 9ad04e0ba497..2e89c2eeb049 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableSet; import org.apache.calcite.avatica.SqlType; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.guice.DruidInjectorBuilder; import org.apache.druid.java.util.common.HumanReadableBytes; import org.apache.druid.java.util.common.Intervals; @@ -130,7 +130,7 @@ public ArraysComponentSupplier(TempDirProducer tempFolderProducer) public void configureGuice(DruidInjectorBuilder builder) { super.configureGuice(builder); - builder.addModule(new DefaultColumnFormatConfigModule()); + builder.addModule(new BuiltInTypesModule()); } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java index 6d769dd6faf8..5efc59420ada 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java @@ -34,7 +34,7 @@ import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.error.DruidException; import org.apache.druid.error.DruidExceptionMatcher; -import org.apache.druid.guice.DefaultColumnFormatConfigModule; +import org.apache.druid.guice.BuiltInTypesModule; import org.apache.druid.guice.DruidInjectorBuilder; import org.apache.druid.java.util.common.HumanReadableBytes; import org.apache.druid.java.util.common.granularity.Granularities; @@ -196,7 +196,7 @@ public NestedComponentSupplier(TempDirProducer tempFolderProducer) public void configureGuice(DruidInjectorBuilder builder) { super.configureGuice(builder); - builder.addModule(new DefaultColumnFormatConfigModule()); + builder.addModule(new BuiltInTypesModule()); } @SuppressWarnings("resource") @@ -207,7 +207,7 @@ public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker( final Injector injector ) { - DefaultColumnFormatConfigModule.registerHandlersAndSerde(); + BuiltInTypesModule.registerHandlersAndSerde(); final QueryableIndex index = IndexBuilder.create() .tmpDir(tempDirProducer.newTempFolder()) From b25a9f57efba821b9d7f8d78d7632ea42d181dc0 Mon Sep 17 00:00:00 2001 From: Abhishek Balaji Radhakrishnan Date: Fri, 2 Aug 2024 16:01:40 -0700 Subject: [PATCH 14/14] Make provider instance method & delegate to static method to init static variable to make spotbugs happy. --- .../org/apache/druid/guice/BuiltInTypesModule.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/guice/BuiltInTypesModule.java b/processing/src/main/java/org/apache/druid/guice/BuiltInTypesModule.java index 36e5cd6f0205..dc9701e7d8b3 100644 --- a/processing/src/main/java/org/apache/druid/guice/BuiltInTypesModule.java +++ b/processing/src/main/java/org/apache/druid/guice/BuiltInTypesModule.java @@ -39,6 +39,7 @@ import org.apache.druid.segment.serde.ComplexMetrics; import org.apache.druid.segment.virtual.NestedFieldVirtualColumn; +import javax.annotation.Nullable; import java.util.Collections; import java.util.List; @@ -69,7 +70,7 @@ public void configure(Binder binder) @Provides @LazySingleton - public static SideEffectRegisterer initDimensionHandlerAndMvHandlingMode(DefaultColumnFormatConfig formatsConfig) + public SideEffectRegisterer initDimensionHandlerAndMvHandlingMode(DefaultColumnFormatConfig formatsConfig) { if (formatsConfig.getNestedColumnFormatVersion() != null && formatsConfig.getNestedColumnFormatVersion() == 4) { DimensionHandlerUtils.registerDimensionHandlerProvider( @@ -83,12 +84,17 @@ public static SideEffectRegisterer initDimensionHandlerAndMvHandlingMode(Default ); } - if (formatsConfig.getStringMultiValueHandlingMode() != null) { - STRING_MV_MODE = DimensionSchema.MultiValueHandling.fromString(formatsConfig.getStringMultiValueHandlingMode()); - } + setStringMultiValueHandlingModeIfConfigured(formatsConfig.getStringMultiValueHandlingMode()); return new SideEffectRegisterer(); } + private static void setStringMultiValueHandlingModeIfConfigured(@Nullable String stringMultiValueHandlingMode) + { + if (stringMultiValueHandlingMode != null) { + STRING_MV_MODE = DimensionSchema.MultiValueHandling.fromString(stringMultiValueHandlingMode); + } + } + /** * @return the configured string multi value handling mode from the system config if set; otherwise, returns * the default.