Druid Catalog basics

.travis.yml

@@ -688,12 +688,17 @@ jobs:
       jdk: openjdk8
       services: *integration_test_services
       env: JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
-      # Uses the installation defined above. Then, builds the test tools and docker image,
-      # and runs one IT. If tests fail, echos log lines of any of
-      # the Druid services that did not exit normally.
       script: ./it.sh travis MultiStageQuery
 
-    # Disabling BatchIndex test as it is failing with due to timeout, fixing it will be taken in a separate PR.
+    - &integration_tests_ex
+      name: "(Compile=openjdk8, Run=openjdk8) catalog integration tests"
+      stage: Tests - phase 2
+      jdk: openjdk8
+      services: *integration_test_services
+      env: JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
+      script: ./it.sh travis Catalog
+
+   # Disabling BatchIndex test as it is failing with due to timeout, fixing it will be taken in a separate PR.
     #- <<: *integration_tests_ex
     #  name: "(Compile=openjdk8, Run=openjdk8) batch index integration test with Indexer (new)"
     #  env: JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='indexer'

core/src/main/java/org/apache/druid/data/input/InputFormat.java

@@ -41,15 +41,17 @@
  * See {@link NestedInputFormat} for nested input formats such as JSON.
  */
 @UnstableApi
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = InputFormat.TYPE_PROPERTY)
 @JsonSubTypes(value = {
-    @Type(name = "csv", value = CsvInputFormat.class),
-    @Type(name = "json", value = JsonInputFormat.class),
-    @Type(name = "regex", value = RegexInputFormat.class),
-    @Type(name = "tsv", value = DelimitedInputFormat.class)
+    @Type(name = CsvInputFormat.TYPE_KEY, value = CsvInputFormat.class),
+    @Type(name = JsonInputFormat.TYPE_KEY, value = JsonInputFormat.class),
+    @Type(name = RegexInputFormat.TYPE_KEY, value = RegexInputFormat.class),
+    @Type(name = DelimitedInputFormat.TYPE_KEY, value = DelimitedInputFormat.class)
 })
 public interface InputFormat
 {
+  String TYPE_PROPERTY = "type";
+
   /**
    * Trait to indicate that a file can be split into multiple {@link InputSplit}s.
    * <p>

core/src/main/java/org/apache/druid/data/input/InputSource.java

@@ -19,6 +19,7 @@
 
 package org.apache.druid.data.input;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
@@ -47,19 +48,22 @@
  * }</pre>
  */
 @UnstableApi
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = InputSource.TYPE_PROPERTY)
 @JsonSubTypes(value = {
-    @Type(name = "local", value = LocalInputSource.class),
-    @Type(name = "http", value = HttpInputSource.class),
-    @Type(name = "inline", value = InlineInputSource.class),
-    @Type(name = "combining", value = CombiningInputSource.class)
+    @Type(name = LocalInputSource.TYPE_KEY, value = LocalInputSource.class),
+    @Type(name = HttpInputSource.TYPE_KEY, value = HttpInputSource.class),
+    @Type(name = InlineInputSource.TYPE_KEY, value = InlineInputSource.class),
+    @Type(name = CombiningInputSource.TYPE_KEY, value = CombiningInputSource.class)
 })
 public interface InputSource
 {
+  String TYPE_PROPERTY = "type";
+
   /**
    * Returns true if this inputSource can be processed in parallel using ParallelIndexSupervisorTask. It must be
    * castable to SplittableInputSource and the various SplittableInputSource methods must work as documented.
    */
+  @JsonIgnore
   boolean isSplittable();
 
   /**

core/src/main/java/org/apache/druid/data/input/impl/CombiningInputSource.java

@@ -45,6 +45,8 @@
 
 public class CombiningInputSource extends AbstractInputSource implements SplittableInputSource
 {
+  public static final String TYPE_KEY = "combining";
+
   private final List<SplittableInputSource> delegates;
 
   @JsonCreator

core/src/main/java/org/apache/druid/data/input/impl/CsvInputFormat.java

@@ -37,6 +37,7 @@
 
 public class CsvInputFormat extends FlatTextInputFormat
 {
+  public static final String TYPE_KEY = "csv";
   private static final char SEPARATOR = ',';
 
   @JsonCreator
@@ -59,6 +60,7 @@ public String getDelimiter()
   }
 
   @Override
+  @JsonIgnore
   public boolean isSplittable()
   {
     return true;

core/src/main/java/org/apache/druid/data/input/impl/DelimitedInputFormat.java

@@ -36,10 +36,11 @@
 import java.util.List;
 
 /**
- * InputFormat for customized Delimitor Separate Value format of input data(default is TSV).
+ * InputFormat for customized Delimiter Separate Value format of input data (default is TSV).
  */
 public class DelimitedInputFormat extends FlatTextInputFormat
 {
+  public static final String TYPE_KEY = "tsv";
   private static final String DEFAULT_DELIMITER = "\t";
 
   @JsonCreator

core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java

@@ -44,6 +44,8 @@
 
 public class HttpInputSource extends AbstractInputSource implements SplittableInputSource<URI>
 {
+  public static final String TYPE_KEY = "http";
+
   private final List<URI> uris;
   @Nullable
   private final String httpAuthenticationUsername;

core/src/main/java/org/apache/druid/data/input/impl/InlineInputSource.java

@@ -20,8 +20,10 @@
 package org.apache.druid.data.input.impl;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
 import org.apache.druid.data.input.AbstractInputSource;
 import org.apache.druid.data.input.InputFormat;
 import org.apache.druid.data.input.InputRowSchema;
@@ -35,12 +37,14 @@
 
 public class InlineInputSource extends AbstractInputSource
 {
+  public static final String TYPE_KEY = "inline";
+
   private final String data;
 
   @JsonCreator
   public InlineInputSource(@JsonProperty("data") String data)
   {
-    Preconditions.checkArgument(data != null && !data.isEmpty(), "empty data");
+    Preconditions.checkArgument(!Strings.isNullOrEmpty(data), "empty data");
     this.data = data;
   }
 
@@ -51,6 +55,7 @@ public String getData()
   }
 
   @Override
+  @JsonIgnore
   public boolean isSplittable()
   {
     return false;

core/src/main/java/org/apache/druid/data/input/impl/JsonInputFormat.java

@@ -39,20 +39,20 @@
 
 public class JsonInputFormat extends NestedInputFormat
 {
+  public static final String TYPE_KEY = "json";
+
   private final Map<String, Boolean> featureSpec;
   private final ObjectMapper objectMapper;
   private final boolean keepNullColumns;
 
   /**
-   *
-   * This parameter indicates whether or not the given InputEntity should be split by lines before parsing it.
+   * Indicates whether or not the given InputEntity should be split by lines before parsing it.
    * If it is set to true, the InputEntity must be split by lines first.
    * If it is set to false, unlike what you could imagine, it means that the InputEntity doesn't have to be split by lines first, but it can still contain multiple lines.
    * A created InputEntityReader from this format will determine by itself if line splitting is necessary.
    *
    * This parameter should always be true for batch ingestion and false for streaming ingestion.
    * For more information, see: https://github.com/apache/druid/pull/10383.
-   *
    */
   private final boolean lineSplittable;
 

core/src/main/java/org/apache/druid/data/input/impl/LocalInputSource.java

@@ -57,6 +57,7 @@
 public class LocalInputSource extends AbstractInputSource implements SplittableInputSource<List<File>>
 {
   private static final Logger log = new Logger(LocalInputSource.class);
+  public static final String TYPE_KEY = "local";
 
   @Nullable
   private final File baseDir;

core/src/main/java/org/apache/druid/data/input/impl/RegexInputFormat.java

@@ -37,6 +37,8 @@
 
 public class RegexInputFormat implements InputFormat
 {
+  public static final String TYPE_KEY = "regex";
+
   private final String pattern;
   private final String listDelimiter;
   private final List<String> columns;

core/src/main/java/org/apache/druid/java/util/common/jackson/JacksonUtils.java

@@ -20,11 +20,13 @@
 package org.apache.druid.java.util.common.jackson;
 
 import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.JsonSerializer;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializerProvider;
+import org.apache.druid.java.util.common.ISE;
 
 import javax.annotation.Nullable;
 import java.io.IOException;
@@ -45,6 +47,10 @@ public final class JacksonUtils
       {
       };
 
+  private JacksonUtils()
+  {
+  }
+
   /**
    * Silences Jackson's {@link IOException}.
    */
@@ -89,7 +95,17 @@ public static void writeObjectUsingSerializerProvider(
     }
   }
 
-  private JacksonUtils()
+  /**
+   * Convert the given object to an array of bytes. Use when the object is
+   * known serializable so that the Jackson exception can be suppressed.
+   */
+  public static byte[] toBytes(ObjectMapper jsonMapper, Object obj)
   {
+    try {
+      return jsonMapper.writeValueAsBytes(obj);
+    }
+    catch (JsonProcessingException e) {
+      throw new ISE("Failed to serialize " + obj.getClass().getSimpleName());
+    }
   }
 }

core/src/main/java/org/apache/druid/metadata/DefaultPasswordProvider.java

@@ -24,6 +24,8 @@
 
 public class DefaultPasswordProvider implements PasswordProvider
 {
+  public static final String TYPE_KEY = "default";
+
   private final String password;
 
   @JsonCreator

core/src/main/java/org/apache/druid/metadata/EnvironmentVariablePasswordProvider.java

@@ -26,6 +26,8 @@
 
 public class EnvironmentVariablePasswordProvider implements PasswordProvider
 {
+  public static final String TYPE_KEY = "environment";
+
   private final String variable;
 
   @JsonCreator

core/src/main/java/org/apache/druid/metadata/PasswordProvider.java

@@ -36,9 +36,8 @@
 @ExtensionPoint
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = DefaultPasswordProvider.class)
 @JsonSubTypes(value = {
-    @JsonSubTypes.Type(name = "default", value = DefaultPasswordProvider.class),
-    @JsonSubTypes.Type(name = "environment", value = EnvironmentVariablePasswordProvider.class),
-
+    @JsonSubTypes.Type(name = DefaultPasswordProvider.TYPE_KEY, value = DefaultPasswordProvider.class),
+    @JsonSubTypes.Type(name = EnvironmentVariablePasswordProvider.TYPE_KEY, value = EnvironmentVariablePasswordProvider.class),
 })
 public interface PasswordProvider
 {

core/src/main/java/org/apache/druid/segment/column/ColumnType.java

@@ -120,6 +120,7 @@ public static ColumnType ofArray(ColumnType elementType)
   {
     return ColumnTypeFactory.getInstance().ofArray(elementType);
   }
+
   public static ColumnType ofComplex(@Nullable String complexTypeName)
   {
     return ColumnTypeFactory.getInstance().ofComplex(complexTypeName);

distribution/pom.xml

@@ -46,6 +46,7 @@
             <version>${project.parent.version}</version>
         </dependency>
     </dependencies>
+
     <properties>
         <!-- the default value is a repeated flag from the command line, since blank value is not allowed -->
         <druid.distribution.pulldeps.opts>--clean</druid.distribution.pulldeps.opts>
@@ -255,6 +256,8 @@
                                         <argument>org.apache.druid.extensions:druid-ranger-security</argument>
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions:druid-kubernetes-extensions</argument>
+                                        <argument>-c</argument>
+                                        <argument>org.apache.druid.extensions:druid-catalog</argument>
                                         <argument>${druid.distribution.pulldeps.opts}</argument>
                                     </arguments>
                                 </configuration>