druid thrift extensions

docs/content/development/extensions-contrib/thrift.md

@@ -0,0 +1,76 @@
+---
+layout: doc_page
+---
+
+# Thrift
+
+This extension enables Druid to ingest and understand the Apache Thrift data format. Make sure to [include](../../operations/including-extensions.html) `druid-thrift-extensions` as an extension.
+
+Notice that for both stream and hadoop parser explained below, the thrift data class file should be included in classpath.
+
+### Thrift Stream Parser
+
+This is for streaming/realtime ingestion.
+
+| Field | Type | Description | Required |
+|-------|------|-------------|----------|
+| type | String | This should say `thrift_stream`. | yes |
+| parseSpec | JSON Object | Specifies the timestamp and dimensions of the data. Should be a timeAndDims parseSpec. | yes |
+| tClassName | String | Specifies the class name of thrift object type. | yes |
+
+For example, using Thrift stream parser with schema repo Thrift bytes decoder:
+
+```json
+"parser" : {
+  "type" : "thrift_stream",
+  "tClassName" : "${YOUR_THRIFT_CLASS_NAME}",
+  "parseSpec" : {
+    "format": "timeAndDims",
+    "timestampSpec": <standard timestampSpec>,
+    "dimensionsSpec": <standard dimensionsSpec>
+  }
+}
+```
+
+### Thrift Hadoop Parser
+
+This is for batch ingestion using the HadoopDruidIndexer. The `inputFormat` of `inputSpec` in `ioConfig` can be set to `"com.twitter.elephantbird.mapreduce.input.LzoThriftBlockInputFormat"` or other inputFormat depending on your demand. If using InputFormat from elephant-bird, the `elephantbird.class.for.MultiInputFormat` of `jobProperties` in `tuningConfig` should be set to your thrift class name. Make sure to include "io.druid.extensions:druid-thrift-extensions" as an extension.
+
+| Field | Type | Description | Required |
+|-------|------|-------------|----------|
+| type | String | This should say `thrift_hadoop`. | yes |
+| parseSpec | JSON Object | Specifies the timestamp and dimensions of the data. Should be a timeAndDims parseSpec. | yes |
+
+For example, using Thrift Hadoop parser with custom reader's schema file:
+
+```json
+{
+  "type" : "index_hadoop",
+  "spec" : {
+    "dataSchema" : {
+      "dataSource" : "",
+      "parser" : {
+        "type" : "thrift_hadoop",
+        "parseSpec" : {
+          "format": "timeAndDims",
+          "timestampSpec": <standard timestampSpec>,
+          "dimensionsSpec": <standard dimensionsSpec>
+        }
+      }
+    },
+    "ioConfig" : {
+      "type" : "hadoop",
+      "inputSpec" : {
+        "type" : "static",
+        "inputFormat": "com.twitter.elephantbird.mapreduce.input.LzoThriftBlockInputFormat",
+        "paths" : ""
+      }
+    },
+    "tuningConfig" : {
+       "jobProperties" : {
+          "elephantbird.class.for.MultiInputFormat" : "${YOUR_THRIFT_CLASS_NAME}"
+      }
+    }
+  }
+}
+```

docs/content/development/extensions.md

@@ -58,6 +58,7 @@ All of these community extensions can be downloaded using *pull-deps* with the c
 |druid-parquet-extensions|Support for data in Apache Parquet data format. Requires druid-avro-extensions to be loaded.|[link](../development/extensions-contrib/parquet.html)|
 |druid-rabbitmq|RabbitMQ firehose.|[link](../development/extensions-contrib/rabbitmq.html)|
 |druid-rocketmq|RocketMQ firehose.|[link](../development/extensions-contrib/rocketmq.html)|
+|druid-thrift-extensions|Support for data in Apache Thrift data format.|[link](../development/extensions-contrib/thrift.html)|
 |graphite-emitter|Graphite metrics emitter|[link](../development/extensions-contrib/graphite.html)|
 |statsd-emitter|StatsD metrics emitter|[link](../development/extensions-contrib/statsd.html)|
 

extensions-contrib/thrift-extensions/pom.xml

@@ -0,0 +1,136 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. Metamarkets licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+  -->
+
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <groupId>io.druid</groupId>
+    <artifactId>druid</artifactId>
+    <version>0.9.3-SNAPSHOT</version>
+    <relativePath>../../pom.xml</relativePath>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>io.druid.extensions.contrib</groupId>
+  <artifactId>druid-thrift-extensions</artifactId>
+  <name>druid-thrift-extensions</name>
+  <description>druid-thrift-extensions</description>
+
+  <pluginRepositories>
+    <pluginRepository>
+      <id>dtrott</id>
+      <url>http://maven.davidtrott.com/repository</url>
+    </pluginRepository>
+  </pluginRepositories>
+
+  <profiles>
+    <profile>
+      <id>mac</id>
+      <activation>
+        <os>
+          <family>mac</family>
+        </os>
+      </activation>
+      <properties>
+        <os.family>mac</os.family>
+      </properties>
+    </profile>
+    <profile>
+      <id>linux</id>
+      <activation>
+        <os>
+          <family>linux</family>
+        </os>
+      </activation>
+      <properties>
+        <os.family>linux</os.family>
+      </properties>
+    </profile>
+  </profiles>
+
+  <properties>
+    <thrift.version>0.6.1</thrift.version>
+    <!--the default thrift binary is under version 0.6.1-->
+    <thrift.exec>${basedir}/src/main/native/${os.family}/bin/thrift</thrift.exec>
+    <elephantbird.version>4.6</elephantbird.version>
+  </properties>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.thrift.tools</groupId>
+        <artifactId>maven-thrift-plugin</artifactId>
+        <version>0.1.10</version>
+        <configuration>
+          <thriftExecutable>${thrift.exec}</thriftExecutable>
+          <thriftSourceRoot>${basedir}/src/test/thrift</thriftSourceRoot>
+          <generator>java:private-members,hashcode</generator>
+        </configuration>
+        <executions>
+          <execution>
+            <id>thrift-test-sources</id>
+            <phase>generate-test-sources</phase>
+            <goals>
+              <goal>testCompile</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+  <dependencies>
+    <dependency>
+      <groupId>io.druid</groupId>
+      <artifactId>druid-api</artifactId>
+      <version>${project.parent.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.thrift</groupId>
+      <artifactId>libthrift</artifactId>
+      <version>${thrift.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>${hadoop.compile.version}</version>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.twitter.elephantbird</groupId>
+      <artifactId>elephant-bird-core</artifactId>
+      <version>${elephantbird.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.twitter.elephantbird</groupId>
+      <artifactId>elephant-bird-hadoop-compat</artifactId>
+      <version>${elephantbird.version}</version>
+    </dependency>
+    <!-- Tests -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+</project>

extensions-contrib/thrift-extensions/src/main/java/io/druid/data/input/ThriftHadoopInputRowParser.java

@@ -0,0 +1,75 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.data.input;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.twitter.elephantbird.mapreduce.io.ThriftWritable;
+import io.druid.data.input.impl.InputRowParser;
+import io.druid.data.input.impl.ParseSpec;
+import io.druid.data.input.impl.TimestampSpec;
+import io.druid.data.input.thrift.TBaseAsMap;
+import org.apache.thrift.TBase;
+import org.joda.time.DateTime;
+
+import java.util.List;
+
+@JsonTypeName("thrift_hadoop")
+public class ThriftHadoopInputRowParser implements InputRowParser<ThriftWritable<?>>
+{
+
+  private final ParseSpec parseSpec;
+  private final List<String> dimensions;
+
+  @JsonCreator
+  public ThriftHadoopInputRowParser(
+      @JsonProperty("parseSpec") ParseSpec parseSpec
+  )
+  {
+    this.parseSpec = parseSpec;
+    this.dimensions = parseSpec.getDimensionsSpec().getDimensionNames();
+  }
+
+  @Override
+  public InputRow parse(ThriftWritable<?> input)
+  {
+    TBase tBase = input.get();
+    @SuppressWarnings("unchecked")
+    TBaseAsMap<?> tBaseAsMap = new TBaseAsMap(tBase);
+    TimestampSpec timestampSpec = parseSpec.getTimestampSpec();
+    DateTime dateTime = timestampSpec.extractTimestamp(tBaseAsMap);
+    return new MapBasedInputRow(dateTime, dimensions, tBaseAsMap);
+  }
+
+  @JsonProperty
+  @Override
+  public ParseSpec getParseSpec()
+  {
+    return parseSpec;
+  }
+
+  @Override
+  public InputRowParser withParseSpec(ParseSpec parseSpec)
+  {
+    return new ThriftHadoopInputRowParser(parseSpec);
+  }
+
+}