Add Rackspace Cloud Files Deep Storage Extension

distribution/pom.xml

@@ -91,6 +91,8 @@
                                 <argument>io.druid.extensions:druid-rabbitmq</argument>
                                 <argument>-c</argument>
                                 <argument>io.druid.extensions:druid-s3-extensions</argument>
+                                <argument>-c</argument>
+                                <argument>io.druid.extensions:druid-cloudfiles-extensions</argument>
                             </arguments>
                         </configuration>
                     </execution>

docs/content/dependencies/deep-storage.md

@@ -69,3 +69,21 @@ Please note that this is a community contributed module and does not support Cas
 |`druid.azure.maxTries`||Number of tries before cancel an Azure operation.|3|
 
 Please note that this is a community contributed module. See [Azure Services](http://azure.microsoft.com/en-us/pricing/free-trial/) for more information.
+
+### Rackspace
+
+[Rackspace Cloud Files](http://www.rackspace.com/cloud/files/) is another option for deep storage. This requires some additional druid configuration.
+
+|Property|Possible Values|Description|Default|
+|--------|---------------|-----------|-------|
+|`druid.storage.type`|cloudfiles||Must be set.|
+|`druid.storage.region`||Rackspace Cloud Files region.|Must be set.|
+|`druid.storage.container`||Rackspace Cloud Files container name.|Must be set.|
+|`druid.storage.basePath`||Rackspace Cloud Files base path to use in the container.|Must be set.|
+|`druid.storage.operationMaxRetries`||Number of tries before cancel a Rackspace operation.|10|
+|`druid.cloudfiles.userName`||Rackspace Cloud username|Must be set.|
+|`druid.cloudfiles.apiKey`||Rackspace Cloud api key.|Must be set.|
+|`druid.cloudfiles.provider`|rackspace-cloudfiles-us,rackspace-cloudfiles-uk|Name of the provider depending on the region.|Must be set.|
+|`druid.cloudfiles.useServiceNet`|true,false|Whether to use the internal service net.|true|
+
+Please note that this is a community contributed module.

extensions/cloudfiles-extensions/pom.xml

@@ -0,0 +1,106 @@
+<?xml version="1.0"?>
+
+<!--
+  ~ Licensed to Metamarkets Group Inc. (Metamarkets) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  Metamarkets licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+    xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>io.druid.extensions</groupId>
+  <artifactId>druid-cloudfiles-extensions</artifactId>
+  <name>druid-cloudfiles-extensions</name>
+  <description>druid-cloudfiles-extensions</description>
+
+  <parent>
+    <groupId>io.druid</groupId>
+    <artifactId>druid</artifactId>
+    <version>0.9.0-SNAPSHOT</version>
+    <relativePath>../../pom.xml</relativePath>
+  </parent>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <jclouds.version>1.9.1</jclouds.version>
+    <!-- The version of guice is forced to 3.0 since JClouds 1.9.1 does not
+      work with guice 4.0-beta -->
+    <guice.version>3.0</guice.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>io.druid</groupId>
+      <artifactId>druid-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.google.inject</groupId>
+      <artifactId>guice</artifactId>
+      <version>${guice.version}</version>
+      <!--$NO-MVN-MAN-VER$ -->
+    </dependency>
+    <dependency>
+      <groupId>com.google.inject.extensions</groupId>
+      <artifactId>guice-servlet</artifactId>
+      <version>${guice.version}</version>
+      <!--$NO-MVN-MAN-VER$ -->
+    </dependency>
+    <dependency>
+      <groupId>com.google.inject.extensions</groupId>
+      <artifactId>guice-multibindings</artifactId>
+      <version>${guice.version}</version>
+      <!--$NO-MVN-MAN-VER$ -->
+    </dependency>
+    <!-- jclouds dependencies -->
+    <dependency>
+      <groupId>org.apache.jclouds.driver</groupId>
+      <artifactId>jclouds-slf4j</artifactId>
+      <version>${jclouds.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.jclouds.driver</groupId>
+      <artifactId>jclouds-sshj</artifactId>
+      <version>${jclouds.version}</version>
+    </dependency>
+    <!-- Rackspace US dependencies -->
+    <dependency>
+      <groupId>org.apache.jclouds.provider</groupId>
+      <artifactId>rackspace-cloudfiles-us</artifactId>
+      <version>${jclouds.version}</version>
+    </dependency>
+    <!-- Rackspace UK dependencies -->
+    <dependency>
+      <groupId>org.apache.jclouds.provider</groupId>
+      <artifactId>rackspace-cloudfiles-uk</artifactId>
+      <version>${jclouds.version}</version>
+    </dependency>
+
+    <!-- Tests -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.easymock</groupId>
+      <artifactId>easymock</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+</project>

extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesAccountConfig.java

@@ -0,0 +1,63 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright 2012 - 2015 Metamarkets Group Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.druid.storage.cloudfiles;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import javax.validation.constraints.NotNull;
+
+public class CloudFilesAccountConfig
+{
+
+  @JsonProperty
+  @NotNull
+  private String provider;
+
+  @JsonProperty
+  @NotNull
+  private String userName;
+
+  @JsonProperty
+  @NotNull
+  private String apiKey;
+
+  @JsonProperty
+  @NotNull
+  private boolean useServiceNet = true;
+
+  public String getProvider()
+  {
+    return provider;
+  }
+
+  public String getUserName()
+  {
+    return userName;
+  }
+
+  public String getApiKey()
+  {
+    return apiKey;
+  }
+
+  public boolean getUseServiceNet()
+  {
+    return useServiceNet;
+  }
+
+}

extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesByteSource.java

@@ -0,0 +1,64 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright 2012 - 2015 Metamarkets Group Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.druid.storage.cloudfiles;
+
+import com.google.common.base.Throwables;
+import com.google.common.io.ByteSource;
+import org.jclouds.io.Payload;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class CloudFilesByteSource extends ByteSource
+{
+
+  final private CloudFilesObjectApiProxy objectApi;
+  final private String path;
+  private Payload payload;
+
+  public CloudFilesByteSource(CloudFilesObjectApiProxy objectApi, String path)
+  {
+    this.objectApi = objectApi;
+    this.path = path;
+    this.payload = null;
+  }
+
+  public void closeStream() throws IOException
+  {
+    if (payload != null) {
+      payload.close();
+      payload = null;
+    }
+  }
+
+  @Override
+  public InputStream openStream() throws IOException
+  {
+    payload = (payload == null) ? objectApi.get(path).getPayload() : payload;
+
+    try {
+      return payload.openStream();
+    }
+    catch (IOException e) {
+      if (CloudFilesUtils.CLOUDFILESRETRY.apply(e)) {
+        throw new IOException("Recoverable exception", e);
+      }
+      throw Throwables.propagate(e);
+    }
+  }
+}

extensions/cloudfiles-extensions/src/main/java/io/druid/storage/cloudfiles/CloudFilesDataSegmentPuller.java

@@ -0,0 +1,110 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright 2012 - 2015 Metamarkets Group Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.druid.storage.cloudfiles;
+
+import com.google.inject.Inject;
+import com.metamx.common.CompressionUtils;
+import com.metamx.common.FileUtils;
+import com.metamx.common.ISE;
+import com.metamx.common.MapUtils;
+import com.metamx.common.logger.Logger;
+import io.druid.segment.loading.DataSegmentPuller;
+import io.druid.segment.loading.SegmentLoadingException;
+import io.druid.timeline.DataSegment;
+import org.jclouds.rackspace.cloudfiles.v1.CloudFilesApi;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * A data segment puller that also handles URI data pulls.
+ */
+public class CloudFilesDataSegmentPuller implements DataSegmentPuller
+{
+
+  private static final Logger log = new Logger(CloudFilesDataSegmentPuller.class);
+  private final CloudFilesApi cloudFilesApi;
+
+  @Inject
+  public CloudFilesDataSegmentPuller(final CloudFilesApi cloudFilesApi)
+  {
+    this.cloudFilesApi = cloudFilesApi;
+  }
+
+  @Override
+  public void getSegmentFiles(final DataSegment segment, final File outDir) throws SegmentLoadingException
+  {
+    final Map<String, Object> loadSpec = segment.getLoadSpec();
+    final String region = MapUtils.getString(loadSpec, "region");
+    final String container = MapUtils.getString(loadSpec, "container");
+    final String path = MapUtils.getString(loadSpec, "path");
+
+    log.info("Pulling index at path[%s] to outDir[%s]", path, outDir);
+    prepareOutDir(outDir);
+    getSegmentFiles(region, container, path, outDir);
+  }
+
+  public FileUtils.FileCopyResult getSegmentFiles(String region, String container, String path, File outDir)
+      throws SegmentLoadingException
+  {
+    CloudFilesObjectApiProxy objectApi = new CloudFilesObjectApiProxy(cloudFilesApi, region, container);
+    final CloudFilesByteSource byteSource = new CloudFilesByteSource(objectApi, path);
+
+    try {
+      final FileUtils.FileCopyResult result = CompressionUtils.unzip(
+          byteSource, outDir,
+          CloudFilesUtils.CLOUDFILESRETRY, true
+      );
+      log.info("Loaded %d bytes from [%s] to [%s]", result.size(), path, outDir.getAbsolutePath());
+      return result;
+    }
+    catch (Exception e) {
+      try {
+        org.apache.commons.io.FileUtils.deleteDirectory(outDir);
+      }
+      catch (IOException ioe) {
+        log.warn(
+            ioe, "Failed to remove output directory [%s] for segment pulled from [%s]",
+            outDir.getAbsolutePath(), path
+        );
+      }
+      throw new SegmentLoadingException(e, e.getMessage());
+    }
+    finally {
+      try {
+        byteSource.closeStream();
+      }
+      catch (IOException ioe) {
+        log.warn(ioe, "Failed to close payload for segmente pulled from [%s]", path);
+      }
+    }
+  }
+
+  private void prepareOutDir(final File outDir) throws ISE
+  {
+    if (!outDir.exists()) {
+      outDir.mkdirs();
+    }
+
+    if (!outDir.isDirectory()) {
+      throw new ISE("outDir[%s] must be a directory.", outDir);
+    }
+  }
+
+}