apache · clintropolis · Apr 28, 2020 · Dec 3, 2019 · Dec 4, 2019 · Dec 4, 2019
diff --git a/distribution/pom.xml b/distribution/pom.xml
@@ -423,6 +423,8 @@
                                         <argument>org.apache.druid.extensions.contrib:druid-moving-average-query</argument>
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions.contrib:druid-tdigestsketch</argument>
+                                        <argument>-c</argument>
+                                        <argument>org.apache.druid.extensions.contrib:gce-extensions</argument>
                                     </arguments>
                                 </configuration>
                             </execution>

diff --git a/docs/configuration/index.md b/docs/configuration/index.md
@@ -891,7 +891,7 @@ There are additional configs for autoscaling (if it is enabled):
 
 |Property|Description|Default|
 |--------|-----------|-------|
-|`druid.indexer.autoscale.strategy`|Choices are "noop" or "ec2". Sets the strategy to run when autoscaling is required.|noop|
+|`druid.indexer.autoscale.strategy`|Choices are "noop", "ec2" or "gce". Sets the strategy to run when autoscaling is required.|noop|
 |`druid.indexer.autoscale.doAutoscale`|If set to "true" autoscaling will be enabled.|false|
 |`druid.indexer.autoscale.provisionPeriod`|How often to check whether or not new MiddleManagers should be added.|PT1M|
 |`druid.indexer.autoscale.terminatePeriod`|How often to check when MiddleManagers should be removed.|PT5M|
@@ -1115,7 +1115,9 @@ field. If not provided, the default is to not use it at all.
 
 ##### Autoscaler
 
-Amazon's EC2 is currently the only supported autoscaler.
+Amazon's EC2 together with Google's GCE are currently the only supported autoscalers.
+
+EC2's autoscaler properties are:
 
 |Property|Description|Default|
 |--------|-----------|-------|
@@ -1125,6 +1127,8 @@ Amazon's EC2 is currently the only supported autoscaler.
 |`nodeData`|A JSON object that describes how to launch new nodes.|none; required|
 |`userData`|A JSON object that describes how to configure new nodes. If you have set druid.indexer.autoscale.workerVersion, this must have a versionReplacementString. Otherwise, a versionReplacementString is not necessary.|none; optional|
 
+For GCE's properties, please refer to the [gce-extensions](../development/extensions-contrib/gce-extensions.md).
+
 ## Data Server
 
 This section contains the configuration options for the processes that reside on Data servers (MiddleManagers/Peons and Historicals) in the suggested [three-server configuration](../design/processes.html#server-types).

diff --git a/docs/development/extensions-contrib/gce-extensions.md b/docs/development/extensions-contrib/gce-extensions.md
@@ -0,0 +1,103 @@
+---
+id: gce-extensions
+title: "GCE Extensions"
+---
+
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+
+To use this Apache Druid (incubating) extension, make sure to [include](../../development/extensions.md#loading-extensions) `gce-extensions`.
+
+At the moment, this extension enables only Druid to autoscale instances in GCE.
+
+The extension manages the instances to be scaled up and down through the use of the [Managed Instance Groups](https://cloud.google.com/compute/docs/instance-groups/creating-groups-of-managed-instances#resize_managed_group)
+of GCE (MIG from now on). This choice has been made to ease the configuration of the machines and simplify their
+management.
+
+For this reason, in order to use this extension, the user must have created
+1. An instance template with the right machine type and image to bu used to run the MiddleManager
+2. A MIG that has been configured to use the instance template created in the point above
+
+Moreover, in order to be able to rescale the machines in the MIG, the Overlord must run with a service account
+guaranteeing the following two scopes from the [Compute Engine API](https://developers.google.com/identity/protocols/googlescopes#computev1)
+- `https://www.googleapis.com/auth/cloud-platform`
+- `https://www.googleapis.com/auth/compute`
+
+## Overlord Dynamic Configuration
+
+The Overlord can dynamically change worker behavior.
+
+The JSON object can be submitted to the Overlord via a POST request at:
+
+```
+http://<OVERLORD_IP>:<port>/druid/indexer/v1/worker
+```
+
+Optional Header Parameters for auditing the config change can also be specified.
+
+|Header Param Name| Description | Default |
+|----------|-------------|---------|
+|`X-Druid-Author`| author making the config change|""|
+|`X-Druid-Comment`| comment describing the change being done|""|
+
+A sample worker config spec is shown below:
+
+```json
+{
+  "autoScaler": {
+    "envConfig" : {
+      "numInstances" : 1,
+      "projectId" : "super-project",
+      "zoneName" : "us-central-1",
+      "managedInstanceGroupName" : "druid-middlemanagers"
+    },
+    "maxNumWorkers" : 4,
+    "minNumWorkers" : 2,
+    "type" : "gce"
+  }
+}
+```
+
+The configuration of the autoscaler is quite simple and it is made of two levels only.
+
+The external level specifies the `type`—always `gce` in this case— and two numeric values,
+the `maxNumWorkers` and `minNumWorkers` used to define the boundaries in between which the
+number of instances must be at any time.
+
+The internal level is the `envConfig` and it is used to specify
+
+- The `numInstances` used to specify how many workers will be spawned at each 
+request to provision more workers.  This is safe to be left to `1`
+- The `projectId` used to specify the name of the project in which the MIG resides
+- The `zoneName` used to identify in which zone of the worlds the MIG is
+- The `managedInstanceGroupName` used to specify the MIG containing the instances created or 
+removed
+
+Please refer to the Overlord Dynamic Configuration section in the main [documentation](../../configuration/index.md)
+for parameters other than the ones specified here, such as `selectStrategy` etc.
+
+## Known limitations
+
+- The module internally uses the [ListManagedInstances](https://cloud.google.com/compute/docs/reference/rest/v1/instanceGroupManagers/listManagedInstances)
+ call from the API and, while the documentation of the API states that the call can be paged through using the
+ `pageToken` argument, the responses to such call do not provide any `nextPageToken` to set such parameter. This means
+ that the extension can operate safely with a maximum of 500 MiddleManagers instances at any time (the maximum number
+ of instances to be returned for each call).
+
diff --git a/docs/development/extensions.md b/docs/development/extensions.md
@@ -91,6 +91,7 @@ All of these community extensions can be downloaded using [pull-deps](../operati
 |druid-influxdb-emitter|InfluxDB metrics emitter|[link](../development/extensions-contrib/influxdb-emitter.md)|
 |druid-momentsketch|Support for approximate quantile queries using the [momentsketch](https://github.com/stanford-futuredata/momentsketch) library|[link](../development/extensions-contrib/momentsketch-quantiles.md)|
 |druid-tdigestsketch|Support for approximate sketch aggregators based on [T-Digest](https://github.com/tdunning/t-digest)|[link](../development/extensions-contrib/tdigestsketch-quantiles.md)|
+|gce-extensions|GCE Extensions|[link](../development/extensions-contrib/gce-extensions.md)|
 
 ## Promoting community extensions to core extensions
 

diff --git a/extensions-contrib/gce-extensions/pom.xml b/extensions-contrib/gce-extensions/pom.xml
@@ -0,0 +1,131 @@
+<?xml version="1.0"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <parent>
+    <groupId>org.apache.druid</groupId>
+    <artifactId>druid</artifactId>
+    <version>0.19.0-SNAPSHOT</version>
+    <relativePath>../../pom.xml</relativePath>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.apache.druid.extensions.contrib</groupId>
+  <artifactId>gce-extensions</artifactId>
+  <name>gce-extensions</name>
+  <description>Extension to support the autoscaling in GCE</description>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.druid</groupId>
+      <artifactId>druid-core</artifactId>
+      <version>${project.parent.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.druid</groupId>
+      <artifactId>druid-indexing-service</artifactId>
+      <version>${project.parent.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.druid</groupId>
+      <artifactId>druid-aws-common</artifactId>
+      <version>${project.parent.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.druid</groupId>
+      <artifactId>druid-processing</artifactId>
+      <version>${project.parent.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.code.findbugs</groupId>
+      <artifactId>jsr305</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-annotations</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.inject</groupId>
+      <artifactId>guice</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.apis</groupId>
+      <artifactId>google-api-services-compute</artifactId>
+      <version>v1-rev214-1.25.0</version>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.http-client</groupId>
+      <artifactId>google-http-client</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.http-client</groupId>
+      <artifactId>google-http-client-jackson2</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.api-client</groupId>
+      <artifactId>google-api-client</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.curator</groupId>
+      <artifactId>curator-client</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <!-- Tests -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.easymock</groupId>
+      <artifactId>easymock</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>nl.jqno.equalsverifier</groupId>
+      <artifactId>equalsverifier</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+</project>