From 21c31a7906f74ae0ebefd79903e1afe342c7feeb Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Tue, 29 Jan 2019 16:17:44 -0800 Subject: [PATCH 01/14] Move GCP to a core extension --- {aws-common => cloud/aws-common}/pom.xml | 1 + .../druid/common/aws/AWSClientConfig.java | 0 .../common/aws/AWSCredentialsConfig.java | 0 .../druid/common/aws/AWSCredentialsUtils.java | 0 .../druid/common/aws/AWSEndpointConfig.java | 0 .../druid/common/aws/AWSProxyConfig.java | 0 ...figDrivenAwsCredentialsConfigProvider.java | 0 .../aws/FileSessionCredentialsProvider.java | 0 .../LazyFileSessionCredentialsProvider.java | 0 .../common/aws/AWSCredentialsConfigTest.java | 0 cloud/gcp-common/pom.xml | 63 +++++++++++++++ .../apache/druid/common/gcp/GcpModule.java | 80 +++++++++++++++++++ ...rg.apache.druid.initialization.DruidModule | 20 +++++ .../druid/common/gcp/GcpMockModule.java | 75 +++++++++++++++++ .../druid/common/gcp/GcpModuleTest.java | 51 ++++++++++++ distribution/pom.xml | 4 +- .../google-extensions/pom.xml | 36 +++------ .../druid/firehose/google/GoogleBlob.java | 0 .../StaticGoogleBlobStoreFirehoseFactory.java | 0 .../storage/google/GoogleAccountConfig.java | 0 .../storage/google/GoogleByteSource.java | 0 .../google/GoogleDataSegmentFinder.java | 0 .../google/GoogleDataSegmentKiller.java | 0 .../google/GoogleDataSegmentPuller.java | 0 .../google/GoogleDataSegmentPusher.java | 0 .../druid/storage/google/GoogleLoadSpec.java | 0 .../druid/storage/google/GoogleStorage.java | 0 .../google/GoogleStorageDruidModule.java | 29 +++---- .../druid/storage/google/GoogleTaskLogs.java | 0 .../storage/google/GoogleTaskLogsConfig.java | 0 .../druid/storage/google/GoogleUtils.java | 0 ...rg.apache.druid.initialization.DruidModule | 19 ----- ...ticGoogleBlobStoreFirehoseFactoryTest.java | 0 .../storage/google/GoogleByteSourceTest.java | 0 .../google/GoogleDataSegmentFinderTest.java | 0 .../google/GoogleDataSegmentKillerTest.java | 0 .../google/GoogleDataSegmentPullerTest.java | 0 .../google/GoogleDataSegmentPusherTest.java | 0 .../storage/google/GoogleTaskLogsTest.java | 0 .../druid/storage/google/GoogleUtilsTest.java | 0 pom.xml | 74 ++++++++++++++++- server/pom.xml | 5 ++ 42 files changed, 393 insertions(+), 64 deletions(-) rename {aws-common => cloud/aws-common}/pom.xml (97%) rename {aws-common => cloud/aws-common}/src/main/java/org/apache/druid/common/aws/AWSClientConfig.java (100%) rename {aws-common => cloud/aws-common}/src/main/java/org/apache/druid/common/aws/AWSCredentialsConfig.java (100%) rename {aws-common => cloud/aws-common}/src/main/java/org/apache/druid/common/aws/AWSCredentialsUtils.java (100%) rename {aws-common => cloud/aws-common}/src/main/java/org/apache/druid/common/aws/AWSEndpointConfig.java (100%) rename {aws-common => cloud/aws-common}/src/main/java/org/apache/druid/common/aws/AWSProxyConfig.java (100%) rename {aws-common => cloud/aws-common}/src/main/java/org/apache/druid/common/aws/ConfigDrivenAwsCredentialsConfigProvider.java (100%) rename {aws-common => cloud/aws-common}/src/main/java/org/apache/druid/common/aws/FileSessionCredentialsProvider.java (100%) rename {aws-common => cloud/aws-common}/src/main/java/org/apache/druid/common/aws/LazyFileSessionCredentialsProvider.java (100%) rename {aws-common => cloud/aws-common}/src/test/java/org/apache/druid/common/aws/AWSCredentialsConfigTest.java (100%) create mode 100644 cloud/gcp-common/pom.xml create mode 100644 cloud/gcp-common/src/main/java/org/apache/druid/common/gcp/GcpModule.java create mode 100644 cloud/gcp-common/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule create mode 100644 cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpMockModule.java create mode 100644 cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpModuleTest.java rename {extensions-contrib => extensions-core}/google-extensions/pom.xml (75%) rename {extensions-contrib => extensions-core}/google-extensions/src/main/java/org/apache/druid/firehose/google/GoogleBlob.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/main/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactory.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleAccountConfig.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleByteSource.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentFinder.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentKiller.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPuller.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPusher.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleLoadSpec.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageDruidModule.java (79%) rename {extensions-contrib => extensions-core}/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogsConfig.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleUtils.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule (51%) rename {extensions-contrib => extensions-core}/google-extensions/src/test/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactoryTest.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleByteSourceTest.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentFinderTest.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentKillerTest.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPullerTest.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPusherTest.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTaskLogsTest.java (100%) rename {extensions-contrib => extensions-core}/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleUtilsTest.java (100%) diff --git a/aws-common/pom.xml b/cloud/aws-common/pom.xml similarity index 97% rename from aws-common/pom.xml rename to cloud/aws-common/pom.xml index 941dc2847b01..39febc2fc656 100644 --- a/aws-common/pom.xml +++ b/cloud/aws-common/pom.xml @@ -29,6 +29,7 @@ org.apache.druid druid 0.13.0-incubating-SNAPSHOT + ../../pom.xml diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/AWSClientConfig.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSClientConfig.java similarity index 100% rename from aws-common/src/main/java/org/apache/druid/common/aws/AWSClientConfig.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSClientConfig.java diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/AWSCredentialsConfig.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSCredentialsConfig.java similarity index 100% rename from aws-common/src/main/java/org/apache/druid/common/aws/AWSCredentialsConfig.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSCredentialsConfig.java diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/AWSCredentialsUtils.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSCredentialsUtils.java similarity index 100% rename from aws-common/src/main/java/org/apache/druid/common/aws/AWSCredentialsUtils.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSCredentialsUtils.java diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/AWSEndpointConfig.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSEndpointConfig.java similarity index 100% rename from aws-common/src/main/java/org/apache/druid/common/aws/AWSEndpointConfig.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSEndpointConfig.java diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/AWSProxyConfig.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSProxyConfig.java similarity index 100% rename from aws-common/src/main/java/org/apache/druid/common/aws/AWSProxyConfig.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSProxyConfig.java diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/ConfigDrivenAwsCredentialsConfigProvider.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/ConfigDrivenAwsCredentialsConfigProvider.java similarity index 100% rename from aws-common/src/main/java/org/apache/druid/common/aws/ConfigDrivenAwsCredentialsConfigProvider.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/ConfigDrivenAwsCredentialsConfigProvider.java diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/FileSessionCredentialsProvider.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/FileSessionCredentialsProvider.java similarity index 100% rename from aws-common/src/main/java/org/apache/druid/common/aws/FileSessionCredentialsProvider.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/FileSessionCredentialsProvider.java diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/LazyFileSessionCredentialsProvider.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/LazyFileSessionCredentialsProvider.java similarity index 100% rename from aws-common/src/main/java/org/apache/druid/common/aws/LazyFileSessionCredentialsProvider.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/LazyFileSessionCredentialsProvider.java diff --git a/aws-common/src/test/java/org/apache/druid/common/aws/AWSCredentialsConfigTest.java b/cloud/aws-common/src/test/java/org/apache/druid/common/aws/AWSCredentialsConfigTest.java similarity index 100% rename from aws-common/src/test/java/org/apache/druid/common/aws/AWSCredentialsConfigTest.java rename to cloud/aws-common/src/test/java/org/apache/druid/common/aws/AWSCredentialsConfigTest.java diff --git a/cloud/gcp-common/pom.xml b/cloud/gcp-common/pom.xml new file mode 100644 index 000000000000..fa249aa55693 --- /dev/null +++ b/cloud/gcp-common/pom.xml @@ -0,0 +1,63 @@ + + + + + 4.0.0 + + druid-gcp-common + druid-gcp-common + Druid GCP Base Module + + + org.apache.druid + druid + 0.13.0-incubating-SNAPSHOT + ../../pom.xml + + + + + org.apache.druid + druid-core + ${project.parent.version} + provided + + + com.google.api-client + google-api-client + + + com.google.http-client + google-http-client-jackson2 + + + com.fasterxml.jackson.module + jackson-module-guice + + + + + junit + junit + test + + + + diff --git a/cloud/gcp-common/src/main/java/org/apache/druid/common/gcp/GcpModule.java b/cloud/gcp-common/src/main/java/org/apache/druid/common/gcp/GcpModule.java new file mode 100644 index 000000000000..c56a38aa4031 --- /dev/null +++ b/cloud/gcp-common/src/main/java/org/apache/druid/common/gcp/GcpModule.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.common.gcp; + +import com.fasterxml.jackson.databind.Module; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; +import com.google.inject.Provides; +import org.apache.druid.guice.LazySingleton; +import org.apache.druid.initialization.DruidModule; + +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.Collections; +import java.util.List; + +public class GcpModule implements DruidModule +{ + @Override + public List getJacksonModules() + { + return ImmutableList.of(); + } + + @Override + public void configure(Binder binder) + { + + } + + + @Provides + @LazySingleton + public HttpRequestInitializer getHttpRequestInitializer(HttpTransport transport, JsonFactory factory) + throws IOException + { + GoogleCredential credential = GoogleCredential.getApplicationDefault(transport, factory); + if (credential.createScopedRequired()) { + credential = credential.createScoped(Collections.singleton("https://www.googleapis.com/auth/cloud-platform")); + } + return credential; + } + + @Provides + @LazySingleton + public HttpTransport getHttpTransport() throws GeneralSecurityException, IOException + { + return GoogleNetHttpTransport.newTrustedTransport(); + } + + @Provides + @LazySingleton + public JsonFactory getJsonFactory() + { + return JacksonFactory.getDefaultInstance(); + } +} diff --git a/cloud/gcp-common/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/cloud/gcp-common/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 000000000000..c94639ec271a --- /dev/null +++ b/cloud/gcp-common/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.druid.common.gcp.GcpModule diff --git a/cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpMockModule.java b/cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpMockModule.java new file mode 100644 index 000000000000..c1f575416153 --- /dev/null +++ b/cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpMockModule.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.common.gcp; + +import com.fasterxml.jackson.databind.Module; +import com.google.api.client.googleapis.testing.auth.oauth2.MockGoogleCredential; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; +import com.google.inject.Provides; +import org.apache.druid.guice.LazySingleton; +import org.apache.druid.initialization.DruidModule; + +import java.util.List; + +public class GcpMockModule implements DruidModule +{ + @Override + public List getJacksonModules() + { + return ImmutableList.of(); + } + + @Override + public void configure(Binder binder) + { + + } + + + @Provides + @LazySingleton + public HttpRequestInitializer mockRequestInitializer( + HttpTransport transport, + JsonFactory factory + ) + { + return new MockGoogleCredential.Builder().setTransport(transport).setJsonFactory(factory).build(); + } + + @Provides + @LazySingleton + public HttpTransport buildMockTransport() + { + return new MockHttpTransport.Builder().build(); + } + + @Provides + @LazySingleton + public JsonFactory getJsonFactory() + { + return JacksonFactory.getDefaultInstance(); + } +} diff --git a/cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpModuleTest.java b/cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpModuleTest.java new file mode 100644 index 000000000000..4790a914b721 --- /dev/null +++ b/cloud/gcp-common/src/test/java/org/apache/druid/common/gcp/GcpModuleTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.common.gcp; + +import com.google.api.client.googleapis.testing.auth.oauth2.MockGoogleCredential; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.inject.Binder; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Scopes; +import com.google.inject.util.Modules; +import org.apache.druid.guice.LazySingleton; +import org.junit.Assert; +import org.junit.Test; + +public class GcpModuleTest +{ + @Test + public void testSimpleModuleLoads() + { + final Injector injector = Guice.createInjector(Modules.override(new GcpModule()).with(new GcpMockModule() + { + @Override + public void configure(Binder binder) + { + binder.bindScope(LazySingleton.class, Scopes.SINGLETON); + } + })); + Assert.assertTrue(injector.getInstance(HttpRequestInitializer.class) instanceof MockGoogleCredential); + Assert.assertTrue(injector.getInstance(HttpTransport.class) instanceof MockHttpTransport); + } +} diff --git a/distribution/pom.xml b/distribution/pom.xml index 96b256e97220..72ac5e6e8e6d 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -157,6 +157,8 @@ -c org.apache.druid.extensions:druid-s3-extensions -c + org.apache.druid.extensions:druid-google-extensions + -c org.apache.druid.extensions:druid-stats -c org.apache.druid.extensions:druid-examples @@ -279,8 +281,6 @@ -c org.apache.druid.extensions.contrib:druid-rocketmq -c - org.apache.druid.extensions.contrib:druid-google-extensions - -c org.apache.druid.extensions.contrib:graphite-emitter -c org.apache.druid.extensions.contrib:druid-influx-extensions diff --git a/extensions-contrib/google-extensions/pom.xml b/extensions-core/google-extensions/pom.xml similarity index 75% rename from extensions-contrib/google-extensions/pom.xml rename to extensions-core/google-extensions/pom.xml index 508aa4880956..9bcfae5af70d 100644 --- a/extensions-contrib/google-extensions/pom.xml +++ b/extensions-core/google-extensions/pom.xml @@ -33,6 +33,10 @@ ../../pom.xml + + v1-rev79-${com.google.apis.client.version} + + org.apache.druid @@ -40,35 +44,21 @@ ${project.parent.version} provided + + org.apache.druid + druid-gcp-common + ${project.parent.version} + provided + com.google.apis google-api-services-storage - v1-rev79-1.22.0 - - - com.google.http-client - google-http-client-jackson2 - 1.22.0 + ${com.google.apis.storage.version} - com.fasterxml.jackson.core - jackson-core - - - - - com.fasterxml.jackson.module - jackson-module-guice - ${jackson.version} - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind + com.google.api-client + google-api-client diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/firehose/google/GoogleBlob.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/firehose/google/GoogleBlob.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/firehose/google/GoogleBlob.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/firehose/google/GoogleBlob.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactory.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactory.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactory.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactory.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleAccountConfig.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleAccountConfig.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleAccountConfig.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleAccountConfig.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleByteSource.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleByteSource.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleByteSource.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleByteSource.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentFinder.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentFinder.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentFinder.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentFinder.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentKiller.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentKiller.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentKiller.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentKiller.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPuller.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPuller.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPuller.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPuller.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPusher.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPusher.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPusher.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleDataSegmentPusher.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleLoadSpec.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleLoadSpec.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleLoadSpec.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleLoadSpec.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorage.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageDruidModule.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageDruidModule.java similarity index 79% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageDruidModule.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageDruidModule.java index 88b9bd804113..5d297fa30145 100644 --- a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageDruidModule.java +++ b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleStorageDruidModule.java @@ -23,13 +23,10 @@ import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.jsontype.NamedType; import com.fasterxml.jackson.databind.module.SimpleModule; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpRequestInitializer; import com.google.api.client.http.HttpTransport; import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; import com.google.api.services.storage.Storage; -import com.google.api.services.storage.StorageScopes; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; import com.google.inject.Provides; @@ -40,15 +37,12 @@ import org.apache.druid.initialization.DruidModule; import org.apache.druid.java.util.common.logger.Logger; -import java.io.IOException; -import java.security.GeneralSecurityException; import java.util.List; public class GoogleStorageDruidModule implements DruidModule { - private static final Logger LOG = new Logger(GoogleStorageDruidModule.class); - public static final String SCHEME = "google"; + private static final Logger LOG = new Logger(GoogleStorageDruidModule.class); private static final String APPLICATION_NAME = "druid-google-extensions"; @Override @@ -103,19 +97,18 @@ public void configure(Binder binder) @Provides @LazySingleton - public GoogleStorage getGoogleStorage(final GoogleAccountConfig config) - throws IOException, GeneralSecurityException + public GoogleStorage getGoogleStorage( + HttpTransport httpTransport, + JsonFactory jsonFactory, + HttpRequestInitializer requestInitializer + ) { LOG.info("Building Cloud Storage Client..."); - HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); - - GoogleCredential credential = GoogleCredential.getApplicationDefault(httpTransport, jsonFactory); - if (credential.createScopedRequired()) { - credential = credential.createScoped(StorageScopes.all()); - } - Storage storage = new Storage.Builder(httpTransport, jsonFactory, credential).setApplicationName(APPLICATION_NAME).build(); + Storage storage = new Storage + .Builder(httpTransport, jsonFactory, requestInitializer) + .setApplicationName(APPLICATION_NAME) + .build(); return new GoogleStorage(storage); } diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogs.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogsConfig.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogsConfig.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogsConfig.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleTaskLogsConfig.java diff --git a/extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleUtils.java b/extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleUtils.java similarity index 100% rename from extensions-contrib/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleUtils.java rename to extensions-core/google-extensions/src/main/java/org/apache/druid/storage/google/GoogleUtils.java diff --git a/extensions-contrib/google-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-core/google-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule similarity index 51% rename from extensions-contrib/google-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule rename to extensions-core/google-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule index 4c2440ca5715..3d89b7320313 100644 --- a/extensions-contrib/google-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule +++ b/extensions-core/google-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -13,23 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - org.apache.druid.storage.google.GoogleStorageDruidModule diff --git a/extensions-contrib/google-extensions/src/test/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactoryTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactoryTest.java similarity index 100% rename from extensions-contrib/google-extensions/src/test/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactoryTest.java rename to extensions-core/google-extensions/src/test/java/org/apache/druid/firehose/google/StaticGoogleBlobStoreFirehoseFactoryTest.java diff --git a/extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleByteSourceTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleByteSourceTest.java similarity index 100% rename from extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleByteSourceTest.java rename to extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleByteSourceTest.java diff --git a/extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentFinderTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentFinderTest.java similarity index 100% rename from extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentFinderTest.java rename to extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentFinderTest.java diff --git a/extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentKillerTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentKillerTest.java similarity index 100% rename from extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentKillerTest.java rename to extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentKillerTest.java diff --git a/extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPullerTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPullerTest.java similarity index 100% rename from extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPullerTest.java rename to extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPullerTest.java diff --git a/extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPusherTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPusherTest.java similarity index 100% rename from extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPusherTest.java rename to extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleDataSegmentPusherTest.java diff --git a/extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTaskLogsTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTaskLogsTest.java similarity index 100% rename from extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTaskLogsTest.java rename to extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleTaskLogsTest.java diff --git a/extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleUtilsTest.java b/extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleUtilsTest.java similarity index 100% rename from extensions-contrib/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleUtilsTest.java rename to extensions-core/google-extensions/src/test/java/org/apache/druid/storage/google/GoogleUtilsTest.java diff --git a/pom.xml b/pom.xml index 668ba3e17702..df0aebcbf15f 100644 --- a/pom.xml +++ b/pom.xml @@ -101,6 +101,8 @@ 3.4.11 2.5.7 + 1.22.0 + apache.snapshots Apache Snapshot Repository https://repository.apache.org/snapshots @@ -120,9 +122,11 @@ services integration-tests benchmarks - aws-common extendedset hll + + cloud/aws-common + cloud/gcp-common extensions-core/avro-extensions extensions-core/datasketches @@ -144,6 +148,7 @@ extensions-core/s3-extensions extensions-core/simple-client-sslcontext extensions-core/druid-basic-security + extensions-core/google-extensions extensions-contrib/influx-extensions extensions-contrib/azure-extensions @@ -157,7 +162,6 @@ extensions-contrib/statsd-emitter extensions-contrib/orc-extensions extensions-contrib/time-min-max - extensions-contrib/google-extensions extensions-contrib/virtual-columns extensions-contrib/thrift-extensions extensions-contrib/ambari-metrics-emitter @@ -863,6 +867,72 @@ + + + + org.apache.druid + druid-core + ${project.parent.version} + provided + + + com.google.api-client + google-api-client + ${com.google.apis.client.version} + + + com.google.code.findbugs + jsr305 + + + com.fasterxml.jackson.core + jackson-core + + + com.google.guava + guava-jdk5 + + + org.apache.httpcomponents + httpcore + + + org.apache.httpcomponents + httpclient + + + + + com.google.http-client + google-http-client-jackson2 + ${com.google.apis.client.version} + + + com.fasterxml.jackson.core + jackson-core + + + org.apache.httpcomponents + httpclient + + + + + com.fasterxml.jackson.module + jackson-module-guice + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + + org.testng testng diff --git a/server/pom.xml b/server/pom.xml index 501d336893cd..cde0e993df43 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -42,6 +42,11 @@ druid-aws-common ${project.parent.version} + + org.apache.druid + druid-gcp-common + ${project.parent.version} + io.druid druid-console From b9d2b9632ea277cfd54869a227dfb77ce18c2b55 Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Tue, 29 Jan 2019 16:24:27 -0800 Subject: [PATCH 02/14] Don't provide druid-core >.< --- pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pom.xml b/pom.xml index df0aebcbf15f..0c32266b3d53 100644 --- a/pom.xml +++ b/pom.xml @@ -869,12 +869,6 @@ - - org.apache.druid - druid-core - ${project.parent.version} - provided - com.google.api-client google-api-client From 21e1f2d069d353cba511b15bc2467d4c73239a49 Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Tue, 29 Jan 2019 16:35:18 -0800 Subject: [PATCH 03/14] Keep AWS and GCP modules separate --- extensions-core/google-extensions/pom.xml | 1 - extensions-core/s3-extensions/pom.xml | 1 - server/pom.xml | 10 ---------- 3 files changed, 12 deletions(-) diff --git a/extensions-core/google-extensions/pom.xml b/extensions-core/google-extensions/pom.xml index 9bcfae5af70d..5db7dc00ffdf 100644 --- a/extensions-core/google-extensions/pom.xml +++ b/extensions-core/google-extensions/pom.xml @@ -48,7 +48,6 @@ org.apache.druid druid-gcp-common ${project.parent.version} - provided diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index c61874adaf2b..7924feba6323 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -45,7 +45,6 @@ org.apache.druid druid-aws-common ${project.parent.version} - provided commons-io diff --git a/server/pom.xml b/server/pom.xml index cde0e993df43..9d5e0bd69d34 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -37,16 +37,6 @@ druid-processing ${project.parent.version} - - org.apache.druid - druid-aws-common - ${project.parent.version} - - - org.apache.druid - druid-gcp-common - ${project.parent.version} - io.druid druid-console From a8f322a4540ef3a5b57b739e56c9a5b4bfb2cc67 Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Tue, 29 Jan 2019 17:27:44 -0800 Subject: [PATCH 04/14] Move AWSModule to its own module --- .../apache/druid/common/aws}/AWSModule.java | 25 +++++++++++-------- ...rg.apache.druid.initialization.DruidModule | 20 +++++++++++++++ .../s3/TestAWSCredentialsProvider.java | 2 +- .../druid/initialization/Initialization.java | 2 -- 4 files changed, 36 insertions(+), 13 deletions(-) rename {server/src/main/java/org/apache/druid/guice => cloud/aws-common/src/main/java/org/apache/druid/common/aws}/AWSModule.java (79%) create mode 100644 cloud/aws-common/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule diff --git a/server/src/main/java/org/apache/druid/guice/AWSModule.java b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSModule.java similarity index 79% rename from server/src/main/java/org/apache/druid/guice/AWSModule.java rename to cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSModule.java index 9d0c568e8b4c..79b60c956c34 100644 --- a/server/src/main/java/org/apache/druid/guice/AWSModule.java +++ b/cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSModule.java @@ -17,23 +17,22 @@ * under the License. */ -package org.apache.druid.guice; +package org.apache.druid.common.aws; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2Client; +import com.fasterxml.jackson.databind.Module; import com.google.inject.Binder; -import com.google.inject.Module; import com.google.inject.Provides; -import org.apache.druid.common.aws.AWSClientConfig; -import org.apache.druid.common.aws.AWSCredentialsConfig; -import org.apache.druid.common.aws.AWSCredentialsUtils; -import org.apache.druid.common.aws.AWSEndpointConfig; -import org.apache.druid.common.aws.AWSProxyConfig; +import org.apache.druid.guice.JsonConfigProvider; +import org.apache.druid.guice.LazySingleton; +import org.apache.druid.initialization.DruidModule; -/** - */ -public class AWSModule implements Module +import java.util.Collections; +import java.util.List; + +public class AWSModule implements DruidModule { @Override public void configure(Binder binder) @@ -57,4 +56,10 @@ public AmazonEC2 getEc2Client(AWSCredentialsProvider credentials) { return new AmazonEC2Client(credentials); } + + @Override + public List getJacksonModules() + { + return Collections.emptyList(); + } } diff --git a/cloud/aws-common/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/cloud/aws-common/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 000000000000..2d43a9c333df --- /dev/null +++ b/cloud/aws-common/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.druid.common.aws.AWSModule diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/TestAWSCredentialsProvider.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/TestAWSCredentialsProvider.java index 27534a498c69..d2ec870fc6fa 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/TestAWSCredentialsProvider.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/TestAWSCredentialsProvider.java @@ -26,8 +26,8 @@ import org.apache.druid.common.aws.AWSClientConfig; import org.apache.druid.common.aws.AWSCredentialsConfig; import org.apache.druid.common.aws.AWSEndpointConfig; +import org.apache.druid.common.aws.AWSModule; import org.apache.druid.common.aws.AWSProxyConfig; -import org.apache.druid.guice.AWSModule; import org.apache.druid.metadata.DefaultPasswordProvider; import org.easymock.EasyMock; import org.junit.Rule; diff --git a/server/src/main/java/org/apache/druid/initialization/Initialization.java b/server/src/main/java/org/apache/druid/initialization/Initialization.java index fcf288ac6a30..0d912295be38 100644 --- a/server/src/main/java/org/apache/druid/initialization/Initialization.java +++ b/server/src/main/java/org/apache/druid/initialization/Initialization.java @@ -30,7 +30,6 @@ import org.apache.commons.io.FileUtils; import org.apache.druid.curator.CuratorModule; import org.apache.druid.curator.discovery.DiscoveryModule; -import org.apache.druid.guice.AWSModule; import org.apache.druid.guice.AnnouncerModule; import org.apache.druid.guice.CoordinatorDiscoveryModule; import org.apache.druid.guice.DruidProcessingConfigModule; @@ -376,7 +375,6 @@ public static Injector makeInjectorWithModules(final Injector baseInjector, Iter new HttpClientModule("druid.broker.http", EscalatedClient.class), new CuratorModule(), new AnnouncerModule(), - new AWSModule(), new MetricsModule(), new SegmentWriteOutMediumModule(), new ServerModule(), From 0bd304bd59ae50e05d86650ad029e20de0c4deea Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Tue, 29 Jan 2019 20:00:56 -0800 Subject: [PATCH 05/14] Add aws ec2 extension and more modules in more places --- extensions-core/ec2-extensions/pom.xml | 63 +++++++++++++++++ .../autoscaling/ec2/EC2AutoScaler.java | 2 + .../autoscaling/ec2/EC2EnvironmentConfig.java | 0 .../autoscaling/ec2/EC2IamProfileData.java | 0 .../overlord/autoscaling/ec2/EC2Module.java | 43 ++++++++++++ .../overlord/autoscaling/ec2/EC2NodeData.java | 0 .../overlord/autoscaling/ec2/EC2UserData.java | 0 .../autoscaling/ec2/GalaxyEC2UserData.java | 0 .../autoscaling/ec2/StringEC2UserData.java | 0 ...rg.apache.druid.initialization.DruidModule | 16 +++++ .../ec2}/EC2AutoScalerSerdeTest.java | 70 ++++++++++--------- .../autoscaling/ec2}/EC2AutoScalerTest.java | 4 +- .../autoscaling/ec2/EC2NodeDataTest.java | 0 .../setup/WorkerBehaviorConfigTest.java | 0 .../kinesis-indexing-service/pom.xml | 12 ++++ .../overlord/autoscaling/AutoScaler.java | 6 +- pom.xml | 1 + 17 files changed, 177 insertions(+), 40 deletions(-) create mode 100644 extensions-core/ec2-extensions/pom.xml rename {indexing-service => extensions-core/ec2-extensions}/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java (99%) rename {indexing-service => extensions-core/ec2-extensions}/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2EnvironmentConfig.java (100%) rename {indexing-service => extensions-core/ec2-extensions}/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2IamProfileData.java (100%) create mode 100644 extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2Module.java rename {indexing-service => extensions-core/ec2-extensions}/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeData.java (100%) rename {indexing-service => extensions-core/ec2-extensions}/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2UserData.java (100%) rename {indexing-service => extensions-core/ec2-extensions}/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/GalaxyEC2UserData.java (100%) rename {indexing-service => extensions-core/ec2-extensions}/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/StringEC2UserData.java (100%) create mode 100644 extensions-core/ec2-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule rename {indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling => extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2}/EC2AutoScalerSerdeTest.java (94%) rename {indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling => extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2}/EC2AutoScalerTest.java (97%) rename {indexing-service => extensions-core/ec2-extensions}/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeDataTest.java (100%) rename {indexing-service => extensions-core/ec2-extensions}/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java (100%) diff --git a/extensions-core/ec2-extensions/pom.xml b/extensions-core/ec2-extensions/pom.xml new file mode 100644 index 000000000000..a2c778a1509c --- /dev/null +++ b/extensions-core/ec2-extensions/pom.xml @@ -0,0 +1,63 @@ + + + + + 4.0.0 + + org.apache.druid.extensions + druid-ec2-extensions + druid-ec2-extensions + druid-ec2-extensions + + + org.apache.druid + druid + 0.13.0-incubating-SNAPSHOT + ../../pom.xml + + + + + org.apache.druid + druid-indexing-service + ${project.parent.version} + provided + + + org.apache.druid + druid-aws-common + ${project.parent.version} + + + + + junit + junit + test + + + org.easymock + easymock + test + + + + diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java similarity index 99% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java rename to extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java index 304023770068..56ffc6b74efc 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java +++ b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScaler.java @@ -33,6 +33,7 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Function; import com.google.common.collect.FluentIterable; import com.google.common.collect.Lists; @@ -46,6 +47,7 @@ /** */ +@JsonTypeName("ec2") public class EC2AutoScaler implements AutoScaler { private static final EmittingLogger log = new EmittingLogger(EC2AutoScaler.class); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2EnvironmentConfig.java b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2EnvironmentConfig.java similarity index 100% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2EnvironmentConfig.java rename to extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2EnvironmentConfig.java diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2IamProfileData.java b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2IamProfileData.java similarity index 100% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2IamProfileData.java rename to extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2IamProfileData.java diff --git a/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2Module.java b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2Module.java new file mode 100644 index 000000000000..aa533d94cc48 --- /dev/null +++ b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2Module.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.autoscaling.ec2; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import org.apache.druid.initialization.DruidModule; + +import java.util.Collections; +import java.util.List; + +public class EC2Module implements DruidModule +{ + @Override + public List getJacksonModules() + { + return Collections.singletonList(new SimpleModule("DruidEc2Module").registerSubtypes(EC2AutoScaler.class)); + } + + @Override + public void configure(Binder binder) + { + + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeData.java b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeData.java similarity index 100% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeData.java rename to extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeData.java diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2UserData.java b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2UserData.java similarity index 100% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2UserData.java rename to extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2UserData.java diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/GalaxyEC2UserData.java b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/GalaxyEC2UserData.java similarity index 100% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/GalaxyEC2UserData.java rename to extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/GalaxyEC2UserData.java diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/StringEC2UserData.java b/extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/StringEC2UserData.java similarity index 100% rename from indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/StringEC2UserData.java rename to extensions-core/ec2-extensions/src/main/java/org/apache/druid/indexing/overlord/autoscaling/ec2/StringEC2UserData.java diff --git a/extensions-core/ec2-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-core/ec2-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 000000000000..082c02d12654 --- /dev/null +++ b/extensions-core/ec2-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.druid.indexing.overlord.autoscaling.ec2.EC2Module diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/EC2AutoScalerSerdeTest.java b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerSerdeTest.java similarity index 94% rename from indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/EC2AutoScalerSerdeTest.java rename to extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerSerdeTest.java index f49d6dcf20bb..29ba9a0c5a3b 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/EC2AutoScalerSerdeTest.java +++ b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerSerdeTest.java @@ -17,13 +17,14 @@ * under the License. */ -package org.apache.druid.indexing.overlord.autoscaling; +package org.apache.druid.indexing.overlord.autoscaling.ec2; import com.fasterxml.jackson.databind.BeanProperty; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.InjectableValues; +import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2AutoScaler; +import org.apache.druid.indexing.overlord.autoscaling.AutoScaler; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.StringUtils; import org.junit.Assert; @@ -57,38 +58,6 @@ public class EC2AutoScalerSerdeTest + " \"type\" : \"ec2\"\n" + "}"; - @Test - public void testSerde() throws Exception - { - final ObjectMapper objectMapper = new DefaultObjectMapper(); - objectMapper.setInjectableValues( - new InjectableValues() - { - @Override - public Object findInjectableValue( - Object o, - DeserializationContext deserializationContext, - BeanProperty beanProperty, - Object o1 - ) - { - return null; - } - } - ); - - final EC2AutoScaler autoScaler = (EC2AutoScaler) objectMapper.readValue(json, AutoScaler.class); - verifyAutoScaler(autoScaler); - - final EC2AutoScaler roundTripAutoScaler = (EC2AutoScaler) objectMapper.readValue( - objectMapper.writeValueAsBytes(autoScaler), - AutoScaler.class - ); - verifyAutoScaler(roundTripAutoScaler); - - Assert.assertEquals("Round trip equals", autoScaler, roundTripAutoScaler); - } - private static void verifyAutoScaler(final EC2AutoScaler autoScaler) { Assert.assertEquals(3, autoScaler.getMaxNumWorkers()); @@ -132,4 +101,37 @@ private static void verifyAutoScaler(final EC2AutoScaler autoScaler) ) ); } + + @Test + public void testSerde() throws Exception + { + final ObjectMapper objectMapper = new DefaultObjectMapper() + .registerModules((Iterable) new EC2Module().getJacksonModules()); + objectMapper.setInjectableValues( + new InjectableValues() + { + @Override + public Object findInjectableValue( + Object o, + DeserializationContext deserializationContext, + BeanProperty beanProperty, + Object o1 + ) + { + return null; + } + } + ); + + final EC2AutoScaler autoScaler = (EC2AutoScaler) objectMapper.readValue(json, AutoScaler.class); + verifyAutoScaler(autoScaler); + + final EC2AutoScaler roundTripAutoScaler = (EC2AutoScaler) objectMapper.readValue( + objectMapper.writeValueAsBytes(autoScaler), + AutoScaler.class + ); + verifyAutoScaler(roundTripAutoScaler); + + Assert.assertEquals("Round trip equals", autoScaler, roundTripAutoScaler); + } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/EC2AutoScalerTest.java b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerTest.java similarity index 97% rename from indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/EC2AutoScalerTest.java rename to extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerTest.java index efdc794ae041..5275895c0c4a 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/EC2AutoScalerTest.java +++ b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.indexing.overlord.autoscaling; +package org.apache.druid.indexing.overlord.autoscaling.ec2; import com.amazonaws.services.ec2.AmazonEC2Client; import com.amazonaws.services.ec2.model.DescribeInstancesRequest; @@ -33,6 +33,8 @@ import com.google.common.collect.DiscreteDomain; import com.google.common.collect.Lists; import com.google.common.collect.Range; +import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData; +import org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig; import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2AutoScaler; import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2EnvironmentConfig; import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2NodeData; diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeDataTest.java b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeDataTest.java similarity index 100% rename from indexing-service/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeDataTest.java rename to extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2NodeDataTest.java diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java similarity index 100% rename from indexing-service/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java rename to extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java diff --git a/extensions-core/kinesis-indexing-service/pom.xml b/extensions-core/kinesis-indexing-service/pom.xml index 8469334641a0..33d6b1598b0b 100644 --- a/extensions-core/kinesis-indexing-service/pom.xml +++ b/extensions-core/kinesis-indexing-service/pom.xml @@ -66,10 +66,22 @@ + + + org.apache.druid + druid-aws-common + ${project.parent.version} + com.amazonaws aws-java-sdk-sts ${aws.sdk.version} + + + aws-java-sdk-core + com.amazonaws + + diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/AutoScaler.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/AutoScaler.java index 1bbaf8b955a8..76b20cf42287 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/AutoScaler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/autoscaling/AutoScaler.java @@ -19,10 +19,8 @@ package org.apache.druid.indexing.overlord.autoscaling; -import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.apache.druid.guice.annotations.ExtensionPoint; -import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2AutoScaler; import javax.annotation.Nullable; import java.util.List; @@ -31,9 +29,6 @@ * The AutoScaler has the actual methods to provision and terminate worker nodes. */ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = NoopAutoScaler.class) -@JsonSubTypes(value = { - @JsonSubTypes.Type(name = "ec2", value = EC2AutoScaler.class) -}) @ExtensionPoint public interface AutoScaler { @@ -71,6 +66,7 @@ public interface AutoScaler * This method is unused, but AutoScaler is an {@link ExtensionPoint}, so we cannot remove it. * * @param nodeIds - nodes ids + * * @return IPs associated with the node */ @SuppressWarnings("unused") diff --git a/pom.xml b/pom.xml index 0c32266b3d53..a83e3bb5d753 100644 --- a/pom.xml +++ b/pom.xml @@ -145,6 +145,7 @@ extensions-core/protobuf-extensions extensions-core/lookups-cached-global extensions-core/lookups-cached-single + extensions-core/ec2-extensions extensions-core/s3-extensions extensions-core/simple-client-sslcontext extensions-core/druid-basic-security From d463c00cc1929bce6a6196d5a6518ea717a6b26e Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Tue, 29 Jan 2019 20:09:47 -0800 Subject: [PATCH 06/14] Fix bad imports --- .../indexing/overlord/autoscaling/ec2/EC2AutoScalerTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerTest.java b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerTest.java index 5275895c0c4a..bdeb9e81221a 100644 --- a/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerTest.java +++ b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/autoscaling/ec2/EC2AutoScalerTest.java @@ -35,10 +35,6 @@ import com.google.common.collect.Range; import org.apache.druid.indexing.overlord.autoscaling.AutoScalingData; import org.apache.druid.indexing.overlord.autoscaling.SimpleWorkerProvisioningConfig; -import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2AutoScaler; -import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2EnvironmentConfig; -import org.apache.druid.indexing.overlord.autoscaling.ec2.EC2NodeData; -import org.apache.druid.indexing.overlord.autoscaling.ec2.GalaxyEC2UserData; import org.apache.druid.jackson.DefaultObjectMapper; import org.easymock.EasyMock; import org.junit.After; From 05b72737ea0ac57820b7519247da48cb4e57f843 Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Wed, 30 Jan 2019 10:11:30 -0800 Subject: [PATCH 07/14] Fix test jackson module --- .../druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java index d15264191afd..a47a79fa9bc1 100644 --- a/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java +++ b/extensions-core/ec2-extensions/src/test/java/org/apache/druid/indexing/overlord/setup/WorkerBehaviorConfigTest.java @@ -75,6 +75,7 @@ public void testSerde() throws Exception ); final ObjectMapper mapper = new DefaultObjectMapper(); + mapper.registerSubtypes(EC2AutoScaler.class); mapper.setInjectableValues( new InjectableValues() { From fbbb666ba40ff950e2c33e329e78b75fc80a8c53 Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Wed, 30 Jan 2019 12:41:53 -0800 Subject: [PATCH 08/14] Include AWS and GCP core in server --- extensions-core/ec2-extensions/pom.xml | 1 + extensions-core/google-extensions/pom.xml | 1 + extensions-core/kinesis-indexing-service/pom.xml | 1 + extensions-core/s3-extensions/pom.xml | 1 + server/pom.xml | 10 ++++++++++ 5 files changed, 14 insertions(+) diff --git a/extensions-core/ec2-extensions/pom.xml b/extensions-core/ec2-extensions/pom.xml index a2c778a1509c..fc4f70d73dc5 100644 --- a/extensions-core/ec2-extensions/pom.xml +++ b/extensions-core/ec2-extensions/pom.xml @@ -45,6 +45,7 @@ org.apache.druid druid-aws-common ${project.parent.version} + provided diff --git a/extensions-core/google-extensions/pom.xml b/extensions-core/google-extensions/pom.xml index 5db7dc00ffdf..9bcfae5af70d 100644 --- a/extensions-core/google-extensions/pom.xml +++ b/extensions-core/google-extensions/pom.xml @@ -48,6 +48,7 @@ org.apache.druid druid-gcp-common ${project.parent.version} + provided diff --git a/extensions-core/kinesis-indexing-service/pom.xml b/extensions-core/kinesis-indexing-service/pom.xml index 33d6b1598b0b..f916a3c37c04 100644 --- a/extensions-core/kinesis-indexing-service/pom.xml +++ b/extensions-core/kinesis-indexing-service/pom.xml @@ -71,6 +71,7 @@ org.apache.druid druid-aws-common ${project.parent.version} + provided com.amazonaws diff --git a/extensions-core/s3-extensions/pom.xml b/extensions-core/s3-extensions/pom.xml index 7924feba6323..c61874adaf2b 100644 --- a/extensions-core/s3-extensions/pom.xml +++ b/extensions-core/s3-extensions/pom.xml @@ -45,6 +45,7 @@ org.apache.druid druid-aws-common ${project.parent.version} + provided commons-io diff --git a/server/pom.xml b/server/pom.xml index 9d5e0bd69d34..cde0e993df43 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -37,6 +37,16 @@ druid-processing ${project.parent.version} + + org.apache.druid + druid-aws-common + ${project.parent.version} + + + org.apache.druid + druid-gcp-common + ${project.parent.version} + io.druid druid-console From 371c66cdcfd95c66f2148febb3952bd4796c96c8 Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Wed, 6 Mar 2019 12:47:14 -0800 Subject: [PATCH 09/14] Add simple empty method comment --- .../src/main/java/org/apache/druid/common/gcp/GcpModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/gcp-common/src/main/java/org/apache/druid/common/gcp/GcpModule.java b/cloud/gcp-common/src/main/java/org/apache/druid/common/gcp/GcpModule.java index c56a38aa4031..56458ee75bcc 100644 --- a/cloud/gcp-common/src/main/java/org/apache/druid/common/gcp/GcpModule.java +++ b/cloud/gcp-common/src/main/java/org/apache/druid/common/gcp/GcpModule.java @@ -48,7 +48,7 @@ public List getJacksonModules() @Override public void configure(Binder binder) { - + // Nothing to proactively bind } From 23f28427c08f29eb977861fd83dddb251d40fd19 Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Wed, 6 Mar 2019 13:04:25 -0800 Subject: [PATCH 10/14] Update version to 15 --- cloud/gcp-common/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/gcp-common/pom.xml b/cloud/gcp-common/pom.xml index fa249aa55693..7b914aa5188a 100644 --- a/cloud/gcp-common/pom.xml +++ b/cloud/gcp-common/pom.xml @@ -28,7 +28,7 @@ org.apache.druid druid - 0.13.0-incubating-SNAPSHOT + 0.15.0-incubating-SNAPSHOT ../../pom.xml From 972d5f65fc41531b8332409227e4c67120f4c728 Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Wed, 6 Mar 2019 13:31:37 -0800 Subject: [PATCH 11/14] One more 0.13.0-->0.15.0 change --- extensions-core/ec2-extensions/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-core/ec2-extensions/pom.xml b/extensions-core/ec2-extensions/pom.xml index fc4f70d73dc5..fb399489712d 100644 --- a/extensions-core/ec2-extensions/pom.xml +++ b/extensions-core/ec2-extensions/pom.xml @@ -30,7 +30,7 @@ org.apache.druid druid - 0.13.0-incubating-SNAPSHOT + 0.15.0-incubating-SNAPSHOT ../../pom.xml From 788e97284fc114bd2b9119efcc29aab0223c7ebb Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Thu, 7 Mar 2019 09:53:36 -0800 Subject: [PATCH 12/14] Fix multi-binding problem --- .../storage/s3/S3StorageDruidModule.java | 106 +++++++++--------- 1 file changed, 51 insertions(+), 55 deletions(-) diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java index 61d145d591e8..2daaadbe7e22 100644 --- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java +++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java @@ -34,7 +34,6 @@ import com.google.inject.multibindings.MapBinder; import org.apache.commons.lang.StringUtils; import org.apache.druid.common.aws.AWSClientConfig; -import org.apache.druid.common.aws.AWSCredentialsConfig; import org.apache.druid.common.aws.AWSEndpointConfig; import org.apache.druid.common.aws.AWSProxyConfig; import org.apache.druid.data.SearchableVersionedDataFinder; @@ -51,6 +50,7 @@ import java.util.List; /** + * */ public class S3StorageDruidModule implements DruidModule { @@ -58,6 +58,56 @@ public class S3StorageDruidModule implements DruidModule private static final Logger log = new Logger(S3StorageDruidModule.class); + private static ClientConfiguration setProxyConfig(ClientConfiguration conf, AWSProxyConfig proxyConfig) + { + if (StringUtils.isNotEmpty(proxyConfig.getHost())) { + conf.setProxyHost(proxyConfig.getHost()); + } + if (proxyConfig.getPort() != -1) { + conf.setProxyPort(proxyConfig.getPort()); + } + if (StringUtils.isNotEmpty(proxyConfig.getUsername())) { + conf.setProxyUsername(proxyConfig.getUsername()); + } + if (StringUtils.isNotEmpty(proxyConfig.getPassword())) { + conf.setProxyPassword(proxyConfig.getPassword()); + } + return conf; + } + + @Nullable + private static Protocol parseProtocol(@Nullable String protocol) + { + if (protocol == null) { + return null; + } + + if (protocol.equalsIgnoreCase("http")) { + return Protocol.HTTP; + } else if (protocol.equalsIgnoreCase("https")) { + return Protocol.HTTPS; + } else { + throw new IAE("Unknown protocol[%s]", protocol); + } + } + + private static Protocol determineProtocol(AWSClientConfig clientConfig, AWSEndpointConfig endpointConfig) + { + final Protocol protocolFromClientConfig = parseProtocol(clientConfig.getProtocol()); + final String endpointUrl = endpointConfig.getUrl(); + if (StringUtils.isNotEmpty(endpointUrl)) { + //noinspection ConstantConditions + final URI uri = URIs.parse(endpointUrl, protocolFromClientConfig.toString()); + final Protocol protocol = parseProtocol(uri.getScheme()); + if (protocol != null && (protocol != protocolFromClientConfig)) { + log.warn("[%s] protocol will be used for endpoint [%s]", protocol, endpointUrl); + } + return protocol; + } else { + return protocolFromClientConfig; + } + } + @Override public List getJacksonModules() { @@ -88,10 +138,6 @@ public void setupModule(SetupContext context) @Override public void configure(Binder binder) { - JsonConfigProvider.bind(binder, "druid.s3", AWSCredentialsConfig.class); - JsonConfigProvider.bind(binder, "druid.s3", AWSClientConfig.class); - JsonConfigProvider.bind(binder, "druid.s3.proxy", AWSProxyConfig.class); - JsonConfigProvider.bind(binder, "druid.s3.endpoint", AWSEndpointConfig.class); MapBinder.newMapBinder(binder, String.class, SearchableVersionedDataFinder.class) .addBinding("s3") .to(S3TimestampVersionedDataFinder.class) @@ -146,54 +192,4 @@ public ServerSideEncryptingAmazonS3 getAmazonS3Client( return new ServerSideEncryptingAmazonS3(builder.build(), storageConfig.getServerSideEncryption()); } - - private static ClientConfiguration setProxyConfig(ClientConfiguration conf, AWSProxyConfig proxyConfig) - { - if (StringUtils.isNotEmpty(proxyConfig.getHost())) { - conf.setProxyHost(proxyConfig.getHost()); - } - if (proxyConfig.getPort() != -1) { - conf.setProxyPort(proxyConfig.getPort()); - } - if (StringUtils.isNotEmpty(proxyConfig.getUsername())) { - conf.setProxyUsername(proxyConfig.getUsername()); - } - if (StringUtils.isNotEmpty(proxyConfig.getPassword())) { - conf.setProxyPassword(proxyConfig.getPassword()); - } - return conf; - } - - @Nullable - private static Protocol parseProtocol(@Nullable String protocol) - { - if (protocol == null) { - return null; - } - - if (protocol.equalsIgnoreCase("http")) { - return Protocol.HTTP; - } else if (protocol.equalsIgnoreCase("https")) { - return Protocol.HTTPS; - } else { - throw new IAE("Unknown protocol[%s]", protocol); - } - } - - private static Protocol determineProtocol(AWSClientConfig clientConfig, AWSEndpointConfig endpointConfig) - { - final Protocol protocolFromClientConfig = parseProtocol(clientConfig.getProtocol()); - final String endpointUrl = endpointConfig.getUrl(); - if (StringUtils.isNotEmpty(endpointUrl)) { - //noinspection ConstantConditions - final URI uri = URIs.parse(endpointUrl, protocolFromClientConfig.toString()); - final Protocol protocol = parseProtocol(uri.getScheme()); - if (protocol != null && (protocol != protocolFromClientConfig)) { - log.warn("[%s] protocol will be used for endpoint [%s]", protocol, endpointUrl); - } - return protocol; - } else { - return protocolFromClientConfig; - } - } } From 50fba4078cf35842da5c9e878c0a04b7fb3bdac0 Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Tue, 26 Mar 2019 13:29:11 -0700 Subject: [PATCH 13/14] Grep for s3-extensions and update docs --- distribution/pom.xml | 2 ++ docs/content/development/extensions.md | 1 + 2 files changed, 3 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index c400aa97d237..7579b46b6d86 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -178,6 +178,8 @@ -c org.apache.druid.extensions:druid-s3-extensions -c + org.apache.druid.extensions:druid-ec2-extensions + -c org.apache.druid.extensions:druid-google-extensions -c org.apache.druid.extensions:druid-stats diff --git a/docs/content/development/extensions.md b/docs/content/development/extensions.md index f24bfc687e17..5e9578da2458 100644 --- a/docs/content/development/extensions.md +++ b/docs/content/development/extensions.md @@ -58,6 +58,7 @@ Core extensions are maintained by Druid committers. |druid-parquet-extensions|Support for data in Apache Parquet data format. Requires druid-avro-extensions to be loaded.|[link](../development/extensions-core/parquet.html)| |druid-protobuf-extensions| Support for data in Protobuf data format.|[link](../development/extensions-core/protobuf.html)| |druid-s3-extensions|Interfacing with data in AWS S3, and using S3 as deep storage.|[link](../development/extensions-core/s3.html)| +|druid-ec2-extensions|Interfacing with data in AWS EC2 for autoscaling middle managers|UNDOCUMENTED| |druid-stats|Statistics related module including variance and standard deviation.|[link](../development/extensions-core/stats.html)| |mysql-metadata-storage|MySQL metadata store.|[link](../development/extensions-core/mysql.html)| |postgresql-metadata-storage|PostgreSQL metadata store.|[link](../development/extensions-core/postgresql.html)| From 15b58ce321813303423429bb78c08cd7aba29532 Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Tue, 26 Mar 2019 16:38:42 -0700 Subject: [PATCH 14/14] Update extensions.md --- docs/content/development/extensions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/development/extensions.md b/docs/content/development/extensions.md index 5e9578da2458..1775edbd725e 100644 --- a/docs/content/development/extensions.md +++ b/docs/content/development/extensions.md @@ -58,7 +58,7 @@ Core extensions are maintained by Druid committers. |druid-parquet-extensions|Support for data in Apache Parquet data format. Requires druid-avro-extensions to be loaded.|[link](../development/extensions-core/parquet.html)| |druid-protobuf-extensions| Support for data in Protobuf data format.|[link](../development/extensions-core/protobuf.html)| |druid-s3-extensions|Interfacing with data in AWS S3, and using S3 as deep storage.|[link](../development/extensions-core/s3.html)| -|druid-ec2-extensions|Interfacing with data in AWS EC2 for autoscaling middle managers|UNDOCUMENTED| +|druid-ec2-extensions|Interfacing with AWS EC2 for autoscaling middle managers|UNDOCUMENTED| |druid-stats|Statistics related module including variance and standard deviation.|[link](../development/extensions-core/stats.html)| |mysql-metadata-storage|MySQL metadata store.|[link](../development/extensions-core/mysql.html)| |postgresql-metadata-storage|PostgreSQL metadata store.|[link](../development/extensions-core/postgresql.html)|