From 1cda01f78d424d3e02768635add18b64c69897e2 Mon Sep 17 00:00:00 2001 From: frank chen Date: Sun, 10 May 2020 15:59:45 +0800 Subject: [PATCH 01/24] init commit, all tests passed --- extensions-contrib/aliyun-extensions/pom.xml | 209 ++++++++++ .../druid/data/input/aliyun/OssEntity.java | 89 +++++ .../data/input/aliyun/OssInputSource.java | 186 +++++++++ .../input/aliyun/OssInputSourceConfig.java | 110 ++++++ .../aliyun/OssInputSourceDruidModule.java | 51 +++ .../aliyun/OssFirehoseDruidModule.java | 48 +++ .../aliyun/StaticOssFirehoseFactory.java | 238 ++++++++++++ .../storage/aliyun/ObjectSummaryIterator.java | 172 +++++++++ .../druid/storage/aliyun/OssClientHelper.java | 130 +++++++ .../aliyun/OssDataSegmentArchiver.java | 103 +++++ .../aliyun/OssDataSegmentArchiverConfig.java | 41 ++ .../storage/aliyun/OssDataSegmentKiller.java | 109 ++++++ .../storage/aliyun/OssDataSegmentMover.java | 252 ++++++++++++ .../storage/aliyun/OssDataSegmentPuller.java | 311 +++++++++++++++ .../storage/aliyun/OssDataSegmentPusher.java | 135 +++++++ .../aliyun/OssDataSegmentPusherConfig.java | 82 ++++ .../storage/aliyun/OssInputDataConfig.java | 49 +++ .../druid/storage/aliyun/OssLoadSpec.java | 75 ++++ .../storage/aliyun/OssStorageConfig.java | 28 ++ .../storage/aliyun/OssStorageDruidModule.java | 99 +++++ .../druid/storage/aliyun/OssTaskLogs.java | 202 ++++++++++ .../storage/aliyun/OssTaskLogsConfig.java | 75 ++++ .../OssTimestampVersionedDataFinder.java | 95 +++++ .../apache/druid/storage/aliyun/OssUtils.java | 288 ++++++++++++++ ...rg.apache.druid.initialization.DruidModule | 18 + .../aliyun/ObjectSummaryIteratorTest.java | 253 +++++++++++++ .../aliyun/OssDataSegmentArchiverTest.java | 176 +++++++++ .../aliyun/OssDataSegmentKillerTest.java | 206 ++++++++++ .../aliyun/OssDataSegmentMoverTest.java | 268 +++++++++++++ .../aliyun/OssDataSegmentPullerTest.java | 206 ++++++++++ .../OssDataSegmentPusherConfigTest.java | 81 ++++ .../aliyun/OssDataSegmentPusherTest.java | 122 ++++++ .../druid/storage/aliyun/OssTaskLogsTest.java | 358 ++++++++++++++++++ .../druid/storage/aliyun/OssTestUtils.java | 176 +++++++++ .../OssTimestampVersionedDataFinderTest.java | 179 +++++++++ 35 files changed, 5220 insertions(+) create mode 100644 extensions-contrib/aliyun-extensions/pom.xml create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssClientHelper.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfig.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java create mode 100644 extensions-contrib/aliyun-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule create mode 100644 extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java create mode 100644 extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java create mode 100644 extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java create mode 100644 extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java create mode 100644 extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java create mode 100644 extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java create mode 100644 extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java create mode 100644 extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java create mode 100644 extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTestUtils.java create mode 100644 extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java diff --git a/extensions-contrib/aliyun-extensions/pom.xml b/extensions-contrib/aliyun-extensions/pom.xml new file mode 100644 index 000000000000..d8400964e26a --- /dev/null +++ b/extensions-contrib/aliyun-extensions/pom.xml @@ -0,0 +1,209 @@ + + + + + 4.0.0 + + org.apache.druid.extensions + druid-aliyun-extensions + druid-aliyun-extensions + druid-aliyun-extensions + + + org.apache.druid + druid + 0.19.0-SNAPSHOT + ../../pom.xml + + + + + org.apache.druid + druid-core + ${project.parent.version} + provided + + + com.aliyun.oss + aliyun-sdk-oss + 3.3.0 + + + + com.fasterxml.jackson.module + jackson-module-guice + ${jackson.version} + provided + + + commons-io + commons-io + provided + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + joda-time + joda-time + provided + + + com.google.inject + guice + provided + + + com.google.inject.extensions + guice-assistedinject + ${guice.version} + + + com.fasterxml.jackson.core + jackson-databind + provided + + + com.fasterxml.jackson.core + jackson-core + provided + + + com.google.inject.extensions + guice-multibindings + provided + + + com.google.guava + guava + provided + + + javax.validation + validation-api + provided + + + com.google.code.findbugs + jsr305 + provided + + + commons-lang + commons-lang + provided + + + + + junit + junit + test + + + org.apache.druid + druid-server + ${project.parent.version} + test + + + org.apache.druid + druid-processing + ${project.parent.version} + test + + + org.easymock + easymock + test + + + nl.jqno.equalsverifier + equalsverifier + test + + + + diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java new file mode 100644 index 000000000000..3c9206c441e5 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.aliyun; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; + +import org.apache.druid.data.input.RetryingInputEntity; +import org.apache.druid.data.input.impl.CloudObjectLocation; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.storage.aliyun.OssStorageDruidModule; +import org.apache.druid.storage.aliyun.OssUtils; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.GetObjectRequest; +import com.aliyun.oss.model.OSSObject; +import com.google.common.base.Predicate; + +public class OssEntity extends RetryingInputEntity +{ + private final OSS ossClient; + private final CloudObjectLocation object; + + OssEntity(OSS ossClient, CloudObjectLocation coords) + { + this.ossClient = ossClient; + this.object = coords; + } + + @Override + public URI getUri() + { + return object.toUri(OssStorageDruidModule.SCHEME); + } + + @Override + protected InputStream readFrom(long offset) throws IOException + { + final GetObjectRequest request = new GetObjectRequest(object.getBucket(), object.getPath()); + request.setRange(offset, request.getRange()[1]); + + try { + final OSSObject ossObject = ossClient.getObject(request); + if (ossObject == null) { + throw new ISE( + "Failed to get an s3 object for bucket[%s], key[%s], and start[%d]", + object.getBucket(), + object.getPath(), + offset + ); + } + return ossObject.getObjectContent(); + } + catch (OSSException e) { + throw new IOException(e); + } + } + + @Override + protected String getPath() + { + return object.getPath(); + } + + @Override + public Predicate getRetryCondition() + { + return OssUtils.RETRYABLE; + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java new file mode 100644 index 000000000000..8419c09989f1 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.aliyun; + +import java.net.URI; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import org.apache.druid.data.input.InputEntity; +import org.apache.druid.data.input.InputFileAttribute; +import org.apache.druid.data.input.InputSplit; +import org.apache.druid.data.input.SplitHintSpec; +import org.apache.druid.data.input.impl.CloudObjectInputSource; +import org.apache.druid.data.input.impl.CloudObjectLocation; +import org.apache.druid.data.input.impl.SplittableInputSource; +import org.apache.druid.storage.aliyun.OssInputDataConfig; +import org.apache.druid.storage.aliyun.OssStorageDruidModule; +import org.apache.druid.storage.aliyun.OssUtils; +import org.apache.druid.utils.Streams; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.model.OSSObjectSummary; +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; + +public class OssInputSource extends CloudObjectInputSource +{ + // We lazily initialize ServerSideEncryptingAmazonS3 to avoid costly s3 operation when we only need S3InputSource + // for stored information (such as for task logs) and not for ingestion. + // (This cost only applies for new ServerSideEncryptingAmazonS3 created with s3InputSourceConfig given). + private final Supplier clientSupplier; + @JsonProperty("properties") + private final OssInputSourceConfig inputSourceConfig; + private final OssInputDataConfig inputDataConfig; + + /** + * Constructor for S3InputSource + * @param client The default ServerSideEncryptingAmazonS3 client built with all default configs + * from Guice. This injected singleton client is use when {@param s3InputSourceConfig} + * is not provided and hence, we can skip building a new client from + * {@param s3ClientBuilder} + * @param clientBulider Use for building a new s3Client to use instead of the default injected + * {@param s3Client}. The configurations of the client can be changed + * before being built + * @param inputDataConfig Stores the configuration for options related to reading input data + * @param uris User provided uris to read input data + * @param prefixes User provided prefixes to read input data + * @param objects User provided cloud objects values to read input data + * @param inputSourceConfig User provided properties for overriding the default S3 configuration + * + */ + @JsonCreator + public OssInputSource( + @JacksonInject OSS client, + @JacksonInject OssInputDataConfig inputDataConfig, + @JsonProperty("uris") @Nullable List uris, + @JsonProperty("prefixes") @Nullable List prefixes, + @JsonProperty("objects") @Nullable List objects, + @JsonProperty("properties") @Nullable OssInputSourceConfig inputSourceConfig + ) + { + super(OssStorageDruidModule.SCHEME, uris, prefixes, objects); + this.inputDataConfig = Preconditions.checkNotNull(inputDataConfig, "S3DataSegmentPusherConfig"); + Preconditions.checkNotNull(client, "client"); + this.inputSourceConfig = inputSourceConfig; + this.clientSupplier = Suppliers.memoize( + () -> { + if (inputSourceConfig != null) { + String accessKeyId = inputSourceConfig.getAccessKeyId().getPassword(); + String secretAccessKey = inputSourceConfig.getSecretAccessKey().getPassword(); + return new OSSClientBuilder().build(inputSourceConfig.getEndpoint(), accessKeyId, secretAccessKey); + } else { + return client; + } + } + ); + } + + +@Nullable + @JsonProperty("properties") + public OssInputSourceConfig getOssInputSourceConfig() + { + return inputSourceConfig; + } + + @Override + protected InputEntity createEntity(CloudObjectLocation location) + { + return new OssEntity(clientSupplier.get(), location); + } + + @Override + protected Stream>> getPrefixesSplitStream(@Nonnull SplitHintSpec splitHintSpec) + { + final Iterator> splitIterator = splitHintSpec.split( + getIterableObjectsFromPrefixes().iterator(), + object -> new InputFileAttribute(object.getSize()) + ); + + return Streams.sequentialStreamFrom(splitIterator) + .map(objects -> objects.stream() + .map(OssUtils::summaryToCloudObjectLocation) + .collect(Collectors.toList())) + .map(InputSplit::new); + } + + @Override + public SplittableInputSource> withSplit(InputSplit> split) + { + return new OssInputSource( + clientSupplier.get(), + inputDataConfig, + null, + null, + split.get(), + getOssInputSourceConfig() + ); + } + + @Override + public int hashCode() + { + return Objects.hash(super.hashCode(), inputSourceConfig); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + OssInputSource that = (OssInputSource) o; + return Objects.equals(inputSourceConfig, that.inputSourceConfig); + } + + @Override + public String toString() + { + return "OssInputSource{" + + "uris=" + getUris() + + ", prefixes=" + getPrefixes() + + ", objects=" + getObjects() + + ", ossInputSourceConfig=" + getOssInputSourceConfig() + + '}'; + } + + private Iterable getIterableObjectsFromPrefixes() + { + return () -> OssUtils.objectSummaryIterator(clientSupplier.get(), getPrefixes(), inputDataConfig.getMaxListingLength()); + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java new file mode 100644 index 000000000000..bde023978a86 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.aliyun; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import org.apache.druid.metadata.PasswordProvider; + +import javax.annotation.Nullable; +import java.util.Objects; + +/** + * Contains properties for s3 input source. + * Properties can be specified by ingestionSpec which will override system default. + */ +public class OssInputSourceConfig +{ + @JsonCreator + public OssInputSourceConfig( + @JsonProperty("endpoint") String endpoint, + @JsonProperty("accessKeyId") @Nullable PasswordProvider accessKeyId, + @JsonProperty("secretAccessKey") @Nullable PasswordProvider secretAccessKey + ) + { + if (accessKeyId != null || secretAccessKey != null) { + this.accessKeyId = Preconditions.checkNotNull(accessKeyId, "accessKeyId cannot be null if secretAccessKey is given"); + this.secretAccessKey = Preconditions.checkNotNull(secretAccessKey, "secretAccessKey cannot be null if accessKeyId is given"); + } + } + + @JsonProperty + private String endpoint; + + @JsonProperty + private PasswordProvider accessKeyId; + + @JsonProperty + private PasswordProvider secretAccessKey; + + public String getEndpoint() { + return endpoint; + } + + public PasswordProvider getAccessKeyId() + { + return accessKeyId; + } + + public PasswordProvider getSecretAccessKey() + { + return secretAccessKey; + } + + @JsonIgnore + public boolean isCredentialsConfigured() + { + return accessKeyId != null && + secretAccessKey != null; + } + + @Override + public String toString() + { + return "OssInputSourceConfig{" + + "endpoint=" + endpoint + + "accessKeyId=" + accessKeyId + + ", secretAccessKey=" + secretAccessKey + + '}'; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + OssInputSourceConfig that = (OssInputSourceConfig) o; + return Objects.equals(accessKeyId, that.accessKeyId) && + Objects.equals(secretAccessKey, that.secretAccessKey) && + Objects.equals(endpoint, that.endpoint); + } + + @Override + public int hashCode() + { + return Objects.hash(accessKeyId, secretAccessKey, endpoint); + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java new file mode 100644 index 000000000000..876e44bcbf42 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.aliyun; + +import java.util.List; + +import org.apache.druid.initialization.DruidModule; +import org.apache.druid.storage.aliyun.OssStorageDruidModule; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; + +/** + * Druid module to wire up native batch support for S3 input + */ +public class OssInputSourceDruidModule implements DruidModule +{ + @Override + public List getJacksonModules() + { + return ImmutableList.of( + new SimpleModule().registerSubtypes(new NamedType(OssInputSource.class, OssStorageDruidModule.SCHEME)) + ); + } + + @Override + public void configure(Binder binder) + { + + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java new file mode 100644 index 000000000000..8963f851a130 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.firehose.aliyun; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; +import org.apache.druid.initialization.DruidModule; + +import java.util.List; + +/** + */ +public class OssFirehoseDruidModule implements DruidModule +{ + @Override + public List getJacksonModules() + { + return ImmutableList.of( + new SimpleModule().registerSubtypes(new NamedType(StaticOssFirehoseFactory.class, "static-s3")) + ); + } + + @Override + public void configure(Binder binder) + { + + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java new file mode 100644 index 000000000000..e43066729f4f --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.firehose.aliyun; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import org.apache.druid.data.input.FiniteFirehoseFactory; +import org.apache.druid.data.input.InputSplit; +import org.apache.druid.data.input.impl.StringInputRowParser; +import org.apache.druid.data.input.impl.prefetch.PrefetchableTextFilesFirehoseFactory; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.storage.aliyun.OssUtils; +import org.apache.druid.utils.CompressionUtils; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.GetObjectRequest; +import com.aliyun.oss.model.OSSObject; +import com.aliyun.oss.model.OSSObjectSummary; +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; + +/** + * Builds firehoses that read from a predefined list of S3 objects and then dry up. + */ +public class StaticOssFirehoseFactory extends PrefetchableTextFilesFirehoseFactory +{ + private static final Logger log = new Logger(StaticOssFirehoseFactory.class); + private static final int MAX_LISTING_LENGTH = 1024; + + private final OSS client; + private final List uris; + private final List prefixes; + + @JsonCreator + public StaticOssFirehoseFactory( + @JacksonInject OSS client, + @JsonProperty("uris") List uris, + @JsonProperty("prefixes") List prefixes, + @JsonProperty("maxCacheCapacityBytes") Long maxCacheCapacityBytes, + @JsonProperty("maxFetchCapacityBytes") Long maxFetchCapacityBytes, + @JsonProperty("prefetchTriggerBytes") Long prefetchTriggerBytes, + @JsonProperty("fetchTimeout") Long fetchTimeout, + @JsonProperty("maxFetchRetry") Integer maxFetchRetry + ) + { + super(maxCacheCapacityBytes, maxFetchCapacityBytes, prefetchTriggerBytes, fetchTimeout, maxFetchRetry); + this.client = Preconditions.checkNotNull(client, "s3Client"); + this.uris = uris == null ? new ArrayList<>() : uris; + this.prefixes = prefixes == null ? new ArrayList<>() : prefixes; + + if (!this.uris.isEmpty() && !this.prefixes.isEmpty()) { + throw new IAE("uris and prefixes cannot be used together"); + } + + if (this.uris.isEmpty() && this.prefixes.isEmpty()) { + throw new IAE("uris or prefixes must be specified"); + } + + for (final URI inputURI : this.uris) { + Preconditions.checkArgument("s3".equals(inputURI.getScheme()), "input uri scheme == s3 (%s)", inputURI); + } + + for (final URI inputURI : this.prefixes) { + Preconditions.checkArgument("s3".equals(inputURI.getScheme()), "input uri scheme == s3 (%s)", inputURI); + } + } + + @JsonProperty + public List getUris() + { + return uris; + } + + @JsonProperty("prefixes") + public List getPrefixes() + { + return prefixes; + } + + @Override + protected Collection initObjects() + { + if (!uris.isEmpty()) { + return uris; + } else { + final List objects = new ArrayList<>(); + for (final URI prefix : prefixes) { + final Iterator objectSummaryIterator = OssUtils.objectSummaryIterator( + client, + Collections.singletonList(prefix), + MAX_LISTING_LENGTH + ); + + objectSummaryIterator.forEachRemaining(objects::add); + } + return objects.stream().map(OssUtils::summaryToUri).collect(Collectors.toList()); + } + } + + @Override + protected InputStream openObjectStream(URI object) throws IOException + { + try { + // Get data of the given object and open an input stream + final String bucket = object.getAuthority(); + final String key = OssUtils.extractKey(object); + + final OSSObject OSSObject = client.getObject(bucket, key); + if (OSSObject == null) { + throw new ISE("Failed to get an s3 object for bucket[%s] and key[%s]", bucket, key); + } + return OSSObject.getObjectContent(); + } + catch (OSSException e) { + throw new IOException(e); + } + } + + @Override + protected InputStream openObjectStream(URI object, long start) throws IOException + { + final String bucket = object.getAuthority(); + final String key = OssUtils.extractKey(object); + + final GetObjectRequest request = new GetObjectRequest(bucket, key); + try { + final OSSObject OSSObject = client.getObject(request); + if (OSSObject == null) { + throw new ISE( + "Failed to get an s3 object for bucket[%s], key[%s], and start[%d]", + bucket, + key, + start + ); + } + InputStream is = OSSObject.getObjectContent(); + is.skip(start); + return is; + } + catch (OSSException e) { + throw new IOException(e); + } + } + + @Override + protected InputStream wrapObjectStream(URI object, InputStream stream) throws IOException + { + return CompressionUtils.decompress(stream, OssUtils.extractKey(object)); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + StaticOssFirehoseFactory that = (StaticOssFirehoseFactory) o; + + return Objects.equals(uris, that.uris) && + Objects.equals(prefixes, that.prefixes) && + getMaxCacheCapacityBytes() == that.getMaxCacheCapacityBytes() && + getMaxFetchCapacityBytes() == that.getMaxFetchCapacityBytes() && + getPrefetchTriggerBytes() == that.getPrefetchTriggerBytes() && + getFetchTimeout() == that.getFetchTimeout() && + getMaxFetchRetry() == that.getMaxFetchRetry(); + } + + @Override + public int hashCode() + { + return Objects.hash( + uris, + prefixes, + getMaxCacheCapacityBytes(), + getMaxFetchCapacityBytes(), + getPrefetchTriggerBytes(), + getFetchTimeout(), + getMaxFetchRetry() + ); + } + + @Override + protected Predicate getRetryCondition() + { + return OssUtils.RETRYABLE; + } + + @Override + public FiniteFirehoseFactory withSplit(InputSplit split) + { + return new StaticOssFirehoseFactory( + client, + Collections.singletonList(split.get()), + null, + getMaxCacheCapacityBytes(), + getMaxFetchCapacityBytes(), + getPrefetchTriggerBytes(), + getFetchTimeout(), + getMaxFetchRetry() + ); + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java new file mode 100644 index 000000000000..4a28011db541 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.net.URI; +import java.util.Iterator; +import java.util.NoSuchElementException; + +import org.apache.druid.java.util.common.RE; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.ObjectListing; + +/** + * Iterator class used by {@link OssUtils#objectSummaryIterator}. + * + * As required by the specification of that method, this iterator is computed incrementally in batches of + * {@code maxListLength}. The first call is made at the same time the iterator is constructed. + */ +public class ObjectSummaryIterator implements Iterator +{ + private final OSS client; + private final Iterator prefixesIterator; + private final int maxListingLength; + + private ListObjectsRequest request; + private ObjectListing result; + private Iterator objectSummaryIterator; + private OSSObjectSummary currentObjectSummary; + + ObjectSummaryIterator( + final OSS client, + final Iterable prefixes, + final int maxListingLength + ) + { + this.client = client; + this.prefixesIterator = prefixes.iterator(); + this.maxListingLength = maxListingLength; + + prepareNextRequest(); + fetchNextBatch(); + advanceObjectSummary(); + } + + @Override + public boolean hasNext() + { + return currentObjectSummary != null; + } + + @Override + public OSSObjectSummary next() + { + if (currentObjectSummary == null) { + throw new NoSuchElementException(); + } + + final OSSObjectSummary retVal = currentObjectSummary; + advanceObjectSummary(); + return retVal; + } + + private void prepareNextRequest() + { + final URI currentUri = prefixesIterator.next(); + final String currentBucket = currentUri.getAuthority(); + final String currentPrefix = OssUtils.extractKey(currentUri); + + request = new ListObjectsRequest(currentBucket,currentPrefix, null, null, maxListingLength); + } + + private void fetchNextBatch() + { + try { + result = OssUtils.retry(() -> client.listObjects(request)); + request.setMarker(result.getNextMarker()); + objectSummaryIterator = result.getObjectSummaries().iterator(); + } + catch (OSSException e) { + throw new RE( + e, + "Failed to get object summaries from S3 bucket[%s], prefix[%s]; S3 error: %s", + request.getBucketName(), + request.getPrefix(), + e.getMessage() + ); + } + catch (Exception e) { + throw new RE( + e, + "Failed to get object summaries from S3 bucket[%s], prefix[%s]", + request.getBucketName(), + request.getPrefix() + ); + } + } + + /** + * Advance objectSummaryIterator to the next non-placeholder, updating "currentObjectSummary". + */ + private void advanceObjectSummary() + { + while (objectSummaryIterator.hasNext() || result.isTruncated() || prefixesIterator.hasNext()) { + while (objectSummaryIterator.hasNext()) { + currentObjectSummary = objectSummaryIterator.next(); + // skips directories and empty objects + if (!isDirectoryPlaceholder(currentObjectSummary) && currentObjectSummary.getSize() > 0) { + return; + } + } + + // Exhausted "objectSummaryIterator" without finding a non-placeholder. + if (result.isTruncated()) { + fetchNextBatch(); + } else if (prefixesIterator.hasNext()) { + prepareNextRequest(); + fetchNextBatch(); + } + } + + // Truly nothing left to read. + currentObjectSummary = null; + } + + /** + * Checks if a given object is a directory placeholder and should be ignored. + * + * Adapted from org.jets3t.service.model.StorageObject.isDirectoryPlaceholder(). Does not include the check for + * legacy JetS3t directory placeholder objects, since it is based on content-type, which isn't available in an + * S3ObjectSummary. + */ + private static boolean isDirectoryPlaceholder(final OSSObjectSummary objectSummary) + { + // Recognize "standard" directory place-holder indications used by Amazon's AWS Console and Panic's Transmit. + if (objectSummary.getKey().endsWith("/") && objectSummary.getSize() == 0) { + return true; + } + + // Recognize s3sync.rb directory placeholders by MD5/ETag value. + if ("d66759af42f282e1ba19144df2d405d0".equals(objectSummary.getETag())) { + return true; + } + + // Recognize place-holder objects created by the Google Storage console or S3 Organizer Firefox extension. + if (objectSummary.getKey().endsWith("_$folder$") && objectSummary.getSize() == 0) { + return true; + } + + return false; + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssClientHelper.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssClientHelper.java new file mode 100644 index 000000000000..ba11fc6ea31d --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssClientHelper.java @@ -0,0 +1,130 @@ +///* +// * Licensed to the Apache Software Foundation (ASF) under one +// * or more contributor license agreements. See the NOTICE file +// * distributed with this work for additional information +// * regarding copyright ownership. The ASF licenses this file +// * to you under the Apache License, Version 2.0 (the +// * "License"); you may not use this file except in compliance +// * with the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, +// * software distributed under the License is distributed on an +// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// * KIND, either express or implied. See the License for the +// * specific language governing permissions and limitations +// * under the License. +// */ +// +//package org.apache.druid.storage.aliyun; +// +//import java.io.ByteArrayInputStream; +//import java.io.File; +//import java.io.InputStream; +// +//import org.apache.druid.java.util.common.StringUtils; +// +//import com.aliyun.oss.OSS; +//import com.aliyun.oss.OSSClientBuilder; +//import com.aliyun.oss.model.AccessControlList; +//import com.aliyun.oss.model.CopyObjectRequest; +//import com.aliyun.oss.model.CopyObjectResult; +//import com.aliyun.oss.model.DeleteObjectsRequest; +//import com.aliyun.oss.model.GenericRequest; +//import com.aliyun.oss.model.GetObjectRequest; +//import com.aliyun.oss.model.ListObjectsRequest; +//import com.aliyun.oss.model.OSSObject; +//import com.aliyun.oss.model.ObjectListing; +//import com.aliyun.oss.model.ObjectMetadata; +//import com.aliyun.oss.model.PutObjectRequest; +//import com.aliyun.oss.model.PutObjectResult; +// +///** +// * {@link AmazonS3} wrapper with {@link ServerSideEncryption}. Every {@link AmazonS3#putObject}, +// * {@link AmazonS3#copyObject}, {@link AmazonS3#getObject}, and {@link AmazonS3#getObjectMetadata} methods should be +// * wrapped using ServerSideEncryption. +// * +// * Additional methods can be added to this class if needed, but subclassing AmazonS3Client is discouraged to reduce +// * human mistakes like some methods are not encoded properly. +// */ +//public class OssClientHelper +//{ +// private final OSS ossClient; +// +// public OssClientHelper(OSS ossClient) +// { +// this.ossClient = ossClient; +// } +// +// public OssClientHelper(String endpoint, String accessKeyId, String secretAccessKey) { +// this.ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, secretAccessKey); +// } +// +//public boolean doesObjectExist(String bucket, String objectName) +// { +// return ossClient.doesObjectExist(bucket, objectName); +// } +// +// public ObjectListing listObjects(ListObjectsRequest request) +// { +// return ossClient.listObjects(request); +// } +// +// public AccessControlList getBucketAcl(String bucket) +// { +// return ossClient.getBucketAcl(bucket); +// } +// +// public ObjectMetadata getObjectMetadata(String bucket, String key) +// { +// GenericRequest request = new GenericRequest(bucket, key); +// return ossClient.getobjectm +// } +// +// public OSSObject getObject(String bucket, String key) +// { +// return getObject(new GetObjectRequest(bucket, key)); +// } +// +// public OSSObject getObject(GetObjectRequest request) +// { +// return ossClient.getObject(request); +// } +// +// public PutObjectResult putObject(String bucket, String key, String content) +// { +// final InputStream in = new ByteArrayInputStream(StringUtils.toUtf8(content)); +// return putObject(new PutObjectRequest(bucket, key, in, new ObjectMetadata())); +// } +// +// public PutObjectResult putObject(String bucket, String key, File file) +// { +// return putObject(new PutObjectRequest(bucket, key, file)); +// } +// +// public PutObjectResult putObject(String bucket, String key, InputStream in, ObjectMetadata objectMetadata) +// { +// return putObject(new PutObjectRequest(bucket, key, in, objectMetadata)); +// } +// +// public PutObjectResult putObject(PutObjectRequest request) +// { +// return ossClient.putObject(request); +// } +// +// public CopyObjectResult copyObject(CopyObjectRequest request) +// { +// return ossClient.copyObject(request); +// } +// +// public void deleteObject(String bucket, String key) +// { +// ossClient.deleteObject(bucket, key); +// } +// +// public void deleteObjects(DeleteObjectsRequest request) +// { +// ossClient.deleteObjects(request); +// } +//} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java new file mode 100644 index 000000000000..f6c8ff66a54c --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import org.apache.druid.guice.annotations.Json; +import org.apache.druid.segment.loading.DataSegmentArchiver; +import org.apache.druid.segment.loading.LoadSpec; +import org.apache.druid.segment.loading.SegmentLoadingException; +import org.apache.druid.timeline.DataSegment; + +import com.aliyun.oss.OSS; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Objects; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; + + +public class OssDataSegmentArchiver extends OssDataSegmentMover implements DataSegmentArchiver +{ + private final OssDataSegmentArchiverConfig archiveConfig; + private final OssDataSegmentPusherConfig restoreConfig; + private final ObjectMapper mapper; + + @Inject + public OssDataSegmentArchiver( + @Json ObjectMapper mapper, + OSS client, + OssDataSegmentArchiverConfig archiveConfig, + OssDataSegmentPusherConfig restoreConfig + ) + { + super(client, restoreConfig); + this.mapper = mapper; + this.archiveConfig = archiveConfig; + this.restoreConfig = restoreConfig; + } + + @Override + public DataSegment archive(DataSegment segment) throws SegmentLoadingException + { + String targetS3Bucket = archiveConfig.getArchiveBucket(); + String targetS3BaseKey = archiveConfig.getArchiveBaseKey(); + + final DataSegment archived = move( + segment, + ImmutableMap.of( + "bucket", targetS3Bucket, + "baseKey", targetS3BaseKey + ) + ); + if (sameLoadSpec(segment, archived)) { + return null; + } + return archived; + } + + @Override + public DataSegment restore(DataSegment segment) throws SegmentLoadingException + { + String targetS3Bucket = restoreConfig.getBucket(); + String targetS3BaseKey = restoreConfig.getBaseKey(); + + final DataSegment restored = move( + segment, + ImmutableMap.of( + "bucket", targetS3Bucket, + "baseKey", targetS3BaseKey + ) + ); + + if (sameLoadSpec(segment, restored)) { + return null; + } + return restored; + } + + boolean sameLoadSpec(DataSegment s1, DataSegment s2) + { + final OssLoadSpec s1LoadSpec = (OssLoadSpec) mapper.convertValue(s1.getLoadSpec(), LoadSpec.class); + final OssLoadSpec s2LoadSpec = (OssLoadSpec) mapper.convertValue(s2.getLoadSpec(), LoadSpec.class); + return Objects.equal(s1LoadSpec.getBucket(), s2LoadSpec.getBucket()) && Objects.equal( + s1LoadSpec.getKey(), + s2LoadSpec.getKey() + ); + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java new file mode 100644 index 000000000000..dd659044e9fe --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class OssDataSegmentArchiverConfig +{ + @JsonProperty + private String archiveBucket = ""; + + @JsonProperty + private String archiveBaseKey = ""; + + public String getArchiveBucket() + { + return archiveBucket; + } + + public String getArchiveBaseKey() + { + return archiveBaseKey; + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java new file mode 100644 index 000000000000..af465cd7e9db --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.io.IOException; +import java.util.Map; + +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.MapUtils; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.segment.loading.DataSegmentKiller; +import org.apache.druid.segment.loading.SegmentLoadingException; +import org.apache.druid.timeline.DataSegment; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.google.common.base.Predicates; +import com.google.inject.Inject; + +/** + * + */ +public class OssDataSegmentKiller implements DataSegmentKiller +{ + private static final Logger log = new Logger(OssDataSegmentKiller.class); + + private final OSS client; + private final OssDataSegmentPusherConfig segmentPusherConfig; + private final OssInputDataConfig inputDataConfig; + + @Inject + public OssDataSegmentKiller( + OSS client, + OssDataSegmentPusherConfig segmentPusherConfig, + OssInputDataConfig inputDataConfig + ) + { + this.client = client; + this.segmentPusherConfig = segmentPusherConfig; + this.inputDataConfig = inputDataConfig; + } + + @Override + public void kill(DataSegment segment) throws SegmentLoadingException + { + try { + Map loadSpec = segment.getLoadSpec(); + String s3Bucket = MapUtils.getString(loadSpec, "bucket"); + String s3Path = MapUtils.getString(loadSpec, "key"); + String s3DescriptorPath = DataSegmentKiller.descriptorPath(s3Path); + + if (client.doesObjectExist(s3Bucket, s3Path)) { + log.info("Removing index file[s3://%s/%s] from s3!", s3Bucket, s3Path); + client.deleteObject(s3Bucket, s3Path); + } + // descriptor.json is a file to store segment metadata in deep storage. This file is deprecated and not stored + // anymore, but we still delete them if exists. + if (client.doesObjectExist(s3Bucket, s3DescriptorPath)) { + log.info("Removing descriptor file[s3://%s/%s] from s3!", s3Bucket, s3DescriptorPath); + client.deleteObject(s3Bucket, s3DescriptorPath); + } + } + catch (OSSException e) { + throw new SegmentLoadingException(e, "Couldn't kill segment[%s]: [%s]", segment.getId(), e); + } + } + + @Override + public void killAll() throws IOException + { + if (segmentPusherConfig.getBucket() == null || segmentPusherConfig.getBaseKey() == null) { + throw new ISE( + "Cannot delete all segment from S3 Deep Storage since druid.storage.bucket and druid.storage.baseKey are not both set."); + } + log.info("Deleting all segment files from s3 location [bucket: '%s' prefix: '%s']", + segmentPusherConfig.getBucket(), segmentPusherConfig.getBaseKey() + ); + try { + OssUtils.deleteObjectsInPath( + client, + inputDataConfig, + segmentPusherConfig.getBucket(), + segmentPusherConfig.getBaseKey(), + Predicates.alwaysTrue() + ); + } + catch (Exception e) { + log.error("Error occurred while deleting segment files from s3. Error: %s", e.getMessage()); + throw new IOException(e); + } + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java new file mode 100644 index 000000000000..2cd495c4f596 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.io.IOException; +import java.util.Map; + +import org.apache.druid.java.util.common.IOE; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.MapUtils; +import org.apache.druid.java.util.common.RetryUtils; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.segment.loading.DataSegmentMover; +import org.apache.druid.segment.loading.DataSegmentPusher; +import org.apache.druid.segment.loading.SegmentLoadingException; +import org.apache.druid.timeline.DataSegment; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.CopyObjectRequest; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.ObjectListing; +import com.aliyun.oss.model.StorageClass; +import com.google.common.base.Predicate; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.inject.Inject; + +public class OssDataSegmentMover implements DataSegmentMover +{ + private static final Logger log = new Logger(OssDataSegmentMover.class); + + private final OSS client; + private final OssDataSegmentPusherConfig config; + + @Inject + public OssDataSegmentMover( + OSS client, + OssDataSegmentPusherConfig config + ) + { + this.client = client; + this.config = config; + } + + @Override + public DataSegment move(DataSegment segment, Map targetLoadSpec) throws SegmentLoadingException + { + try { + Map loadSpec = segment.getLoadSpec(); + String bucket = MapUtils.getString(loadSpec, "bucket"); + String key = MapUtils.getString(loadSpec, "key"); + + final String targetBucket = MapUtils.getString(targetLoadSpec, "bucket"); + final String targetKey = MapUtils.getString(targetLoadSpec, "baseKey"); + + final String targetS3Path = OssUtils.constructSegmentPath( + targetKey, + DataSegmentPusher.getDefaultStorageDir(segment, false) + ); + + if (targetBucket.isEmpty()) { + throw new SegmentLoadingException("Target OSS bucket is not specified"); + } + if (targetS3Path.isEmpty()) { + throw new SegmentLoadingException("Target OSS baseKey is not specified"); + } + + safeMove(bucket, key, targetBucket, targetS3Path); + + return segment.withLoadSpec( + ImmutableMap.builder() + .putAll( + Maps.filterKeys( + loadSpec, + new Predicate() + { + @Override + public boolean apply(String input) + { + return !("bucket".equals(input) || "key".equals(input)); + } + } + ) + ) + .put("bucket", targetBucket) + .put("key", targetS3Path) + .build() + ); + } + catch (OSSException e) { + throw new SegmentLoadingException(e, "Unable to move segment[%s]: [%s]", segment.getId(), e); + } + } + + private void safeMove( + final String s3Bucket, + final String s3Path, + final String targetS3Bucket, + final String targetS3Path + ) throws SegmentLoadingException + { + try { + OssUtils.retry( + () -> { + final String copyMsg = StringUtils.format( + "[s3://%s/%s] to [s3://%s/%s]", + s3Bucket, + s3Path, + targetS3Bucket, + targetS3Path + ); + try { + selfCheckingMove(s3Bucket, targetS3Bucket, s3Path, targetS3Path, copyMsg); + return null; + } + catch (OSSException | IOException | SegmentLoadingException e) { + log.info(e, "Error while trying to move " + copyMsg); + throw e; + } + } + ); + } + catch (Exception e) { + Throwables.propagateIfInstanceOf(e, OSSException.class); + Throwables.propagateIfInstanceOf(e, SegmentLoadingException.class); + throw new RuntimeException(e); + } + } + + /** + * Copies an object and after that checks that the object is present at the target location, via a separate API call. + * If it is not, an exception is thrown, and the object is not deleted at the old location. This "paranoic" check + * is added after it was observed that S3 may report a successful move, and the object is not found at the target + * location. + */ + private void selfCheckingMove( + String srcBucket, + String dstBucket, + String srcPath, + String dstPath, + String copyMsg + ) throws IOException, SegmentLoadingException + { + if (srcBucket.equals(dstBucket) && srcPath.equals(dstPath)) { + log.info("No need to move file[s3://%s/%s] onto itself", srcBucket, srcPath); + return; + } + if (client.doesObjectExist(srcBucket, srcPath)) { + final ObjectListing listResult = client.listObjects( + new ListObjectsRequest(srcBucket,srcPath, null, null, 1) + ); + // Using getObjectSummaries().size() instead of getKeyCount as, in some cases + // it is observed that even though the getObjectSummaries returns some data + // keyCount is still zero. + if (listResult.getObjectSummaries().size() == 0) { + // should never happen + throw new ISE("Unable to list object [s3://%s/%s]", srcBucket, srcPath); + } + final OSSObjectSummary objectSummary = listResult.getObjectSummaries().get(0); + if (objectSummary.getStorageClass() != null && + objectSummary.getStorageClass().equals(StorageClass.IA.name())) { + throw new OSSException( + StringUtils.format( + "Cannot move file[oss://%s/%s] of storage class glacier, skipping.", + srcBucket, + srcPath + ) + ); + } else { + log.info("Moving file %s", copyMsg); + final CopyObjectRequest copyRequest = new CopyObjectRequest(srcBucket, srcPath, dstBucket, dstPath); + if (!config.getDisableAcl()) { + //copyRequest.set(OssUtils.grantFullControlToBucketOwner(client, dstBucket)); + } + client.copyObject(copyRequest); + if (!client.doesObjectExist(dstBucket, dstPath)) { + throw new IOE( + "After copy was reported as successful the file doesn't exist in the target location [%s]", + copyMsg + ); + } + deleteWithRetriesSilent(srcBucket, srcPath); + log.debug("Finished moving file %s", copyMsg); + } + } else { + // ensure object exists in target location + if (client.doesObjectExist(dstBucket, dstPath)) { + log.info( + "Not moving file [s3://%s/%s], already present in target location [s3://%s/%s]", + srcBucket, + srcPath, + dstBucket, + dstPath + ); + } else { + throw new SegmentLoadingException( + "Unable to move file %s, not present in either source or target location", + copyMsg + ); + } + } + } + + private void deleteWithRetriesSilent(final String s3Bucket, final String s3Path) + { + try { + deleteWithRetries(s3Bucket, s3Path); + } + catch (Exception e) { + log.error(e, "Failed to delete file [s3://%s/%s], giving up", s3Bucket, s3Path); + } + } + + private void deleteWithRetries(final String s3Bucket, final String s3Path) throws Exception + { + RetryUtils.retry( + () -> { + try { + client.deleteObject(s3Bucket, s3Path); + return null; + } + catch (Exception e) { + log.info(e, "Error while trying to delete [s3://%s/%s]", s3Bucket, s3Path); + throw e; + } + }, + OssUtils.RETRYABLE, + 3 + ); + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java new file mode 100644 index 000000000000..afa0751c466a --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java @@ -0,0 +1,311 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.io.File; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Reader; +import java.io.Writer; +import java.net.URI; + +import javax.tools.FileObject; + +import org.apache.druid.data.input.impl.CloudObjectLocation; +import org.apache.druid.java.util.common.FileUtils; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.IOE; +import org.apache.druid.java.util.common.RE; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.UOE; +import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.segment.loading.SegmentLoadingException; +import org.apache.druid.segment.loading.URIDataPuller; +import org.apache.druid.utils.CompressionUtils; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.OSSObject; +import com.aliyun.oss.model.OSSObjectSummary; +import com.google.common.base.Predicate; +import com.google.common.base.Strings; +import com.google.common.io.ByteSource; +import com.google.common.io.Files; +import com.google.inject.Inject; + +/** + * A data segment puller that also hanldes URI data pulls. + */ +public class OssDataSegmentPuller implements URIDataPuller +{ + private static final Logger log = new Logger(OssDataSegmentPuller.class); + + static final String BUCKET = "bucket"; + protected static final String KEY = "key"; + + protected final OSS client; + + @Inject + public OssDataSegmentPuller(OSS s3Client) + { + this.client = s3Client; + } + + FileUtils.FileCopyResult getSegmentFiles(final CloudObjectLocation ossCoords, final File outDir) throws SegmentLoadingException + { + + log.info("Pulling index at path[%s] to outDir[%s]", ossCoords, outDir); + + if (!isObjectInBucket(ossCoords)) { + throw new SegmentLoadingException("IndexFile[%s] does not exist.", ossCoords); + } + + try { + org.apache.commons.io.FileUtils.forceMkdir(outDir); + + final URI uri = ossCoords.toUri(OssStorageDruidModule.SCHEME); + final ByteSource byteSource = new ByteSource() + { + @Override + public InputStream openStream() throws IOException + { + try { + return buildFileObject(uri).openInputStream(); + } + catch (OSSException e) { + if (e.getCause() != null) { + if (OssUtils.RETRYABLE.apply(e)) { + throw new IOException("Recoverable exception", e); + } + } + throw new RuntimeException(e); + } + } + }; + if (CompressionUtils.isZip(ossCoords.getPath())) { + final FileUtils.FileCopyResult result = CompressionUtils.unzip( + byteSource, + outDir, + OssUtils.RETRYABLE, + false + ); + log.info("Loaded %d bytes from [%s] to [%s]", result.size(), ossCoords.toString(), outDir.getAbsolutePath()); + return result; + } + if (CompressionUtils.isGz(ossCoords.getPath())) { + final String fname = Files.getNameWithoutExtension(uri.getPath()); + final File outFile = new File(outDir, fname); + + final FileUtils.FileCopyResult result = CompressionUtils.gunzip(byteSource, outFile, OssUtils.RETRYABLE); + log.info("Loaded %d bytes from [%s] to [%s]", result.size(), ossCoords.toString(), outFile.getAbsolutePath()); + return result; + } + throw new IAE("Do not know how to load file type at [%s]", uri.toString()); + } + catch (Exception e) { + try { + FileUtils.deleteDirectory(outDir); + } + catch (IOException ioe) { + log.warn( + ioe, + "Failed to remove output directory [%s] for segment pulled from [%s]", + outDir.getAbsolutePath(), + ossCoords.toString() + ); + } + throw new SegmentLoadingException(e, e.getMessage()); + } + } + + @Override + public InputStream getInputStream(URI uri) throws IOException + { + try { + return buildFileObject(uri).openInputStream(); + } + catch (OSSException e) { + throw new IOE(e, "Could not load URI [%s]", uri); + } + } + + private FileObject buildFileObject(final URI uri) throws OSSException + { + final CloudObjectLocation coords = new CloudObjectLocation(OssUtils.checkURI(uri)); + final OSSObjectSummary objectSummary = + OssUtils.getSingleObjectSummary(client, coords.getBucket(), coords.getPath()); + final String path = uri.getPath(); + + return new FileObject() + { + OSSObject ossObject = null; + + @Override + public URI toUri() + { + return uri; + } + + @Override + public String getName() + { + final String ext = Files.getFileExtension(path); + return Files.getNameWithoutExtension(path) + (Strings.isNullOrEmpty(ext) ? "" : ("." + ext)); + } + + /** + * Returns an input stream for a s3 object. The returned input stream is not thread-safe. + */ + @Override + public InputStream openInputStream() throws IOException + { + try { + if (ossObject == null) { + // lazily promote to full GET + ossObject = client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); + } + + final InputStream in = ossObject.getObjectContent(); + final Closer closer = Closer.create(); + closer.register(in); + closer.register(ossObject); + + return new FilterInputStream(in) + { + @Override + public void close() throws IOException + { + closer.close(); + } + }; + } + catch (OSSException e) { + throw new IOE(e, "Could not load S3 URI [%s]", uri); + } + } + + @Override + public OutputStream openOutputStream() + { + throw new UOE("Cannot stream S3 output"); + } + + @Override + public Reader openReader(boolean ignoreEncodingErrors) + { + throw new UOE("Cannot open reader"); + } + + @Override + public CharSequence getCharContent(boolean ignoreEncodingErrors) + { + throw new UOE("Cannot open character sequence"); + } + + @Override + public Writer openWriter() + { + throw new UOE("Cannot open writer"); + } + + @Override + public long getLastModified() + { + return objectSummary.getLastModified().getTime(); + } + + @Override + public boolean delete() + { + throw new UOE("Cannot delete S3 items anonymously. jetS3t doesn't support authenticated deletes easily."); + } + }; + } + + @Override + public Predicate shouldRetryPredicate() + { + // Yay! smart retries! + return new Predicate() + { + @Override + public boolean apply(Throwable e) + { + if (e == null) { + return false; + } + if (e instanceof OSSException) { + return OssUtils.isServiceExceptionRecoverable((OSSException) e); + } + if (OssUtils.RETRYABLE.apply(e)) { + return true; + } + // Look all the way down the cause chain, just in case something wraps it deep. + return apply(e.getCause()); + } + }; + } + + /** + * Returns the "version" (aka last modified timestamp) of the URI + * + * @param uri The URI to check the last timestamp + * + * @return The time in ms of the last modification of the URI in String format + * + * @throws IOException + */ + @Override + public String getVersion(URI uri) throws IOException + { + try { + final CloudObjectLocation coords = new CloudObjectLocation(OssUtils.checkURI(uri)); + final OSSObjectSummary objectSummary = + OssUtils.getSingleObjectSummary(client, coords.getBucket(), coords.getPath()); + return StringUtils.format("%d", objectSummary.getLastModified().getTime()); + } + catch (OSSException e) { + if (OssUtils.isServiceExceptionRecoverable(e)) { + // The recoverable logic is always true for IOException, so we want to only pass IOException if it is recoverable + throw new IOE(e, "Could not fetch last modified timestamp from URI [%s]", uri); + } else { + throw new RE(e, "Error fetching last modified timestamp from URI [%s]", uri); + } + } + } + + private boolean isObjectInBucket(final CloudObjectLocation coords) throws SegmentLoadingException + { + try { + return OssUtils.retry( + () -> OssUtils.isObjectInBucketIgnoringPermission(client, coords.getBucket(), coords.getPath()) + ); + } + catch (OSSException | IOException e) { + throw new SegmentLoadingException(e, "S3 fail! Key[%s]", coords); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java new file mode 100644 index 000000000000..065c47b73deb --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.List; +import java.util.Map; + +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.segment.SegmentUtils; +import org.apache.druid.segment.loading.DataSegmentPusher; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.utils.CompressionUtils; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; + +public class OssDataSegmentPusher implements DataSegmentPusher +{ + private static final EmittingLogger log = new EmittingLogger(OssDataSegmentPusher.class); + + private final OSS client; + private final OssDataSegmentPusherConfig config; + + @Inject + public OssDataSegmentPusher( + OSS client, + OssDataSegmentPusherConfig config + ) + { + this.client = client; + this.config = config; + } + + @Override + public String getPathForHadoop() + { + return StringUtils.format("%s/%s", config.getBucket(), config.getBaseKey()); + } + + @Deprecated + @Override + public String getPathForHadoop(String dataSource) + { + return getPathForHadoop(); + } + + @Override + public List getAllowedPropertyPrefixesForHadoop() + { + return ImmutableList.of("druid.oss"); + } + + @Override + public DataSegment push(final File indexFilesDir, final DataSegment inSegment, final boolean useUniquePath) + throws IOException + { + final String s3Path = OssUtils.constructSegmentPath(config.getBaseKey(), getStorageDir(inSegment, useUniquePath)); + + log.debug("Copying segment[%s] to S3 at location[%s]", inSegment.getId(), s3Path); + + final File zipOutFile = File.createTempFile("druid", "index.zip"); + final long indexSize = CompressionUtils.zip(indexFilesDir, zipOutFile); + + final DataSegment outSegment = inSegment.withSize(indexSize) + .withLoadSpec(makeLoadSpec(config.getBucket(), s3Path)) + .withBinaryVersion(SegmentUtils.getVersionFromDir(indexFilesDir)); + + try { + return OssUtils.retry( + () -> { + OssUtils.uploadFileIfPossible(client, config.getDisableAcl(), config.getBucket(), s3Path, zipOutFile); + + return outSegment; + } + ); + } + catch (OSSException e) { + throw new IOException(e); + } + catch (Exception e) { + throw new RuntimeException(e); + } + finally { + log.debug("Deleting temporary cached index.zip"); + zipOutFile.delete(); + } + } + + @Override + public Map makeLoadSpec(URI finalIndexZipFilePath) + { + // remove the leading "/" + return makeLoadSpec(finalIndexZipFilePath.getHost(), finalIndexZipFilePath.getPath().substring(1)); + } + + /** + * Any change in loadSpec need to be reflected {@link org.apache.druid.indexer.JobHelper#getURIFromSegment()} + */ + private Map makeLoadSpec(String bucket, String key) + { + return ImmutableMap.of( + "type", + "oss_zip", + "bucket", + bucket, + "key", + key + ); + } + +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfig.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfig.java new file mode 100644 index 000000000000..7d051d72136f --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfig.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.Min; + +/** + */ +public class OssDataSegmentPusherConfig +{ + @JsonProperty + private String bucket = ""; + + @JsonProperty + private String baseKey = ""; + + @JsonProperty + private boolean disableAcl = true; + + @JsonProperty + @Min(1) + private int maxListingLength = 1024; + // use s3n by default for backward compatibility + public void setBucket(String bucket) + { + this.bucket = bucket; + } + + public void setBaseKey(String baseKey) + { + this.baseKey = baseKey; + } + + public void setDisableAcl(boolean disableAcl) + { + this.disableAcl = disableAcl; + } + + public void setMaxListingLength(int maxListingLength) + { + this.maxListingLength = maxListingLength; + } + + public String getBucket() + { + return bucket; + } + + public String getBaseKey() + { + return baseKey; + } + + public boolean getDisableAcl() + { + return disableAcl; + } + + public int getMaxListingLength() + { + return maxListingLength; + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java new file mode 100644 index 000000000000..5b917084db3a --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.Min; + +/** + * Stores the configuration for options related to reading + * input data from Amazon S3 into Druid + */ +public class OssInputDataConfig +{ + /** + * The maximum number of input files matching a given prefix to retrieve + * from Amazon S3 at a time. + */ + @JsonProperty + @Min(1) + private int maxListingLength = 1024; + + public void setMaxListingLength(int maxListingLength) + { + this.maxListingLength = maxListingLength; + } + + public int getMaxListingLength() + { + return maxListingLength; + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java new file mode 100644 index 000000000000..e35a48e34499 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.common.base.Preconditions; +import org.apache.druid.data.input.impl.CloudObjectLocation; +import org.apache.druid.segment.loading.LoadSpec; +import org.apache.druid.segment.loading.SegmentLoadingException; + +import java.io.File; + +/** + * + */ +@JsonTypeName(OssStorageDruidModule.SCHEME_S3_ZIP) +public class OssLoadSpec implements LoadSpec +{ + private final String bucket; + private final String key; + + private final OssDataSegmentPuller puller; + + @JsonCreator + public OssLoadSpec( + @JacksonInject OssDataSegmentPuller puller, + @JsonProperty(OssDataSegmentPuller.BUCKET) String bucket, + @JsonProperty(OssDataSegmentPuller.KEY) String key + ) + { + Preconditions.checkNotNull(bucket); + Preconditions.checkNotNull(key); + this.bucket = bucket; + this.key = key; + this.puller = puller; + } + + @Override + public LoadSpecResult loadSegment(File outDir) throws SegmentLoadingException + { + return new LoadSpecResult(puller.getSegmentFiles(new CloudObjectLocation(bucket, key), outDir).size()); + } + + @JsonProperty(OssDataSegmentPuller.BUCKET) + public String getBucket() + { + return bucket; + } + + @JsonProperty(OssDataSegmentPuller.KEY) + public String getKey() + { + return key; + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java new file mode 100644 index 000000000000..cf0f9e3ecc7e --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +/** + * General configurations for Amazon S3 storage. + */ +public class OssStorageConfig +{ + +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java new file mode 100644 index 000000000000..95f83f0a87d0 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.util.List; + +import org.apache.druid.data.SearchableVersionedDataFinder; +import org.apache.druid.guice.Binders; +import org.apache.druid.guice.JsonConfigProvider; +import org.apache.druid.guice.LazySingleton; +import org.apache.druid.initialization.DruidModule; + +import com.fasterxml.jackson.core.Version; +import com.fasterxml.jackson.databind.Module; +import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; +import com.google.inject.multibindings.MapBinder; + +/** + * + */ +public class OssStorageDruidModule implements DruidModule +{ + public static final String SCHEME = "oss"; + public static final String SCHEME_S3N = "oss_3n"; + public static final String SCHEME_S3_ZIP = "oss_zip"; + + @Override + public List getJacksonModules() + { + return ImmutableList.of( + new Module() + { + @Override + public String getModuleName() + { + return "DruidOss-" + System.identityHashCode(this); + } + + @Override + public Version version() + { + return Version.unknownVersion(); + } + + @Override + public void setupModule(SetupContext context) + { + context.registerSubtypes(OssLoadSpec.class); + } + } + ); + } + + @Override + public void configure(Binder binder) + { + MapBinder.newMapBinder(binder, String.class, SearchableVersionedDataFinder.class) + .addBinding(SCHEME) + .to(OssTimestampVersionedDataFinder.class) + .in(LazySingleton.class); + MapBinder.newMapBinder(binder, String.class, SearchableVersionedDataFinder.class) + .addBinding(SCHEME_S3N) + .to(OssTimestampVersionedDataFinder.class) + .in(LazySingleton.class); + Binders.dataSegmentKillerBinder(binder).addBinding(SCHEME_S3_ZIP).to(OssDataSegmentKiller.class).in(LazySingleton.class); + Binders.dataSegmentMoverBinder(binder).addBinding(SCHEME_S3_ZIP).to(OssDataSegmentMover.class).in(LazySingleton.class); + Binders.dataSegmentArchiverBinder(binder) + .addBinding(SCHEME_S3_ZIP) + .to(OssDataSegmentArchiver.class) + .in(LazySingleton.class); + Binders.dataSegmentPusherBinder(binder).addBinding(SCHEME).to(OssDataSegmentPusher.class).in(LazySingleton.class); + JsonConfigProvider.bind(binder, "druid.storage", OssInputDataConfig.class); + JsonConfigProvider.bind(binder, "druid.storage", OssDataSegmentPusherConfig.class); + JsonConfigProvider.bind(binder, "druid.storage", OssDataSegmentArchiverConfig.class); + JsonConfigProvider.bind(binder, "druid.storage", OssStorageConfig.class); + + Binders.taskLogsBinder(binder).addBinding(SCHEME).to(OssTaskLogs.class); + JsonConfigProvider.bind(binder, "druid.indexer.logs", OssTaskLogsConfig.class); + binder.bind(OssTaskLogs.class).in(LazySingleton.class); + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java new file mode 100644 index 000000000000..6ee55f3adda1 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Date; + +import org.apache.druid.common.utils.CurrentTimeMillisSupplier; +import org.apache.druid.java.util.common.IOE; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.tasklogs.TaskLogs; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.GetObjectRequest; +import com.aliyun.oss.model.ObjectMetadata; +import com.google.common.base.Optional; +import com.google.common.base.Throwables; +import com.google.common.io.ByteSource; +import com.google.inject.Inject; + +/** + * Provides task logs archived on S3. + */ +public class OssTaskLogs implements TaskLogs +{ + private static final Logger log = new Logger(OssTaskLogs.class); + + private final OSS client; + private final OssTaskLogsConfig config; + private final OssInputDataConfig inputDataConfig; + private final CurrentTimeMillisSupplier timeSupplier; + + @Inject + public OssTaskLogs( + OSS service, + OssTaskLogsConfig config, + OssInputDataConfig inputDataConfig, + CurrentTimeMillisSupplier timeSupplier + ) + { + this.client = service; + this.config = config; + this.inputDataConfig = inputDataConfig; + this.timeSupplier = timeSupplier; + } + + @Override + public Optional streamTaskLog(final String taskid, final long offset) throws IOException + { + final String taskKey = getTaskLogKey(taskid, "log"); + return streamTaskFile(offset, taskKey); + } + + @Override + public Optional streamTaskReports(String taskid) throws IOException + { + final String taskKey = getTaskLogKey(taskid, "report.json"); + return streamTaskFile(0, taskKey); + } + + private Optional streamTaskFile(final long offset, String taskKey) throws IOException + { + try { + final ObjectMetadata objectMetadata = client.getObjectMetadata(config.getBucket(), taskKey); + + return Optional.of( + new ByteSource() + { + @Override + public InputStream openStream() throws IOException + { + try { + final long start; + final long end = objectMetadata.getContentLength() - 1; + + if (offset > 0 && offset < objectMetadata.getContentLength()) { + start = offset; + } else if (offset < 0 && (-1 * offset) < objectMetadata.getContentLength()) { + start = objectMetadata.getContentLength() + offset; + } else { + start = 0; + } + + final GetObjectRequest request = new GetObjectRequest(config.getBucket(), taskKey); + request.setMatchingETagConstraints(Arrays.asList(objectMetadata.getETag())); + request.setRange(start, end); + + return client.getObject(request).getObjectContent(); + } + catch (OSSException e) { + throw new IOException(e); + } + } + } + ); + } + catch (OSSException e) { + if ( "NoSuchKey".equals(e.getErrorCode()) + || "NoSuchBucket".equals(e.getErrorCode())) { + return Optional.absent(); + } else { + throw new IOE(e, "Failed to stream logs from: %s", taskKey); + } + } + } + + @Override + public void pushTaskLog(final String taskid, final File logFile) throws IOException + { + final String taskKey = getTaskLogKey(taskid, "log"); + log.info("Pushing task log %s to: %s", logFile, taskKey); + pushTaskFile(logFile, taskKey); + } + + @Override + public void pushTaskReports(String taskid, File reportFile) throws IOException + { + final String taskKey = getTaskLogKey(taskid, "report.json"); + log.info("Pushing task reports %s to: %s", reportFile, taskKey); + pushTaskFile(reportFile, taskKey); + } + + private void pushTaskFile(final File logFile, String taskKey) throws IOException + { + try { + OssUtils.retry( + () -> { + OssUtils.uploadFileIfPossible(client, config.getDisableAcl(), config.getBucket(), taskKey, logFile); + return null; + } + ); + } + catch (Exception e) { + Throwables.propagateIfInstanceOf(e, IOException.class); + throw new RuntimeException(e); + } + } + + String getTaskLogKey(String taskid, String filename) + { + return StringUtils.format("%s/%s/%s", config.getPrefix(), taskid, filename); + } + + @Override + public void killAll() throws IOException + { + log.info( + "Deleting all task logs from s3 location [bucket: '%s' prefix: '%s'].", + config.getBucket(), + config.getPrefix() + ); + + long now = timeSupplier.getAsLong(); + killOlderThan(now); + } + + @Override + public void killOlderThan(long timestamp) throws IOException + { + log.info( + "Deleting all task logs from s3 location [bucket: '%s' prefix: '%s'] older than %s.", + config.getBucket(), + config.getPrefix(), + new Date(timestamp) + ); + try { + OssUtils.deleteObjectsInPath( + client, + inputDataConfig, + config.getBucket(), + config.getPrefix(), + (object) -> object.getLastModified().getTime() < timestamp + ); + } + catch (Exception e) { + log.error("Error occurred while deleting task log files from oss. Error: %s", e.getMessage()); + throw new IOException(e); + } + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java new file mode 100644 index 000000000000..e2be7478132c --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; + +import javax.validation.constraints.NotNull; + +/** + */ +public class OssTaskLogsConfig +{ + @JsonProperty + @NotNull + private String bucket = null; + + @JsonProperty + @NotNull + private String prefix = null; + + @JsonProperty + private boolean disableAcl = false; + + @VisibleForTesting + void setDisableAcl(boolean disableAcl) + { + this.disableAcl = disableAcl; + } + + public String getBucket() + { + return bucket; + } + + @VisibleForTesting + void setBucket(String bucket) + { + this.bucket = bucket; + } + + public String getPrefix() + { + return prefix; + } + + @VisibleForTesting + void setPrefix(String prefix) + { + this.prefix = prefix; + } + + public boolean getDisableAcl() + { + return disableAcl; + } + +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java new file mode 100644 index 000000000000..bf884f0decf0 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.net.URI; +import java.util.Collections; +import java.util.Iterator; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +import org.apache.druid.data.SearchableVersionedDataFinder; +import org.apache.druid.data.input.impl.CloudObjectLocation; +import org.apache.druid.java.util.common.StringUtils; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.model.OSSObjectSummary; +import com.google.inject.Inject; + +public class OssTimestampVersionedDataFinder extends OssDataSegmentPuller implements SearchableVersionedDataFinder +{ + private static final int MAX_LISTING_KEYS = 1000; + + @Inject + public OssTimestampVersionedDataFinder(OSS client) + { + super(client); + } + + /** + * Gets the key with the most recently modified timestamp. + * `pattern` is evaluated against the entire key AFTER the path given in `uri`. + * The substring `pattern` is matched against will have a leading `/` removed. + * For example `s3://some_bucket/some_prefix/some_key` with a URI of `s3://some_bucket/some_prefix` will match against `some_key`. + * `s3://some_bucket/some_prefixsome_key` with a URI of `s3://some_bucket/some_prefix` will match against `some_key` + * `s3://some_bucket/some_prefix//some_key` with a URI of `s3://some_bucket/some_prefix` will match against `/some_key` + * + * @param uri The URI of in the form of `s3://some_bucket/some_key` + * @param pattern The pattern matcher to determine if a *key* is of interest, or `null` to match everything. + * + * @return A URI to the most recently modified object which matched the pattern. + */ + @Override + public URI getLatestVersion(final URI uri, final @Nullable Pattern pattern) + { + try { + final CloudObjectLocation coords = new CloudObjectLocation(OssUtils.checkURI(uri)); + long mostRecent = Long.MIN_VALUE; + URI latest = null; + final Iterator objectSummaryIterator = OssUtils.objectSummaryIterator( + client, + Collections.singletonList(uri), + MAX_LISTING_KEYS + ); + while (objectSummaryIterator.hasNext()) { + final OSSObjectSummary objectSummary = objectSummaryIterator.next(); + final CloudObjectLocation objectLocation = OssUtils.summaryToCloudObjectLocation(objectSummary); + // remove coords path prefix from object path + String keyString = StringUtils.maybeRemoveLeadingSlash( + objectLocation.getPath().substring(coords.getPath().length()) + ); + if (pattern != null && !pattern.matcher(keyString).matches()) { + continue; + } + final long latestModified = objectSummary.getLastModified().getTime(); + if (latestModified >= mostRecent) { + mostRecent = latestModified; + latest = objectLocation.toUri(OssStorageDruidModule.SCHEME); + } + } + return latest; + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + +} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java new file mode 100644 index 000000000000..4ef3c394c742 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.druid.data.input.impl.CloudObjectLocation; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.RetryUtils; +import org.apache.druid.java.util.common.RetryUtils.Task; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.AccessControlList; +import com.aliyun.oss.model.CannedAccessControlList; +import com.aliyun.oss.model.DeleteObjectsRequest; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.ObjectListing; +import com.aliyun.oss.model.PutObjectRequest; +import com.google.common.base.Joiner; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; + +/** + * + */ +public class OssUtils +{ + private static final String SCHEME = OssStorageDruidModule.SCHEME; + private static final Joiner JOINER = Joiner.on("/").skipNulls(); + private static final Logger log = new Logger(OssUtils.class); + + + static boolean isServiceExceptionRecoverable(OSSException ex) + { + final boolean isIOException = ex.getCause() instanceof IOException; + final boolean isTimeout = "RequestTimeout".equals(ex.getErrorCode()); + final boolean badStatusCode = false;//ex. == 400 || ex.getStatusCode() == 403 || ex.getStatusCode() == 404; + return !badStatusCode && (isIOException || isTimeout); + } + + public static final Predicate RETRYABLE = new Predicate() + { + @Override + public boolean apply(Throwable e) + { + if (e == null) { + return false; + } else if (e instanceof IOException) { + return true; + } else if (e instanceof OSSException) { + return isServiceExceptionRecoverable((OSSException) e); + } else { + return apply(e.getCause()); + } + } + }; + + /** + * Retries S3 operations that fail due to io-related exceptions. Service-level exceptions (access denied, file not + * found, etc) are not retried. + */ + static T retry(Task f) throws Exception + { + return RetryUtils.retry(f, RETRYABLE, RetryUtils.DEFAULT_MAX_TRIES); + } + + static boolean isObjectInBucketIgnoringPermission( + OSS client, + String bucketName, + String objectKey + ) + { + try { + return client.doesObjectExist(bucketName, objectKey); + } + catch (OSSException e) { + if (e.getErrorCode().equals("NoSuchKey")) { + // Object is inaccessible to current user, but does exist. + return true; + } + // Something else has gone wrong + throw e; + } + } + + /** + * Create an iterator over a set of S3 objects specified by a set of prefixes. + * + * For each provided prefix URI, the iterator will walk through all objects that are in the same bucket as the + * provided URI and whose keys start with that URI's path, except for directory placeholders (which will be + * ignored). The iterator is computed incrementally by calling {@link OssClientHelper#listObjectsV2} for + * each prefix in batches of {@param maxListLength}. The first call is made at the same time the iterator is + * constructed. + */ + public static Iterator objectSummaryIterator( + final OSS client, + final Iterable prefixes, + final int maxListingLength + ) + { + return new ObjectSummaryIterator(client, prefixes, maxListingLength); + } + + /** + * Create an {@link URI} from the given {@link S3ObjectSummary}. The result URI is composed as below. + * + * 
+   * {@code s3://{BUCKET_NAME}/{OBJECT_KEY}}
+   *
+ */ + public static URI summaryToUri(OSSObjectSummary object) + { + return summaryToCloudObjectLocation(object).toUri(SCHEME); + } + + public static CloudObjectLocation summaryToCloudObjectLocation(OSSObjectSummary object) + { + return new CloudObjectLocation(object.getBucketName(), object.getKey()); + } + + static String constructSegmentPath(String baseKey, String storageDir) + { + return JOINER.join( + baseKey.isEmpty() ? null : baseKey, + storageDir + ) + "/index.zip"; + } + + static CannedAccessControlList grantFullControlToBucketOwner(OSS client, String bucket) + { + final AccessControlList acl = client.getBucketAcl(bucket); + return acl.getCannedACL(); + //acl.grantAllPermissions(new Grant(new Grantee(acl.getOwner().getId()), Permission.FullControl)); + } + + public static String extractKey(URI uri) + { + return StringUtils.maybeRemoveLeadingSlash(uri.getPath()); + } + + public static URI checkURI(URI uri) + { + if (uri.getScheme().equalsIgnoreCase(OssStorageDruidModule.SCHEME_S3_ZIP)) { + uri = URI.create(SCHEME + uri.toString().substring(OssStorageDruidModule.SCHEME_S3_ZIP.length())); + } + return CloudObjectLocation.validateUriScheme(SCHEME, uri); + } + + /** + * Gets a single {@link S3ObjectSummary} from s3. Since this method might return a wrong object if there are multiple + * objects that match the given key, this method should be used only when it's guaranteed that the given key is unique + * in the given bucket. + * + * @param client s3 client + * @param bucket s3 bucket + * @param key unique key for the object to be retrieved + */ + public static OSSObjectSummary getSingleObjectSummary(OSS client, String bucket, String key) + { + final ListObjectsRequest request = new ListObjectsRequest(); + request.setBucketName(bucket); + request.setPrefix(key); + request.setMaxKeys(1); + final ObjectListing result = client.listObjects(request); + + // Using getObjectSummaries().size() instead of getKeyCount as, in some cases + // it is observed that even though the getObjectSummaries returns some data + // keyCount is still zero. + if (result.getObjectSummaries().size() == 0) { + throw new ISE("Cannot find object for bucket[%s] and key[%s]", bucket, key); + } + final OSSObjectSummary objectSummary = result.getObjectSummaries().get(0); + if (!objectSummary.getBucketName().equals(bucket) || !objectSummary.getKey().equals(key)) { + throw new ISE("Wrong object[%s] for bucket[%s] and key[%s]", objectSummary, bucket, key); + } + + return objectSummary; + } + + /** + * Delete the files from S3 in a specified bucket, matching a specified prefix and filter + * @param client s3 client + * @param config specifies the configuration to use when finding matching files in S3 to delete + * @param bucket s3 bucket + * @param prefix the file prefix + * @param filter function which returns true if the prefix file found should be deleted and false otherwise. + * @throws Exception + */ + public static void deleteObjectsInPath( + OSS client, + OssInputDataConfig config, + String bucket, + String prefix, + Predicate filter + ) + throws Exception + { + final List keysToDelete = new ArrayList<>(config.getMaxListingLength()); + final ObjectSummaryIterator iterator = new ObjectSummaryIterator( + client, + ImmutableList.of(new CloudObjectLocation(bucket, prefix).toUri("http")), + config.getMaxListingLength() + ); + + while (iterator.hasNext()) { + final OSSObjectSummary nextObject = iterator.next(); + if (filter.apply(nextObject)) { + keysToDelete.add(nextObject.getKey()); + if (keysToDelete.size() == config.getMaxListingLength()) { + deleteBucketKeys(client, bucket, keysToDelete); + log.info("Deleted %d files", keysToDelete.size()); + keysToDelete.clear(); + } + } + } + + if (keysToDelete.size() > 0) { + deleteBucketKeys(client, bucket, keysToDelete); + log.info("Deleted %d files", keysToDelete.size()); + } + } + + private static void deleteBucketKeys( + OSS client, + String bucket, + List keysToDelete + ) + throws Exception + { + DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket).withKeys(keysToDelete); + OssUtils.retry(() -> { + client.deleteObjects(deleteRequest); + return null; + }); + } + + /** + * Uploads a file to S3 if possible. First trying to set ACL to give the bucket owner full control of the file before uploading. + * + * @param client aliyun OSS client + * @param disableAcl true if ACL shouldn't be set for the file + * @param key The key under which to store the new object. + * @param file The path of the file to upload to Amazon S3. + */ + static void uploadFileIfPossible( + OSS client, + boolean disableAcl, + String bucket, + String key, + File file + ) + { + final PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, file); + + log.info("Pushing [%s] to bucket[%s] and key[%s].", file, bucket, key); + client.putObject(putObjectRequest); + + if (!disableAcl) { + client.setObjectAcl(bucket, key, OssUtils.grantFullControlToBucketOwner(client, bucket)); + } + } +} diff --git a/extensions-contrib/aliyun-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/aliyun-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 000000000000..3d434e7c9021 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.druid.storage.aliyun.OssStorageDruidModule +org.apache.druid.firehose.aliyun.OssFirehoseDruidModule +org.apache.druid.data.input.aliyun.OssInputSourceDruidModule \ No newline at end of file diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java new file mode 100644 index 000000000000..7252b8c267cc --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.Assert; +import org.junit.Test; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClient; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.ObjectListing; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; + +public class ObjectSummaryIteratorTest +{ + private static final ImmutableList TEST_OBJECTS = + ImmutableList.of( + makeObjectSummary("b", "foo", 10L), + makeObjectSummary("b", "foo/", 0L), // directory + makeObjectSummary("b", "foo/bar1", 10L), + makeObjectSummary("b", "foo/bar2", 10L), + makeObjectSummary("b", "foo/bar3", 10L), + makeObjectSummary("b", "foo/bar4", 10L), + makeObjectSummary("b", "foo/bar5", 0L), // empty object + makeObjectSummary("b", "foo/baz", 10L), + makeObjectSummary("bucketnotmine", "a/different/bucket", 10L), + makeObjectSummary("b", "foo/bar/", 0L) // another directory at the end of list + ); + + @Test + public void testSingleObject() + { + test( + ImmutableList.of("oss://b/foo/baz"), + ImmutableList.of("oss://b/foo/baz"), + 5 + ); + } + + @Test + public void testMultiObjectOneKeyAtATime() + { + test( + ImmutableList.of("oss://b/foo/bar1", "oss://b/foo/bar2", "oss://b/foo/bar3", "oss://b/foo/bar4", "oss://b/foo/baz"), + ImmutableList.of("oss://b/foo/"), + 1 + ); + } + + @Test + public void testMultiObjectTwoKeysAtATime() + { + test( + ImmutableList.of("oss://b/foo/bar1", "oss://b/foo/bar2", "oss://b/foo/bar3", "oss://b/foo/bar4", "oss://b/foo/baz"), + ImmutableList.of("oss://b/foo/"), + 2 + ); + } + + @Test + public void testMultiObjectTenKeysAtATime() + { + test( + ImmutableList.of("oss://b/foo/bar1", "oss://b/foo/bar2", "oss://b/foo/bar3", "oss://b/foo/bar4", "oss://b/foo/baz"), + ImmutableList.of("oss://b/foo/"), + 10 + ); + } + + @Test + public void testPrefixInMiddleOfKey() + { + test( + ImmutableList.of("oss://b/foo/bar1", "oss://b/foo/bar2", "oss://b/foo/bar3", "oss://b/foo/bar4"), + ImmutableList.of("oss://b/foo/bar"), + 10 + ); + } + + @Test + public void testNoPath() + { + test( + ImmutableList.of( + "oss://b/foo", + "oss://b/foo/bar1", + "oss://b/foo/bar2", + "oss://b/foo/bar3", + "oss://b/foo/bar4", + "oss://b/foo/baz" + ), + ImmutableList.of("oss://b"), + 10 + ); + } + + @Test + public void testSlashPath() + { + test( + ImmutableList.of( + "oss://b/foo", + "oss://b/foo/bar1", + "oss://b/foo/bar2", + "oss://b/foo/bar3", + "oss://b/foo/bar4", + "oss://b/foo/baz" + ), + ImmutableList.of("oss://b/"), + 10 + ); + } + + @Test + public void testDifferentBucket() + { + test( + ImmutableList.of(), + ImmutableList.of("oss://bx/foo/"), + 10 + ); + } + + @Test + public void testWithMultiplePrefixesReturningAllNonEmptyObjectsStartingWithOneOfPrefixes() + { + test( + ImmutableList.of("oss://b/foo/bar1", "oss://b/foo/bar2", "oss://b/foo/bar3", "oss://b/foo/bar4", "oss://b/foo/baz"), + ImmutableList.of("oss://b/foo/bar", "oss://b/foo/baz"), + 10 + ); + } + + private static void test( + final List expectedUris, + final List prefixes, + final int maxListingLength + ) + { + final List expectedObjects = new ArrayList<>(); + + // O(N^2) but who cares -- the list is short. + for (final String uri : expectedUris) { + final List matches = TEST_OBJECTS.stream() + .filter( + summary -> + OssUtils.summaryToUri(summary).toString().equals(uri) + ) + .collect(Collectors.toList()); + + expectedObjects.add(Iterables.getOnlyElement(matches)); + } + + final List actualObjects = ImmutableList.copyOf( + OssUtils.objectSummaryIterator( + makeMockClient(TEST_OBJECTS), + prefixes.stream().map(URI::create).collect(Collectors.toList()), + maxListingLength + ) + ); + + Assert.assertEquals( + prefixes.toString(), + expectedObjects.stream().map(OssUtils::summaryToUri).collect(Collectors.toList()), + actualObjects.stream().map(OssUtils::summaryToUri).collect(Collectors.toList()) + ); + } + + /** + * Makes a mock S3 client that handles enough of "listObjectsV2" to test the functionality of the + * {@link ObjectSummaryIterator} class. + */ + private static OSS makeMockClient( + final List objects + ) + { + return new OSSClient("endpoint", "accessKey", "keySecret") + { + @Override + public ObjectListing listObjects(final ListObjectsRequest request) + { + // Continuation token is an index in the "objects" list.q + final String continuationToken = request.getMarker(); + final int startIndex = continuationToken == null ? 0 : Integer.parseInt(continuationToken); + + // Find matching objects. + final List summaries = new ArrayList<>(); + int nextIndex = -1; + + for (int i = startIndex; i < objects.size(); i++) { + final OSSObjectSummary summary = objects.get(i); + + if (summary.getBucketName().equals(request.getBucketName()) + && summary.getKey().startsWith(request.getPrefix())) { + + if (summaries.size() == request.getMaxKeys()) { + // We reached our max key limit; set nextIndex (which will lead to a result with truncated = true). + nextIndex = i; + break; + } + + // Generate a summary. + summaries.add(summary); + } + } + + // Generate the result. + final ObjectListing retVal = new ObjectListing(); + retVal.getObjectSummaries().addAll(summaries); + + if (nextIndex >= 0) { + retVal.setTruncated(true); + retVal.setNextMarker(String.valueOf(nextIndex)); + } + + return retVal; + } + }; + } + + private static OSSObjectSummary makeObjectSummary(final String bucket, final String key, final long size) + { + final OSSObjectSummary summary = new OSSObjectSummary(); + summary.setBucketName(bucket); + summary.setKey(key); + summary.setSize(size); + return summary; + } +} diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java new file mode 100644 index 000000000000..7bc09214c965 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.util.Map; + +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.timeline.DataSegment; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClient; +import com.fasterxml.jackson.databind.BeanProperty; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.InjectableValues; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +public class OssDataSegmentArchiverTest +{ + private static final ObjectMapper MAPPER = new DefaultObjectMapper() + .setInjectableValues( + new InjectableValues() + { + @Override + public Object findInjectableValue( + Object valueId, + DeserializationContext ctxt, + BeanProperty forProperty, + Object beanInstance + ) + { + return PULLER; + } + } + ) + .registerModule(new SimpleModule("s3-archive-test-module").registerSubtypes(OssLoadSpec.class)); + private static final OssDataSegmentArchiverConfig ARCHIVER_CONFIG = new OssDataSegmentArchiverConfig() + { + @Override + public String getArchiveBucket() + { + return "archive_bucket"; + } + + @Override + public String getArchiveBaseKey() + { + return "archive_base_key"; + } + }; + private static final OssDataSegmentPusherConfig PUSHER_CONFIG = new OssDataSegmentPusherConfig(); + private static final OSS OSS_CLIENT = EasyMock.createStrictMock(OSSClient.class); + private static final OssDataSegmentPuller PULLER = new OssDataSegmentPuller(OSS_CLIENT); + private static final DataSegment SOURCE_SEGMENT = DataSegment + .builder() + .binaryVersion(1) + .dataSource("dataSource") + .dimensions(ImmutableList.of()) + .interval(Intervals.of("2015/2016")) + .version("version") + .loadSpec(ImmutableMap.of( + "type", + OssStorageDruidModule.SCHEME_S3_ZIP, + OssDataSegmentPuller.BUCKET, + "source_bucket", + OssDataSegmentPuller.KEY, + "source_key" + )) + .size(0) + .build(); + + @BeforeClass + public static void setUpStatic() + { + PUSHER_CONFIG.setBaseKey("push_base"); + PUSHER_CONFIG.setBucket("push_bucket"); + } + + @Test + public void testSimpleArchive() throws Exception + { + final DataSegment archivedSegment = SOURCE_SEGMENT + .withLoadSpec(ImmutableMap.of( + "type", + OssStorageDruidModule.SCHEME_S3_ZIP, + OssDataSegmentPuller.BUCKET, + ARCHIVER_CONFIG.getArchiveBucket(), + OssDataSegmentPuller.KEY, + ARCHIVER_CONFIG.getArchiveBaseKey() + "archived" + )); + final OssDataSegmentArchiver archiver = new OssDataSegmentArchiver(MAPPER, OSS_CLIENT, ARCHIVER_CONFIG, PUSHER_CONFIG) + { + @Override + public DataSegment move(DataSegment segment, Map targetLoadSpec) + { + return archivedSegment; + } + }; + Assert.assertEquals(archivedSegment, archiver.archive(SOURCE_SEGMENT)); + } + + @Test + public void testSimpleArchiveDoesntMove() throws Exception + { + final OssDataSegmentArchiver archiver = new OssDataSegmentArchiver(MAPPER, OSS_CLIENT, ARCHIVER_CONFIG, PUSHER_CONFIG) + { + @Override + public DataSegment move(DataSegment segment, Map targetLoadSpec) + { + return SOURCE_SEGMENT; + } + }; + Assert.assertNull(archiver.archive(SOURCE_SEGMENT)); + } + + @Test + public void testSimpleRestore() throws Exception + { + final DataSegment archivedSegment = SOURCE_SEGMENT + .withLoadSpec(ImmutableMap.of( + "type", + OssStorageDruidModule.SCHEME_S3_ZIP, + OssDataSegmentPuller.BUCKET, + ARCHIVER_CONFIG.getArchiveBucket(), + OssDataSegmentPuller.KEY, + ARCHIVER_CONFIG.getArchiveBaseKey() + "archived" + )); + final OssDataSegmentArchiver archiver = new OssDataSegmentArchiver(MAPPER, OSS_CLIENT, ARCHIVER_CONFIG, PUSHER_CONFIG) + { + @Override + public DataSegment move(DataSegment segment, Map targetLoadSpec) + { + return archivedSegment; + } + }; + Assert.assertEquals(archivedSegment, archiver.restore(SOURCE_SEGMENT)); + } + + @Test + public void testSimpleRestoreDoesntMove() throws Exception + { + final OssDataSegmentArchiver archiver = new OssDataSegmentArchiver(MAPPER, OSS_CLIENT, ARCHIVER_CONFIG, PUSHER_CONFIG) + { + @Override + public DataSegment move(DataSegment segment, Map targetLoadSpec) + { + return SOURCE_SEGMENT; + } + }; + Assert.assertNull(archiver.restore(SOURCE_SEGMENT)); + } +} diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java new file mode 100644 index 000000000000..fd854ec2c256 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.io.IOException; +import java.net.URI; +import java.util.Arrays; + +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.StringUtils; +import org.easymock.EasyMock; +import org.easymock.EasyMockRunner; +import org.easymock.EasyMockSupport; +import org.easymock.Mock; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.model.DeleteObjectsRequest; +import com.aliyun.oss.model.OSSObjectSummary; +import com.amazonaws.SdkClientException; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +@RunWith(EasyMockRunner.class) +public class OssDataSegmentKillerTest extends EasyMockSupport +{ + private static final String KEY_1 = "key1"; + private static final String KEY_2 = "key2"; + private static final String TEST_BUCKET = "test_bucket"; + private static final String TEST_PREFIX = "test_prefix"; + private static final URI PREFIX_URI = URI.create(StringUtils.format("s3://%s/%s", TEST_BUCKET, TEST_PREFIX)); + private static final long TIME_0 = 0L; + private static final long TIME_1 = 1L; + private static final int MAX_KEYS = 1; + private static final Exception RECOVERABLE_EXCEPTION = new SdkClientException(new IOException()); + private static final Exception NON_RECOVERABLE_EXCEPTION = new SdkClientException(new NullPointerException()); + + @Mock + private OSS client; + @Mock + private OssDataSegmentPusherConfig segmentPusherConfig; + @Mock + private OssInputDataConfig inputDataConfig; + + private OssDataSegmentKiller segmentKiller; + + @Test + public void test_killAll_accountConfigWithNullBucketAndBaseKey_throwsISEException() throws IOException + { + EasyMock.expect(segmentPusherConfig.getBucket()).andReturn(null); + EasyMock.expectLastCall().atLeastOnce(); + EasyMock.expect(segmentPusherConfig.getBaseKey()).andReturn(null); + EasyMock.expectLastCall().anyTimes(); + + boolean thrownISEException = false; + + try { + + EasyMock.replay(client, segmentPusherConfig, inputDataConfig); + + segmentKiller = new OssDataSegmentKiller(client, segmentPusherConfig, inputDataConfig); + segmentKiller.killAll(); + } + catch (ISE e) { + thrownISEException = true; + } + Assert.assertTrue(thrownISEException); + EasyMock.verify(client, segmentPusherConfig, inputDataConfig); + } + + @Test + public void test_killAll_noException_deletesAllSegments() throws IOException + { + OSSObjectSummary objectSummary1 = OssTestUtils.newOSSObjectSummary(TEST_BUCKET, KEY_1, TIME_0); + OSSObjectSummary objectSummary2 = OssTestUtils.newOSSObjectSummary(TEST_BUCKET, KEY_2, TIME_1); + + OssTestUtils.expectListObjects( + client, + PREFIX_URI, + ImmutableList.of(objectSummary1, objectSummary2) + ); + + DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET); + deleteRequest1.setKeys(Arrays.asList(KEY_1)); + DeleteObjectsRequest deleteRequest2 = new DeleteObjectsRequest(TEST_BUCKET); + deleteRequest2.setKeys(Arrays.asList(KEY_2)); + + OssTestUtils.mockClientDeleteObjects( + client, + ImmutableList.of(deleteRequest1, deleteRequest2), + ImmutableMap.of() + ); + + EasyMock.expect(segmentPusherConfig.getBucket()).andReturn(TEST_BUCKET); + EasyMock.expectLastCall().anyTimes(); + EasyMock.expect(segmentPusherConfig.getBaseKey()).andReturn(TEST_PREFIX); + EasyMock.expectLastCall().anyTimes(); + + EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS); + EasyMock.expectLastCall().anyTimes(); + + EasyMock.replay(client, segmentPusherConfig, inputDataConfig); + + segmentKiller = new OssDataSegmentKiller(client, segmentPusherConfig, inputDataConfig); + segmentKiller.killAll(); + EasyMock.verify(client, segmentPusherConfig, inputDataConfig); + } + + @Test + public void test_killAll_recoverableExceptionWhenListingObjects_deletesAllSegments() throws IOException + { + OSSObjectSummary objectSummary1 = OssTestUtils.newOSSObjectSummary(TEST_BUCKET, KEY_1, TIME_0); + + OssTestUtils.expectListObjects( + client, + PREFIX_URI, + ImmutableList.of(objectSummary1) + ); + + DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET); + deleteRequest1.setKeys(Arrays.asList(KEY_1)); + + OssTestUtils.mockClientDeleteObjects( + client, + ImmutableList.of(deleteRequest1), + ImmutableMap.of(deleteRequest1, RECOVERABLE_EXCEPTION) + ); + + EasyMock.expect(segmentPusherConfig.getBucket()).andReturn(TEST_BUCKET); + EasyMock.expectLastCall().anyTimes(); + EasyMock.expect(segmentPusherConfig.getBaseKey()).andReturn(TEST_PREFIX); + EasyMock.expectLastCall().anyTimes(); + + EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS); + EasyMock.expectLastCall().anyTimes(); + + EasyMock.replay(client, segmentPusherConfig, inputDataConfig); + + segmentKiller = new OssDataSegmentKiller(client, segmentPusherConfig, inputDataConfig); + segmentKiller.killAll(); + EasyMock.verify(client, segmentPusherConfig, inputDataConfig); + } + + @Test + public void test_killAll_nonrecoverableExceptionWhenListingObjects_deletesAllSegments() + { + boolean ioExceptionThrown = false; + try { + OSSObjectSummary objectSummary1 = OssTestUtils.newOSSObjectSummary(TEST_BUCKET, KEY_1, TIME_0); + + OssTestUtils.expectListObjects( + client, + PREFIX_URI, + ImmutableList.of(objectSummary1) + ); + + DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET); + deleteRequest1.withKeys(ImmutableList.of(KEY_1)); + + OssTestUtils.mockClientDeleteObjects( + client, + ImmutableList.of(), + ImmutableMap.of(deleteRequest1, NON_RECOVERABLE_EXCEPTION) + ); + + + EasyMock.expect(segmentPusherConfig.getBucket()).andReturn(TEST_BUCKET); + EasyMock.expectLastCall().anyTimes(); + EasyMock.expect(segmentPusherConfig.getBaseKey()).andReturn(TEST_PREFIX); + EasyMock.expectLastCall().anyTimes(); + + EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS); + EasyMock.expectLastCall().anyTimes(); + + EasyMock.replay(client, segmentPusherConfig, inputDataConfig); + + segmentKiller = new OssDataSegmentKiller(client, segmentPusherConfig, inputDataConfig); + segmentKiller.killAll(); + } + catch (IOException e) { + ioExceptionThrown = true; + } + + Assert.assertTrue(ioExceptionThrown); + EasyMock.verify(client, segmentPusherConfig, inputDataConfig); + } +} diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java new file mode 100644 index 000000000000..110b4ad7962b --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.io.File; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.MapUtils; +import org.apache.druid.segment.loading.SegmentLoadingException; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.NoneShardSpec; +import org.junit.Assert; +import org.junit.Test; + +import com.aliyun.oss.OSSClient; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.CopyObjectRequest; +import com.aliyun.oss.model.CopyObjectResult; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.ObjectListing; +import com.aliyun.oss.model.PutObjectResult; +import com.aliyun.oss.model.StorageClass; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +public class OssDataSegmentMoverTest +{ + private static final DataSegment SOURCE_SEGMENT = new DataSegment( + "test", + Intervals.of("2013-01-01/2013-01-02"), + "1", + ImmutableMap.of( + "key", + "baseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip", + "bucket", + "main" + ), + ImmutableList.of("dim1", "dim1"), + ImmutableList.of("metric1", "metric2"), + NoneShardSpec.instance(), + 0, + 1 + ); + + @Test + public void testMove() throws Exception + { + MockClient mockClient = new MockClient(); + OssDataSegmentMover mover = new OssDataSegmentMover(mockClient, new OssDataSegmentPusherConfig()); + + mockClient.putObject( + "main", + "baseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip" + ); + + DataSegment movedSegment = mover.move( + SOURCE_SEGMENT, + ImmutableMap.of("baseKey", "targetBaseKey", "bucket", "archive") + ); + + Map targetLoadSpec = movedSegment.getLoadSpec(); + Assert.assertEquals( + "targetBaseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip", + MapUtils.getString(targetLoadSpec, "key") + ); + Assert.assertEquals("archive", MapUtils.getString(targetLoadSpec, "bucket")); + Assert.assertTrue(mockClient.didMove()); + } + + @Test + public void testMoveNoop() throws Exception + { + MockClient mockS3Client = new MockClient(); + OssDataSegmentMover mover = new OssDataSegmentMover(mockS3Client, new OssDataSegmentPusherConfig()); + + mockS3Client.putObject( + "archive", + "targetBaseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip" + ); + + DataSegment movedSegment = mover.move( + SOURCE_SEGMENT, + ImmutableMap.of("baseKey", "targetBaseKey", "bucket", "archive") + ); + + Map targetLoadSpec = movedSegment.getLoadSpec(); + + Assert.assertEquals( + "targetBaseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip", + MapUtils.getString(targetLoadSpec, "key") + ); + Assert.assertEquals("archive", MapUtils.getString(targetLoadSpec, "bucket")); + Assert.assertFalse(mockS3Client.didMove()); + } + + @Test(expected = SegmentLoadingException.class) + public void testMoveException() throws Exception + { + MockClient mockS3Client = new MockClient(); + OssDataSegmentMover mover = new OssDataSegmentMover(mockS3Client, new OssDataSegmentPusherConfig()); + + mover.move( + SOURCE_SEGMENT, + ImmutableMap.of("baseKey", "targetBaseKey", "bucket", "archive") + ); + } + + @Test + public void testIgnoresGoneButAlreadyMoved() throws Exception + { + MockClient mockS3Client = new MockClient(); + OssDataSegmentMover mover = new OssDataSegmentMover(mockS3Client, new OssDataSegmentPusherConfig()); + mover.move(new DataSegment( + "test", + Intervals.of("2013-01-01/2013-01-02"), + "1", + ImmutableMap.of( + "key", + "baseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip", + "bucket", + "DOES NOT EXIST" + ), + ImmutableList.of("dim1", "dim1"), + ImmutableList.of("metric1", "metric2"), + NoneShardSpec.instance(), + 0, + 1 + ), ImmutableMap.of("bucket", "DOES NOT EXIST", "baseKey", "baseKey")); + } + + @Test(expected = SegmentLoadingException.class) + public void testFailsToMoveMissing() throws Exception + { + MockClient client = new MockClient(); + OssDataSegmentMover mover = new OssDataSegmentMover(client, new OssDataSegmentPusherConfig()); + mover.move(new DataSegment( + "test", + Intervals.of("2013-01-01/2013-01-02"), + "1", + ImmutableMap.of( + "key", + "baseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip", + "bucket", + "DOES NOT EXIST" + ), + ImmutableList.of("dim1", "dim1"), + ImmutableList.of("metric1", "metric2"), + NoneShardSpec.instance(), + 0, + 1 + ), ImmutableMap.of("bucket", "DOES NOT EXIST", "baseKey", "baseKey2")); + } + + private static class MockClient extends OSSClient + { + Map> storage = new HashMap<>(); + boolean copied = false; + boolean deletedOld = false; + + private MockClient() + { + super("endpoint","accessKeyId","keySecret"); + } + + public boolean didMove() + { + return copied && deletedOld; + } + +// @Override +// public AccessControlList getBucketAcl(String bucketName) +// { +// final AccessControlList acl = new AccessControlList(); +// acl.setOwner(new Owner("ownerId", "owner")); +// acl.grantAllPermissions(new Grant(new CanonicalGrantee(acl.getOwner().getId()), Permission.FullControl)); +// return acl; +// } + + @Override + public boolean doesObjectExist(String bucketName, String objectKey) + { + Set objects = storage.get(bucketName); + return (objects != null && objects.contains(objectKey)); + } + + @Override + public ObjectListing listObjects(ListObjectsRequest listObjectsV2Request) + { + final String bucketName = listObjectsV2Request.getBucketName(); + final String objectKey = listObjectsV2Request.getPrefix(); + if (doesObjectExist(bucketName, objectKey)) { + final OSSObjectSummary objectSummary = new OSSObjectSummary(); + objectSummary.setBucketName(bucketName); + objectSummary.setKey(objectKey); + objectSummary.setStorageClass(StorageClass.Standard.name()); + + final ObjectListing result = new ObjectListing(); + result.setBucketName(bucketName); + result.setPrefix(objectKey); + //result.setKeyCount(1); + result.getObjectSummaries().add(objectSummary); + result.setTruncated(true); + return result; + } else { + return new ObjectListing(); + } + } + + @Override + public CopyObjectResult copyObject(CopyObjectRequest copyObjectRequest) + { + final String sourceBucketName = copyObjectRequest.getSourceBucketName(); + final String sourceObjectKey = copyObjectRequest.getSourceKey(); + final String destinationBucketName = copyObjectRequest.getDestinationBucketName(); + final String destinationObjectKey = copyObjectRequest.getDestinationKey(); + copied = true; + if (doesObjectExist(sourceBucketName, sourceObjectKey)) { + storage.computeIfAbsent(destinationBucketName, k -> new HashSet<>()) + .add(destinationObjectKey); + return new CopyObjectResult(); + } else { + final OSSException exception = new OSSException("OssDataSegmentMoverTest", "NoSuchKey",null, null, null, null, null); + throw exception; + } + } + + @Override + public void deleteObject(String bucket, String objectKey) + { + deletedOld = true; + storage.get(bucket).remove(objectKey); + } + + public PutObjectResult putObject(String bucketName, String key) + { + return putObject(bucketName, key, (File) null); + } + + @Override + public PutObjectResult putObject(String bucketName, String key, File file) + { + storage.computeIfAbsent(bucketName, bName -> new HashSet<>()).add(key); + return new PutObjectResult(); + } + } +} diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java new file mode 100644 index 000000000000..60e0b1070071 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.Date; +import java.util.zip.GZIPOutputStream; + +import org.apache.druid.data.input.impl.CloudObjectLocation; +import org.apache.druid.java.util.common.FileUtils; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.segment.loading.SegmentLoadingException; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.OSSObject; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.ObjectListing; + +/** + * + */ +public class OssDataSegmentPullerTest +{ + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testSimpleGetVersion() throws IOException + { + String bucket = "bucket"; + String keyPrefix = "prefix/dir/0"; + OSS s3Client = EasyMock.createStrictMock(OSS.class); + + final OSSObjectSummary objectSummary = new OSSObjectSummary(); + objectSummary.setBucketName(bucket); + objectSummary.setKey(keyPrefix + "/renames-0.gz"); + objectSummary.setLastModified(new Date(0)); + + final ObjectListing result = new ObjectListing(); + result.getObjectSummaries().add(objectSummary); + + EasyMock.expect(s3Client.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) + .andReturn(result) + .once(); + OssDataSegmentPuller puller = new OssDataSegmentPuller(s3Client); + + EasyMock.replay(s3Client); + + String version = puller.getVersion(URI.create(StringUtils.format("oss://%s/%s", bucket, objectSummary.getKey()))); + + EasyMock.verify(s3Client); + + Assert.assertEquals(StringUtils.format("%d", new Date(0).getTime()), version); + } + + @Test + public void testGZUncompress() throws IOException, SegmentLoadingException + { + final String bucket = "bucket"; + final String keyPrefix = "prefix/dir/0"; + final OSS s3Client = EasyMock.createStrictMock(OSS.class); + final byte[] value = bucket.getBytes(StandardCharsets.UTF_8); + + final File tmpFile = temporaryFolder.newFile("gzTest.gz"); + + try (OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(tmpFile))) { + outputStream.write(value); + } + + final OSSObject object0 = new OSSObject(); + object0.setBucketName(bucket); + object0.setKey(keyPrefix + "/renames-0.gz"); + object0.getObjectMetadata().setLastModified(new Date(0)); + object0.setObjectContent(new FileInputStream(tmpFile)); + + final OSSObjectSummary objectSummary = new OSSObjectSummary(); + objectSummary.setBucketName(bucket); + objectSummary.setKey(keyPrefix + "/renames-0.gz"); + objectSummary.setLastModified(new Date(0)); + + final ObjectListing listObjectsResult = new ObjectListing(); + listObjectsResult.getObjectSummaries().add(objectSummary); + + final File tmpDir = temporaryFolder.newFolder("gzTestDir"); + + EasyMock.expect(s3Client.doesObjectExist(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) + .andReturn(true) + .once(); + EasyMock.expect(s3Client.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) + .andReturn(listObjectsResult) + .once(); + EasyMock.expect(s3Client.getObject(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) + .andReturn(object0) + .once(); + OssDataSegmentPuller puller = new OssDataSegmentPuller(s3Client); + + EasyMock.replay(s3Client); + FileUtils.FileCopyResult result = puller.getSegmentFiles( + new CloudObjectLocation( + bucket, + object0.getKey() + ), tmpDir + ); + EasyMock.verify(s3Client); + + Assert.assertEquals(value.length, result.size()); + File expected = new File(tmpDir, "renames-0"); + Assert.assertTrue(expected.exists()); + Assert.assertEquals(value.length, expected.length()); + } + + @Test + public void testGZUncompressRetries() throws IOException, SegmentLoadingException + { + final String bucket = "bucket"; + final String keyPrefix = "prefix/dir/0"; + final OSS s3Client = EasyMock.createStrictMock(OSS.class); + final byte[] value = bucket.getBytes(StandardCharsets.UTF_8); + + final File tmpFile = temporaryFolder.newFile("gzTest.gz"); + + try (OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(tmpFile))) { + outputStream.write(value); + } + + OSSObject object0 = new OSSObject(); + + object0.setBucketName(bucket); + object0.setKey(keyPrefix + "/renames-0.gz"); + object0.getObjectMetadata().setLastModified(new Date(0)); + object0.setObjectContent(new FileInputStream(tmpFile)); + + final OSSObjectSummary objectSummary = new OSSObjectSummary(); + objectSummary.setBucketName(bucket); + objectSummary.setKey(keyPrefix + "/renames-0.gz"); + objectSummary.setLastModified(new Date(0)); + + final ObjectListing listObjectsResult = new ObjectListing(); + listObjectsResult.getObjectSummaries().add(objectSummary); + + File tmpDir = temporaryFolder.newFolder("gzTestDir"); + + OSSException exception = new OSSException("OssDataSegmentPullerTest","NoSuchKey",null,null,null,null,null); + EasyMock.expect(s3Client.doesObjectExist(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) + .andReturn(true) + .once(); + EasyMock.expect(s3Client.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) + .andReturn(listObjectsResult) + .once(); + EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))) + .andThrow(exception) + .once(); + EasyMock.expect(s3Client.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) + .andReturn(listObjectsResult) + .once(); + EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))) + .andReturn(object0) + .once(); + OssDataSegmentPuller puller = new OssDataSegmentPuller(s3Client); + + EasyMock.replay(s3Client); + FileUtils.FileCopyResult result = puller.getSegmentFiles( + new CloudObjectLocation( + bucket, + object0.getKey() + ), tmpDir + ); + EasyMock.verify(s3Client); + + Assert.assertEquals(value.length, result.size()); + File expected = new File(tmpDir, "renames-0"); + Assert.assertTrue(expected.exists()); + Assert.assertEquals(value.length, expected.length()); + } + +} diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java new file mode 100644 index 000000000000..09fd47f40094 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.io.IOException; +import java.util.Locale; +import java.util.Set; + +import javax.validation.ConstraintViolation; +import javax.validation.Validation; +import javax.validation.Validator; + +import org.apache.druid.jackson.DefaultObjectMapper; +import org.junit.Assert; +import org.junit.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Iterators; + +public class OssDataSegmentPusherConfigTest +{ + private static final ObjectMapper JSON_MAPPER = new DefaultObjectMapper(); + + @Test + public void testSerialization() throws IOException + { + String jsonConfig = "{\"bucket\":\"bucket1\",\"baseKey\":\"dataSource1\"," + + "\"disableAcl\":false,\"maxListingLength\":2000}"; + + OssDataSegmentPusherConfig config = JSON_MAPPER.readValue(jsonConfig, OssDataSegmentPusherConfig.class); + Assert.assertEquals(jsonConfig, JSON_MAPPER.writeValueAsString(config)); + } + + @Test + public void testSerializationWithDefaults() throws IOException + { + String jsonConfig = "{\"bucket\":\"bucket1\",\"baseKey\":\"dataSource1\"}"; + String expectedJsonConfig = "{\"bucket\":\"bucket1\",\"baseKey\":\"dataSource1\"," + + "\"disableAcl\":true,\"maxListingLength\":1024}"; + + OssDataSegmentPusherConfig config = JSON_MAPPER.readValue(jsonConfig, OssDataSegmentPusherConfig.class); + Assert.assertEquals(expectedJsonConfig, JSON_MAPPER.writeValueAsString(config)); + } + + @Test + public void testSerializationValidatingMaxListingLength() throws IOException + { + Locale old = Locale.getDefault(); + Locale.setDefault(Locale.ENGLISH); + try { + String jsonConfig = "{\"bucket\":\"bucket1\",\"baseKey\":\"dataSource1\"," + + "\"disableAcl\":false,\"maxListingLength\":-1}"; + Validator validator = Validation.buildDefaultValidatorFactory().getValidator(); + + OssDataSegmentPusherConfig config = JSON_MAPPER.readValue(jsonConfig, OssDataSegmentPusherConfig.class); + Set> violations = validator.validate(config); + Assert.assertEquals(1, violations.size()); + ConstraintViolation violation = Iterators.getOnlyElement(violations.iterator()); + Assert.assertEquals("must be greater than or equal to 1", violation.getMessage()); + }finally { + Locale.setDefault(old); + } + } +} diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java new file mode 100644 index 000000000000..00a1f2e15e98 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.regex.Pattern; + +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.NoneShardSpec; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.model.PutObjectResult; +import com.google.common.io.Files; + +/** + */ +public class OssDataSegmentPusherTest +{ + private static class ValueContainer + { + private T value; + + public T getValue() + { + return value; + } + + public void setValue(T value) + { + this.value = value; + } + } + + @Rule + public final TemporaryFolder tempFolder = new TemporaryFolder(); + + @Test + public void testPush() throws Exception + { + testPushInternal(false, "key/foo/2015-01-01T00:00:00\\.000Z_2016-01-01T00:00:00\\.000Z/0/0/index\\.zip"); + } + + @Test + public void testPushUseUniquePath() throws Exception + { + testPushInternal(true, "key/foo/2015-01-01T00:00:00\\.000Z_2016-01-01T00:00:00\\.000Z/0/0/[A-Za-z0-9-]{36}/index\\.zip"); + } + + private void testPushInternal(boolean useUniquePath, String matcher) throws Exception + { + OSS client = EasyMock.createStrictMock(OSS.class); + + EasyMock.expect(client.putObject(EasyMock.anyObject())) + .andReturn(new PutObjectResult()) + .once(); + + EasyMock.replay(client); + + OssDataSegmentPusherConfig config = new OssDataSegmentPusherConfig(); + config.setBucket("bucket"); + config.setBaseKey("key"); + + OssDataSegmentPusher pusher = new OssDataSegmentPusher(client, config); + + // Create a mock segment on disk + File tmp = tempFolder.newFile("version.bin"); + + final byte[] data = new byte[]{0x0, 0x0, 0x0, 0x1}; + Files.write(data, tmp); + final long size = data.length; + + DataSegment segmentToPush = new DataSegment( + "foo", + Intervals.of("2015/2016"), + "0", + new HashMap<>(), + new ArrayList<>(), + new ArrayList<>(), + NoneShardSpec.instance(), + 0, + size + ); + + DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush, useUniquePath); + + Assert.assertEquals(segmentToPush.getSize(), segment.getSize()); + Assert.assertEquals(1, (int) segment.getBinaryVersion()); + Assert.assertEquals("bucket", segment.getLoadSpec().get("bucket")); + Assert.assertTrue( + segment.getLoadSpec().get("key").toString(), + Pattern.compile(matcher).matcher(segment.getLoadSpec().get("key").toString()).matches() + ); + Assert.assertEquals("oss_zip", segment.getLoadSpec().get("type")); + + EasyMock.verify(client); + } +} diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java new file mode 100644 index 000000000000..90850ee27879 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java @@ -0,0 +1,358 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.druid.common.utils.CurrentTimeMillisSupplier; +import org.apache.druid.java.util.common.StringUtils; +import org.easymock.EasyMock; +import org.easymock.EasyMockRunner; +import org.easymock.EasyMockSupport; +import org.easymock.Mock; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +import com.aliyun.oss.ClientException; +import com.aliyun.oss.OSS; +import com.aliyun.oss.model.AccessControlList; +import com.aliyun.oss.model.DeleteObjectsRequest; +import com.aliyun.oss.model.Grant; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.Owner; +import com.aliyun.oss.model.Permission; +import com.aliyun.oss.model.PutObjectRequest; +import com.aliyun.oss.model.PutObjectResult; +import com.aliyun.oss.model.SetObjectAclRequest; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +@RunWith(EasyMockRunner.class) +public class OssTaskLogsTest extends EasyMockSupport +{ + + private static final String KEY_1 = "key1"; + private static final String KEY_2 = "key2"; + private static final String TEST_BUCKET = "test_bucket"; + private static final String TEST_PREFIX = "test_prefix"; + private static final URI PREFIX_URI = URI.create(StringUtils.format("s3://%s/%s", TEST_BUCKET, TEST_PREFIX)); + private static final long TIME_0 = 0L; + private static final long TIME_1 = 1L; + private static final long TIME_NOW = 2L; + private static final long TIME_FUTURE = 3L; + private static final int MAX_KEYS = 1; + private static final Exception RECOVERABLE_EXCEPTION = new ClientException(new IOException()); + private static final Exception NON_RECOVERABLE_EXCEPTION = new ClientException(new NullPointerException()); + + @Mock + private CurrentTimeMillisSupplier timeSupplier; + @Mock + private OSS ossClient; + + @Rule + public final TemporaryFolder tempFolder = new TemporaryFolder(); + + @Test + public void testTaskLogsPushWithAclDisabled() throws Exception + { + String ownerId = "test_owner"; + String ownerDisplayName = "test_owner"; + + List grantList = testPushInternal(true, ownerId, ownerDisplayName); + + Assert.assertNotNull("Grant list should not be null", grantList); + Assert.assertEquals("Grant list should be empty as ACL is disabled", 0, grantList.size()); + } + +// @Test +// public void testTaskLogsPushWithAclEnabled() throws Exception +// { +// String ownerId = "test_owner"; +// String ownerDisplayName = "test_owner"; +// +// List grantList = testPushInternal(false, ownerId, ownerDisplayName); +// +// Assert.assertNotNull("Grant list should not be null", grantList); +// Assert.assertEquals("Grant list size should be equal to 1", 1, grantList.size()); +// Grant grant = grantList.get(0); +// Assert.assertEquals( +// "The Grantee identifier should be test_owner", +// "test_owner", +// grant.getGrantee().getIdentifier() +// ); +// Assert.assertEquals("The Grant should have full control permission", Permission.FullControl, grant.getPermission()); +// } + + @Test + public void test_killAll_noException_deletesAllTaskLogs() throws IOException + { + OSSObjectSummary objectSummary1 = OssTestUtils.newOSSObjectSummary(TEST_BUCKET, KEY_1, TIME_0); + OSSObjectSummary objectSummary2 = OssTestUtils.newOSSObjectSummary(TEST_BUCKET, KEY_2, TIME_1); + + EasyMock.expect(timeSupplier.getAsLong()).andReturn(TIME_NOW); + + OssTestUtils.expectListObjects( + ossClient, + PREFIX_URI, + ImmutableList.of(objectSummary1, objectSummary2) + ); + + DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET); + deleteRequest1.setKeys(Arrays.asList(KEY_1)); + DeleteObjectsRequest deleteRequest2 = new DeleteObjectsRequest(TEST_BUCKET); + deleteRequest2.setKeys(Arrays.asList(KEY_2)); + + OssTestUtils.mockClientDeleteObjects( + ossClient, + ImmutableList.of(deleteRequest1, deleteRequest2), + ImmutableMap.of() + ); + + EasyMock.replay(ossClient, timeSupplier); + + OssTaskLogsConfig config = new OssTaskLogsConfig(); + config.setBucket(TEST_BUCKET); + config.setPrefix(TEST_PREFIX); + OssInputDataConfig inputDataConfig = new OssInputDataConfig(); + inputDataConfig.setMaxListingLength(MAX_KEYS); + OssTaskLogs OssTaskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); + OssTaskLogs.killAll(); + + EasyMock.verify(ossClient, timeSupplier); + } + + @Test + public void test_killAll_recoverableExceptionWhenDeletingObjects_deletesAllTaskLogs() throws IOException + { + OSSObjectSummary objectSummary1 = OssTestUtils.newOSSObjectSummary(TEST_BUCKET, KEY_1, TIME_0); + + EasyMock.expect(timeSupplier.getAsLong()).andReturn(TIME_NOW); + + OssTestUtils.expectListObjects( + ossClient, + PREFIX_URI, + ImmutableList.of(objectSummary1) + ); + + DeleteObjectsRequest expectedRequest = new DeleteObjectsRequest(TEST_BUCKET); + expectedRequest.setKeys(Arrays.asList(KEY_1)); + OssTestUtils.mockClientDeleteObjects( + ossClient, + ImmutableList.of(expectedRequest), + ImmutableMap.of(expectedRequest, RECOVERABLE_EXCEPTION) + ); + + EasyMock.replay(ossClient, timeSupplier); + + OssTaskLogsConfig config = new OssTaskLogsConfig(); + config.setBucket(TEST_BUCKET); + config.setPrefix(TEST_PREFIX); + OssInputDataConfig inputDataConfig = new OssInputDataConfig(); + inputDataConfig.setMaxListingLength(MAX_KEYS); + OssTaskLogs OssTaskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); + OssTaskLogs.killAll(); + + EasyMock.verify(ossClient, timeSupplier); + } + + @Test + public void test_killAll_nonrecoverableExceptionWhenListingObjects_doesntDeleteAnyTaskLogs() + { + boolean ioExceptionThrown = false; + try { + OSSObjectSummary objectSummary1 = OssTestUtils.newOSSObjectSummary(TEST_BUCKET, KEY_1, TIME_0); + EasyMock.expect(timeSupplier.getAsLong()).andReturn(TIME_NOW); + OssTestUtils.expectListObjects( + ossClient, + PREFIX_URI, + ImmutableList.of(objectSummary1) + ); + + DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET); + deleteRequest1.setKeys(Arrays.asList(KEY_1)); + OssTestUtils.mockClientDeleteObjects( + ossClient, + ImmutableList.of(), + ImmutableMap.of(deleteRequest1, NON_RECOVERABLE_EXCEPTION) + ); + + EasyMock.replay(ossClient, timeSupplier); + + OssTaskLogsConfig config = new OssTaskLogsConfig(); + config.setBucket(TEST_BUCKET); + config.setPrefix(TEST_PREFIX); + OssInputDataConfig inputDataConfig = new OssInputDataConfig(); + inputDataConfig.setMaxListingLength(MAX_KEYS); + OssTaskLogs OssTaskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); + OssTaskLogs.killAll(); + } + catch (IOException e) { + ioExceptionThrown = true; + } + + Assert.assertTrue(ioExceptionThrown); + + EasyMock.verify(ossClient, timeSupplier); + } + + @Test + public void test_killOlderThan_noException_deletesOnlyTaskLogsOlderThan() throws IOException + { + OSSObjectSummary objectSummary1 = OssTestUtils.newOSSObjectSummary(TEST_BUCKET, KEY_1, TIME_0); + OSSObjectSummary objectSummary2 = OssTestUtils.newOSSObjectSummary(TEST_BUCKET, KEY_2, TIME_FUTURE); + + OssTestUtils.expectListObjects( + ossClient, + PREFIX_URI, + ImmutableList.of(objectSummary1, objectSummary2) + ); + + DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET); + deleteRequest1.setKeys(Arrays.asList(KEY_1)); + + OssTestUtils.mockClientDeleteObjects(ossClient, ImmutableList.of(deleteRequest1), ImmutableMap.of()); + + EasyMock.replay(ossClient, timeSupplier); + + OssTaskLogsConfig config = new OssTaskLogsConfig(); + config.setBucket(TEST_BUCKET); + config.setPrefix(TEST_PREFIX); + OssInputDataConfig inputDataConfig = new OssInputDataConfig(); + inputDataConfig.setMaxListingLength(MAX_KEYS); + OssTaskLogs OssTaskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); + OssTaskLogs.killOlderThan(TIME_NOW); + + EasyMock.verify(ossClient, timeSupplier); + } + + @Test + public void test_killOlderThan_recoverableExceptionWhenListingObjects_deletesAllTaskLogs() throws IOException + { + OSSObjectSummary objectSummary1 = OssTestUtils.newOSSObjectSummary(TEST_BUCKET, KEY_1, TIME_0); + + OssTestUtils.expectListObjects( + ossClient, + PREFIX_URI, + ImmutableList.of(objectSummary1) + ); + + DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET); + deleteRequest1.setKeys(Arrays.asList(KEY_1)); + + OssTestUtils.mockClientDeleteObjects( + ossClient, + ImmutableList.of(deleteRequest1), + ImmutableMap.of(deleteRequest1, RECOVERABLE_EXCEPTION) + ); + + EasyMock.replay(ossClient, timeSupplier); + + OssTaskLogsConfig config = new OssTaskLogsConfig(); + config.setBucket(TEST_BUCKET); + config.setPrefix(TEST_PREFIX); + OssInputDataConfig inputDataConfig = new OssInputDataConfig(); + inputDataConfig.setMaxListingLength(MAX_KEYS); + OssTaskLogs OssTaskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); + OssTaskLogs.killOlderThan(TIME_NOW); + + EasyMock.verify(ossClient, timeSupplier); + } + + @Test + public void test_killOlderThan_nonrecoverableExceptionWhenListingObjects_doesntDeleteAnyTaskLogs() + { + boolean ioExceptionThrown = false; + try { + OSSObjectSummary objectSummary1 = OssTestUtils.newOSSObjectSummary(TEST_BUCKET, KEY_1, TIME_0); + OssTestUtils.expectListObjects( + ossClient, + PREFIX_URI, + ImmutableList.of(objectSummary1) + ); + + DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET); + deleteRequest1.setKeys(Arrays.asList(KEY_1)); + OssTestUtils.mockClientDeleteObjects( + ossClient, + ImmutableList.of(), + ImmutableMap.of(deleteRequest1, NON_RECOVERABLE_EXCEPTION) + ); + + EasyMock.replay(ossClient, timeSupplier); + + OssTaskLogsConfig config = new OssTaskLogsConfig(); + config.setBucket(TEST_BUCKET); + config.setPrefix(TEST_PREFIX); + OssInputDataConfig inputDataConfig = new OssInputDataConfig(); + inputDataConfig.setMaxListingLength(MAX_KEYS); + OssTaskLogs OssTaskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); + OssTaskLogs.killOlderThan(TIME_NOW); + } + catch (IOException e) { + ioExceptionThrown = true; + } + + Assert.assertTrue(ioExceptionThrown); + + EasyMock.verify(ossClient, timeSupplier); + } + + private List testPushInternal(boolean disableAcl, String ownerId, String ownerDisplayName) throws Exception + { + EasyMock.expect(ossClient.putObject(EasyMock.anyObject())) + .andReturn(new PutObjectResult()) + .once(); + + AccessControlList aclExpected = new AccessControlList(); + aclExpected.setOwner(new Owner(ownerId, ownerDisplayName)); + + EasyMock.expect(ossClient.getBucketAcl(TEST_BUCKET)) + .andReturn(aclExpected) + .once(); + + EasyMock.expect(ossClient.putObject(EasyMock.anyObject(PutObjectRequest.class))) + .andReturn(new PutObjectResult()) + .once(); + + EasyMock.replay(ossClient); + + OssTaskLogsConfig config = new OssTaskLogsConfig(); + config.setDisableAcl(disableAcl); + config.setBucket(TEST_BUCKET); + CurrentTimeMillisSupplier timeSupplier = new CurrentTimeMillisSupplier(); + OssInputDataConfig inputDataConfig = new OssInputDataConfig(); + OssTaskLogs OssTaskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); + + String taskId = "index_test-datasource_2019-06-18T13:30:28.887Z"; + File logFile = tempFolder.newFile("test_log_file"); + + OssTaskLogs.pushTaskLog(taskId, logFile); + + return aclExpected.getGrants().stream().collect(Collectors.toList()); + } +} diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTestUtils.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTestUtils.java new file mode 100644 index 000000000000..a67230103575 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTestUtils.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.net.URI; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.StringUtils; +import org.easymock.EasyMock; +import org.easymock.EasyMockSupport; +import org.easymock.IArgumentMatcher; +import org.easymock.IExpectationSetters; +import org.joda.time.DateTime; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.model.DeleteObjectsRequest; +import com.aliyun.oss.model.DeleteObjectsResult; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.ObjectListing; + +public class OssTestUtils extends EasyMockSupport +{ + private static final DateTime NOW = DateTimes.nowUtc(); + private static final byte[] CONTENT = + StringUtils.toUtf8(StringUtils.format("%d,hello,world", NOW.getMillis())); + + public static DeleteObjectsRequest deleteObjectsRequestArgumentMatcher(DeleteObjectsRequest deleteObjectsRequest) + { + EasyMock.reportMatcher(new IArgumentMatcher() + { + @Override + public boolean matches(Object argument) + { + + boolean matches = argument instanceof DeleteObjectsRequest + && deleteObjectsRequest.getBucketName() + .equals(((DeleteObjectsRequest) argument).getBucketName()) + && deleteObjectsRequest.getKeys().size() == ((DeleteObjectsRequest) argument).getKeys() + .size(); + if (matches) { + List expectedKeysAndVersions = deleteObjectsRequest.getKeys(); + List actualKeysAndVersions = ((DeleteObjectsRequest) argument).getKeys(); + matches = expectedKeysAndVersions.equals(actualKeysAndVersions); + } + return matches; + } + + @Override + public void appendTo(StringBuffer buffer) + { + String str = "DeleteObjectsRequest(\"bucketName:\" \"" + + deleteObjectsRequest.getBucketName() + + "\", \"keys:\"" + + deleteObjectsRequest.getKeys() + + "\")"; + buffer.append(str); + } + }); + return null; + } + + public static void expectListObjects( + OSS client, + URI prefix, + List objectSummaries) + { + final ObjectListing result = new ObjectListing(); + result.setBucketName(prefix.getAuthority()); + //result.setsetKeyCount(objectSummaries.size()); + for (OSSObjectSummary objectSummary : objectSummaries) { + result.getObjectSummaries().add(objectSummary); + } + + EasyMock.expect( + client.listObjects(matchListObjectsRequest(prefix)) + ).andReturn(result).once(); + } + + public static void mockClientDeleteObjects( + OSS client, + List deleteRequestsExpected, + Map requestToException + ) + { + Map> requestToResultExpectationSetter = new HashMap<>(); + + for (Map.Entry requestsAndErrors : requestToException.entrySet()) { + DeleteObjectsRequest request = requestsAndErrors.getKey(); + Exception exception = requestsAndErrors.getValue(); + IExpectationSetters resultExpectationSetter = requestToResultExpectationSetter.get(request); + if (resultExpectationSetter == null) { + client.deleteObjects( + OssTestUtils.deleteObjectsRequestArgumentMatcher(request)); + resultExpectationSetter = EasyMock.expectLastCall().andThrow(exception); + requestToResultExpectationSetter.put(request, resultExpectationSetter); + } else { + resultExpectationSetter.andThrow(exception); + } + } + + for (DeleteObjectsRequest request : deleteRequestsExpected) { + IExpectationSetters resultExpectationSetter = requestToResultExpectationSetter.get(request); + if (resultExpectationSetter == null) { + client.deleteObjects(OssTestUtils.deleteObjectsRequestArgumentMatcher(request)); + resultExpectationSetter = EasyMock.expectLastCall(); + requestToResultExpectationSetter.put(request, resultExpectationSetter); + } + resultExpectationSetter.andReturn(new DeleteObjectsResult()); + } + } + + public static ListObjectsRequest matchListObjectsRequest(final URI prefixUri) + { + // Use an IArgumentMatcher to verify that the request has the correct bucket and prefix. + EasyMock.reportMatcher( + new IArgumentMatcher() + { + @Override + public boolean matches(Object argument) + { + if (!(argument instanceof ListObjectsRequest)) { + return false; + } + + final ListObjectsRequest request = (ListObjectsRequest) argument; + return prefixUri.getAuthority().equals(request.getBucketName()) + && OssUtils.extractKey(prefixUri).equals(request.getPrefix()); + } + + @Override + public void appendTo(StringBuffer buffer) + { + buffer.append(""); + } + } + ); + + return null; + } + + public static OSSObjectSummary newOSSObjectSummary( + String bucket, + String key, + long lastModifiedTimestamp) + { + OSSObjectSummary objectSummary = new OSSObjectSummary(); + objectSummary.setBucketName(bucket); + objectSummary.setKey(key); + objectSummary.setLastModified(new Date(lastModifiedTimestamp)); + objectSummary.setETag("etag"); + objectSummary.setSize(CONTENT.length); + return objectSummary; + } +} diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java new file mode 100644 index 000000000000..8c9f5a325c91 --- /dev/null +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.aliyun; + +import java.net.URI; +import java.util.Date; +import java.util.regex.Pattern; + +import org.apache.druid.java.util.common.StringUtils; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Test; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.ObjectListing; + +public class OssTimestampVersionedDataFinderTest +{ + + @Test + public void testSimpleLatestVersion() + { + String bucket = "bucket"; + String keyPrefix = "prefix/dir/0"; + OSS client = EasyMock.createStrictMock(OSS.class); + + OSSObjectSummary object0 = new OSSObjectSummary(), object1 = new OSSObjectSummary(); + + object0.setBucketName(bucket); + object0.setKey(keyPrefix + "/renames-0.gz"); + object0.setLastModified(new Date(0)); + object0.setSize(10); + + object1.setBucketName(bucket); + object1.setKey(keyPrefix + "/renames-1.gz"); + object1.setLastModified(new Date(1)); + object1.setSize(10); + + final ObjectListing result = new ObjectListing(); + result.getObjectSummaries().add(object0); + result.getObjectSummaries().add(object1); + result.setTruncated(false); + + EasyMock.expect(client.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) + .andReturn(result) + .once(); + OssTimestampVersionedDataFinder finder = new OssTimestampVersionedDataFinder(client); + + Pattern pattern = Pattern.compile("renames-[0-9]*\\.gz"); + + EasyMock.replay(client); + + + URI latest = finder.getLatestVersion(URI.create(StringUtils.format("oss://%s/%s", bucket, keyPrefix)), pattern); + + EasyMock.verify(client); + + URI expected = URI.create(StringUtils.format("oss://%s/%s", bucket, object1.getKey())); + + Assert.assertEquals(expected, latest); + } + + @Test + public void testMissing() + { + String bucket = "bucket"; + String keyPrefix = "prefix/dir/0"; + OSS oss = EasyMock.createStrictMock(OSS.class); + + final ObjectListing result = new ObjectListing(); + result.setTruncated(false); + + EasyMock.expect(oss.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) + .andReturn(result) + .once(); + OssTimestampVersionedDataFinder finder = new OssTimestampVersionedDataFinder(oss); + + Pattern pattern = Pattern.compile("renames-[0-9]*\\.gz"); + + EasyMock.replay(oss); + + + URI latest = finder.getLatestVersion(URI.create(StringUtils.format("oss://%s/%s", bucket, keyPrefix)), pattern); + + EasyMock.verify(oss); + + Assert.assertEquals(null, latest); + } + + @Test + public void testFindSelf() + { + String bucket = "bucket"; + String keyPrefix = "prefix/dir/0"; + OSS s3Client = EasyMock.createStrictMock(OSS.class); + + OSSObjectSummary object0 = new OSSObjectSummary(); + + object0.setBucketName(bucket); + object0.setKey(keyPrefix + "/renames-0.gz"); + object0.setLastModified(new Date(0)); + object0.setSize(10); + + final ObjectListing result = new ObjectListing(); + result.getObjectSummaries().add(object0); + result.setTruncated(false); + + EasyMock.expect(s3Client.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) + .andReturn(result) + .once(); + OssTimestampVersionedDataFinder finder = new OssTimestampVersionedDataFinder(s3Client); + + Pattern pattern = Pattern.compile("renames-[0-9]*\\.gz"); + + EasyMock.replay(s3Client); + + + URI latest = finder.getLatestVersion(URI.create(StringUtils.format("oss://%s/%s", bucket, keyPrefix)), pattern); + + EasyMock.verify(s3Client); + + URI expected = URI.create(StringUtils.format("oss://%s/%s", bucket, object0.getKey())); + + Assert.assertEquals(expected, latest); + } + + @Test + public void testFindExact() + { + String bucket = "bucket"; + String keyPrefix = "prefix/dir/0"; + OSS s3Client = EasyMock.createStrictMock(OSS.class); + + OSSObjectSummary object0 = new OSSObjectSummary(); + + object0.setBucketName(bucket); + object0.setKey(keyPrefix + "/renames-0.gz"); + object0.setLastModified(new Date(0)); + object0.setSize(10); + + final ObjectListing result = new ObjectListing(); + result.getObjectSummaries().add(object0); + result.setTruncated(false); + + EasyMock.expect(s3Client.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) + .andReturn(result) + .once(); + OssTimestampVersionedDataFinder finder = new OssTimestampVersionedDataFinder(s3Client); + + EasyMock.replay(s3Client); + + URI latest = finder.getLatestVersion(URI.create(StringUtils.format("oss://%s/%s", bucket, object0.getKey())), null); + + EasyMock.verify(s3Client); + + URI expected = URI.create(StringUtils.format("oss://%s/%s", bucket, object0.getKey())); + + Assert.assertEquals(expected, latest); + } +} From baa94bc2329d66ec3d3e217a51f5c12e52298fc2 Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 15 May 2020 23:20:01 +0800 Subject: [PATCH 02/24] fix format Signed-off-by: frank chen --- .../druid/data/input/aliyun/OssEntity.java | 19 ++- .../data/input/aliyun/OssInputSource.java | 74 +++++----- .../input/aliyun/OssInputSourceConfig.java | 23 ++-- .../aliyun/OssInputSourceDruidModule.java | 9 +- .../aliyun/OssFirehoseDruidModule.java | 1 + .../aliyun/StaticOssFirehoseFactory.java | 53 ++++--- .../storage/aliyun/ObjectSummaryIterator.java | 17 ++- .../druid/storage/aliyun/OssClientHelper.java | 130 ------------------ .../aliyun/OssDataSegmentArchiver.java | 11 +- .../storage/aliyun/OssDataSegmentKiller.java | 13 +- .../storage/aliyun/OssDataSegmentMover.java | 29 ++-- .../storage/aliyun/OssDataSegmentPuller.java | 43 +++--- .../storage/aliyun/OssDataSegmentPusher.java | 23 ++-- .../aliyun/OssDataSegmentPusherConfig.java | 2 + .../storage/aliyun/OssStorageDruidModule.java | 38 +++-- .../druid/storage/aliyun/OssTaskLogs.java | 25 ++-- .../storage/aliyun/OssTaskLogsConfig.java | 1 + .../OssTimestampVersionedDataFinder.java | 19 ++- .../apache/druid/storage/aliyun/OssUtils.java | 50 +++---- .../aliyun/ObjectSummaryIteratorTest.java | 57 +++++--- .../aliyun/OssDataSegmentArchiverTest.java | 47 +++++-- .../aliyun/OssDataSegmentKillerTest.java | 19 ++- .../aliyun/OssDataSegmentMoverTest.java | 50 ++++--- .../aliyun/OssDataSegmentPullerTest.java | 33 +++-- .../OssDataSegmentPusherConfigTest.java | 49 ++++--- .../aliyun/OssDataSegmentPusherTest.java | 21 +-- .../druid/storage/aliyun/OssTaskLogsTest.java | 90 +++++------- .../druid/storage/aliyun/OssTestUtils.java | 29 ++-- .../OssTimestampVersionedDataFinderTest.java | 15 +- 29 files changed, 448 insertions(+), 542 deletions(-) delete mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssClientHelper.java diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java index 3c9206c441e5..aacaec3d3eb1 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java @@ -19,21 +19,20 @@ package org.apache.druid.data.input.aliyun; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; - +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.GetObjectRequest; +import com.aliyun.oss.model.OSSObject; +import com.google.common.base.Predicate; import org.apache.druid.data.input.RetryingInputEntity; import org.apache.druid.data.input.impl.CloudObjectLocation; import org.apache.druid.java.util.common.ISE; import org.apache.druid.storage.aliyun.OssStorageDruidModule; import org.apache.druid.storage.aliyun.OssUtils; -import com.aliyun.oss.OSS; -import com.aliyun.oss.OSSException; -import com.aliyun.oss.model.GetObjectRequest; -import com.aliyun.oss.model.OSSObject; -import com.google.common.base.Predicate; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; public class OssEntity extends RetryingInputEntity { @@ -57,7 +56,7 @@ protected InputStream readFrom(long offset) throws IOException { final GetObjectRequest request = new GetObjectRequest(object.getBucket(), object.getPath()); request.setRange(offset, request.getRange()[1]); - + try { final OSSObject ossObject = ossClient.getObject(request); if (ossObject == null) { diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java index 8419c09989f1..355110cea7f1 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java @@ -19,16 +19,15 @@ package org.apache.druid.data.input.aliyun; -import java.net.URI; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.model.OSSObjectSummary; +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import org.apache.druid.data.input.InputEntity; import org.apache.druid.data.input.InputFileAttribute; import org.apache.druid.data.input.InputSplit; @@ -41,15 +40,14 @@ import org.apache.druid.storage.aliyun.OssUtils; import org.apache.druid.utils.Streams; -import com.aliyun.oss.OSS; -import com.aliyun.oss.OSSClientBuilder; -import com.aliyun.oss.model.OSSObjectSummary; -import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Preconditions; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.net.URI; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class OssInputSource extends CloudObjectInputSource { @@ -63,19 +61,19 @@ public class OssInputSource extends CloudObjectInputSource /** * Constructor for S3InputSource - * @param client The default ServerSideEncryptingAmazonS3 client built with all default configs - * from Guice. This injected singleton client is use when {@param s3InputSourceConfig} - * is not provided and hence, we can skip building a new client from - * {@param s3ClientBuilder} - * @param clientBulider Use for building a new s3Client to use instead of the default injected - * {@param s3Client}. The configurations of the client can be changed - * before being built - * @param inputDataConfig Stores the configuration for options related to reading input data - * @param uris User provided uris to read input data - * @param prefixes User provided prefixes to read input data - * @param objects User provided cloud objects values to read input data - * @param inputSourceConfig User provided properties for overriding the default S3 configuration * + * @param client The default ServerSideEncryptingAmazonS3 client built with all default configs + * from Guice. This injected singleton client is use when {@param s3InputSourceConfig} + * is not provided and hence, we can skip building a new client from + * {@param s3ClientBuilder} + * @param clientBulider Use for building a new s3Client to use instead of the default injected + * {@param s3Client}. The configurations of the client can be changed + * before being built + * @param inputDataConfig Stores the configuration for options related to reading input data + * @param uris User provided uris to read input data + * @param prefixes User provided prefixes to read input data + * @param objects User provided cloud objects values to read input data + * @param inputSourceConfig User provided properties for overriding the default S3 configuration */ @JsonCreator public OssInputSource( @@ -94,9 +92,9 @@ public OssInputSource( this.clientSupplier = Suppliers.memoize( () -> { if (inputSourceConfig != null) { - String accessKeyId = inputSourceConfig.getAccessKeyId().getPassword(); - String secretAccessKey = inputSourceConfig.getSecretAccessKey().getPassword(); - return new OSSClientBuilder().build(inputSourceConfig.getEndpoint(), accessKeyId, secretAccessKey); + String accessKeyId = inputSourceConfig.getAccessKeyId().getPassword(); + String secretAccessKey = inputSourceConfig.getSecretAccessKey().getPassword(); + return new OSSClientBuilder().build(inputSourceConfig.getEndpoint(), accessKeyId, secretAccessKey); } else { return client; } @@ -105,7 +103,7 @@ public OssInputSource( } -@Nullable + @Nullable @JsonProperty("properties") public OssInputSourceConfig getOssInputSourceConfig() { @@ -181,6 +179,10 @@ public String toString() private Iterable getIterableObjectsFromPrefixes() { - return () -> OssUtils.objectSummaryIterator(clientSupplier.get(), getPrefixes(), inputDataConfig.getMaxListingLength()); + return () -> OssUtils.objectSummaryIterator( + clientSupplier.get(), + getPrefixes(), + inputDataConfig.getMaxListingLength() + ); } } diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java index bde023978a86..4ed95e85808a 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java @@ -36,17 +36,23 @@ public class OssInputSourceConfig { @JsonCreator public OssInputSourceConfig( - @JsonProperty("endpoint") String endpoint, + @JsonProperty("endpoint") String endpoint, @JsonProperty("accessKeyId") @Nullable PasswordProvider accessKeyId, @JsonProperty("secretAccessKey") @Nullable PasswordProvider secretAccessKey ) { if (accessKeyId != null || secretAccessKey != null) { - this.accessKeyId = Preconditions.checkNotNull(accessKeyId, "accessKeyId cannot be null if secretAccessKey is given"); - this.secretAccessKey = Preconditions.checkNotNull(secretAccessKey, "secretAccessKey cannot be null if accessKeyId is given"); + this.accessKeyId = Preconditions.checkNotNull( + accessKeyId, + "accessKeyId cannot be null if secretAccessKey is given" + ); + this.secretAccessKey = Preconditions.checkNotNull( + secretAccessKey, + "secretAccessKey cannot be null if accessKeyId is given" + ); } } - + @JsonProperty private String endpoint; @@ -56,8 +62,9 @@ public OssInputSourceConfig( @JsonProperty private PasswordProvider secretAccessKey; - public String getEndpoint() { - return endpoint; + public String getEndpoint() + { + return endpoint; } public PasswordProvider getAccessKeyId() @@ -81,7 +88,7 @@ public boolean isCredentialsConfigured() public String toString() { return "OssInputSourceConfig{" + - "endpoint=" + endpoint + + "endpoint=" + endpoint + "accessKeyId=" + accessKeyId + ", secretAccessKey=" + secretAccessKey + '}'; @@ -99,7 +106,7 @@ public boolean equals(Object o) OssInputSourceConfig that = (OssInputSourceConfig) o; return Objects.equals(accessKeyId, that.accessKeyId) && Objects.equals(secretAccessKey, that.secretAccessKey) && - Objects.equals(endpoint, that.endpoint); + Objects.equals(endpoint, that.endpoint); } @Override diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java index 876e44bcbf42..a1a9bf29fc74 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java @@ -19,16 +19,15 @@ package org.apache.druid.data.input.aliyun; -import java.util.List; - -import org.apache.druid.initialization.DruidModule; -import org.apache.druid.storage.aliyun.OssStorageDruidModule; - import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.jsontype.NamedType; import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; +import org.apache.druid.initialization.DruidModule; +import org.apache.druid.storage.aliyun.OssStorageDruidModule; + +import java.util.List; /** * Druid module to wire up native batch support for S3 input diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java index 8963f851a130..1382f17153b2 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java @@ -29,6 +29,7 @@ import java.util.List; /** + * */ public class OssFirehoseDruidModule implements DruidModule { diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java index e43066729f4f..c6e9ffda973f 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java @@ -19,27 +19,6 @@ package org.apache.druid.firehose.aliyun; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -import org.apache.druid.data.input.FiniteFirehoseFactory; -import org.apache.druid.data.input.InputSplit; -import org.apache.druid.data.input.impl.StringInputRowParser; -import org.apache.druid.data.input.impl.prefetch.PrefetchableTextFilesFirehoseFactory; -import org.apache.druid.java.util.common.IAE; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.storage.aliyun.OssUtils; -import org.apache.druid.utils.CompressionUtils; - import com.aliyun.oss.OSS; import com.aliyun.oss.OSSException; import com.aliyun.oss.model.GetObjectRequest; @@ -50,6 +29,26 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; +import org.apache.druid.data.input.FiniteFirehoseFactory; +import org.apache.druid.data.input.InputSplit; +import org.apache.druid.data.input.impl.StringInputRowParser; +import org.apache.druid.data.input.impl.prefetch.PrefetchableTextFilesFirehoseFactory; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.storage.aliyun.OssUtils; +import org.apache.druid.utils.CompressionUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; /** * Builds firehoses that read from a predefined list of S3 objects and then dry up. @@ -137,11 +136,11 @@ protected InputStream openObjectStream(URI object) throws IOException final String bucket = object.getAuthority(); final String key = OssUtils.extractKey(object); - final OSSObject OSSObject = client.getObject(bucket, key); - if (OSSObject == null) { + final OSSObject ossObject = client.getObject(bucket, key); + if (ossObject == null) { throw new ISE("Failed to get an s3 object for bucket[%s] and key[%s]", bucket, key); } - return OSSObject.getObjectContent(); + return ossObject.getObjectContent(); } catch (OSSException e) { throw new IOException(e); @@ -156,8 +155,8 @@ protected InputStream openObjectStream(URI object, long start) throws IOExceptio final GetObjectRequest request = new GetObjectRequest(bucket, key); try { - final OSSObject OSSObject = client.getObject(request); - if (OSSObject == null) { + final OSSObject ossObject = client.getObject(request); + if (ossObject == null) { throw new ISE( "Failed to get an s3 object for bucket[%s], key[%s], and start[%d]", bucket, @@ -165,7 +164,7 @@ protected InputStream openObjectStream(URI object, long start) throws IOExceptio start ); } - InputStream is = OSSObject.getObjectContent(); + InputStream is = ossObject.getObjectContent(); is.skip(start); return is; } diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java index 4a28011db541..8c5ae866c1cf 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java @@ -19,21 +19,20 @@ package org.apache.druid.storage.aliyun; -import java.net.URI; -import java.util.Iterator; -import java.util.NoSuchElementException; - -import org.apache.druid.java.util.common.RE; - import com.aliyun.oss.OSS; import com.aliyun.oss.OSSException; import com.aliyun.oss.model.ListObjectsRequest; import com.aliyun.oss.model.OSSObjectSummary; import com.aliyun.oss.model.ObjectListing; +import org.apache.druid.java.util.common.RE; + +import java.net.URI; +import java.util.Iterator; +import java.util.NoSuchElementException; /** * Iterator class used by {@link OssUtils#objectSummaryIterator}. - * + *

* As required by the specification of that method, this iterator is computed incrementally in batches of * {@code maxListLength}. The first call is made at the same time the iterator is constructed. */ @@ -87,7 +86,7 @@ private void prepareNextRequest() final String currentBucket = currentUri.getAuthority(); final String currentPrefix = OssUtils.extractKey(currentUri); - request = new ListObjectsRequest(currentBucket,currentPrefix, null, null, maxListingLength); + request = new ListObjectsRequest(currentBucket, currentPrefix, null, null, maxListingLength); } private void fetchNextBatch() @@ -145,7 +144,7 @@ private void advanceObjectSummary() /** * Checks if a given object is a directory placeholder and should be ignored. - * + *

* Adapted from org.jets3t.service.model.StorageObject.isDirectoryPlaceholder(). Does not include the check for * legacy JetS3t directory placeholder objects, since it is based on content-type, which isn't available in an * S3ObjectSummary. diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssClientHelper.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssClientHelper.java deleted file mode 100644 index ba11fc6ea31d..000000000000 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssClientHelper.java +++ /dev/null @@ -1,130 +0,0 @@ -///* -// * Licensed to the Apache Software Foundation (ASF) under one -// * or more contributor license agreements. See the NOTICE file -// * distributed with this work for additional information -// * regarding copyright ownership. The ASF licenses this file -// * to you under the Apache License, Version 2.0 (the -// * "License"); you may not use this file except in compliance -// * with the License. You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, -// * software distributed under the License is distributed on an -// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// * KIND, either express or implied. See the License for the -// * specific language governing permissions and limitations -// * under the License. -// */ -// -//package org.apache.druid.storage.aliyun; -// -//import java.io.ByteArrayInputStream; -//import java.io.File; -//import java.io.InputStream; -// -//import org.apache.druid.java.util.common.StringUtils; -// -//import com.aliyun.oss.OSS; -//import com.aliyun.oss.OSSClientBuilder; -//import com.aliyun.oss.model.AccessControlList; -//import com.aliyun.oss.model.CopyObjectRequest; -//import com.aliyun.oss.model.CopyObjectResult; -//import com.aliyun.oss.model.DeleteObjectsRequest; -//import com.aliyun.oss.model.GenericRequest; -//import com.aliyun.oss.model.GetObjectRequest; -//import com.aliyun.oss.model.ListObjectsRequest; -//import com.aliyun.oss.model.OSSObject; -//import com.aliyun.oss.model.ObjectListing; -//import com.aliyun.oss.model.ObjectMetadata; -//import com.aliyun.oss.model.PutObjectRequest; -//import com.aliyun.oss.model.PutObjectResult; -// -///** -// * {@link AmazonS3} wrapper with {@link ServerSideEncryption}. Every {@link AmazonS3#putObject}, -// * {@link AmazonS3#copyObject}, {@link AmazonS3#getObject}, and {@link AmazonS3#getObjectMetadata} methods should be -// * wrapped using ServerSideEncryption. -// * -// * Additional methods can be added to this class if needed, but subclassing AmazonS3Client is discouraged to reduce -// * human mistakes like some methods are not encoded properly. -// */ -//public class OssClientHelper -//{ -// private final OSS ossClient; -// -// public OssClientHelper(OSS ossClient) -// { -// this.ossClient = ossClient; -// } -// -// public OssClientHelper(String endpoint, String accessKeyId, String secretAccessKey) { -// this.ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, secretAccessKey); -// } -// -//public boolean doesObjectExist(String bucket, String objectName) -// { -// return ossClient.doesObjectExist(bucket, objectName); -// } -// -// public ObjectListing listObjects(ListObjectsRequest request) -// { -// return ossClient.listObjects(request); -// } -// -// public AccessControlList getBucketAcl(String bucket) -// { -// return ossClient.getBucketAcl(bucket); -// } -// -// public ObjectMetadata getObjectMetadata(String bucket, String key) -// { -// GenericRequest request = new GenericRequest(bucket, key); -// return ossClient.getobjectm -// } -// -// public OSSObject getObject(String bucket, String key) -// { -// return getObject(new GetObjectRequest(bucket, key)); -// } -// -// public OSSObject getObject(GetObjectRequest request) -// { -// return ossClient.getObject(request); -// } -// -// public PutObjectResult putObject(String bucket, String key, String content) -// { -// final InputStream in = new ByteArrayInputStream(StringUtils.toUtf8(content)); -// return putObject(new PutObjectRequest(bucket, key, in, new ObjectMetadata())); -// } -// -// public PutObjectResult putObject(String bucket, String key, File file) -// { -// return putObject(new PutObjectRequest(bucket, key, file)); -// } -// -// public PutObjectResult putObject(String bucket, String key, InputStream in, ObjectMetadata objectMetadata) -// { -// return putObject(new PutObjectRequest(bucket, key, in, objectMetadata)); -// } -// -// public PutObjectResult putObject(PutObjectRequest request) -// { -// return ossClient.putObject(request); -// } -// -// public CopyObjectResult copyObject(CopyObjectRequest request) -// { -// return ossClient.copyObject(request); -// } -// -// public void deleteObject(String bucket, String key) -// { -// ossClient.deleteObject(bucket, key); -// } -// -// public void deleteObjects(DeleteObjectsRequest request) -// { -// ossClient.deleteObjects(request); -// } -//} diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java index f6c8ff66a54c..6b5c9b625635 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java @@ -19,17 +19,16 @@ package org.apache.druid.storage.aliyun; -import org.apache.druid.guice.annotations.Json; -import org.apache.druid.segment.loading.DataSegmentArchiver; -import org.apache.druid.segment.loading.LoadSpec; -import org.apache.druid.segment.loading.SegmentLoadingException; -import org.apache.druid.timeline.DataSegment; - import com.aliyun.oss.OSS; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Objects; import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; +import org.apache.druid.guice.annotations.Json; +import org.apache.druid.segment.loading.DataSegmentArchiver; +import org.apache.druid.segment.loading.LoadSpec; +import org.apache.druid.segment.loading.SegmentLoadingException; +import org.apache.druid.timeline.DataSegment; public class OssDataSegmentArchiver extends OssDataSegmentMover implements DataSegmentArchiver diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java index af465cd7e9db..d2abd3ce6176 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java @@ -19,9 +19,10 @@ package org.apache.druid.storage.aliyun; -import java.io.IOException; -import java.util.Map; - +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.google.common.base.Predicates; +import com.google.inject.Inject; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.MapUtils; import org.apache.druid.java.util.common.logger.Logger; @@ -29,10 +30,8 @@ import org.apache.druid.segment.loading.SegmentLoadingException; import org.apache.druid.timeline.DataSegment; -import com.aliyun.oss.OSS; -import com.aliyun.oss.OSSException; -import com.google.common.base.Predicates; -import com.google.inject.Inject; +import java.io.IOException; +import java.util.Map; /** * diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java index 2cd495c4f596..05f96a38ce8b 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java @@ -19,20 +19,6 @@ package org.apache.druid.storage.aliyun; -import java.io.IOException; -import java.util.Map; - -import org.apache.druid.java.util.common.IOE; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.MapUtils; -import org.apache.druid.java.util.common.RetryUtils; -import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.segment.loading.DataSegmentMover; -import org.apache.druid.segment.loading.DataSegmentPusher; -import org.apache.druid.segment.loading.SegmentLoadingException; -import org.apache.druid.timeline.DataSegment; - import com.aliyun.oss.OSS; import com.aliyun.oss.OSSException; import com.aliyun.oss.model.CopyObjectRequest; @@ -45,6 +31,19 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.inject.Inject; +import org.apache.druid.java.util.common.IOE; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.MapUtils; +import org.apache.druid.java.util.common.RetryUtils; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.segment.loading.DataSegmentMover; +import org.apache.druid.segment.loading.DataSegmentPusher; +import org.apache.druid.segment.loading.SegmentLoadingException; +import org.apache.druid.timeline.DataSegment; + +import java.io.IOException; +import java.util.Map; public class OssDataSegmentMover implements DataSegmentMover { @@ -168,7 +167,7 @@ private void selfCheckingMove( } if (client.doesObjectExist(srcBucket, srcPath)) { final ObjectListing listResult = client.listObjects( - new ListObjectsRequest(srcBucket,srcPath, null, null, 1) + new ListObjectsRequest(srcBucket, srcPath, null, null, 1) ); // Using getObjectSummaries().size() instead of getKeyCount as, in some cases // it is observed that even though the getObjectSummaries returns some data diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java index afa0751c466a..b4e1cfb94b1d 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java @@ -19,17 +19,15 @@ package org.apache.druid.storage.aliyun; -import java.io.File; -import java.io.FilterInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.Reader; -import java.io.Writer; -import java.net.URI; - -import javax.tools.FileObject; - +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.OSSObject; +import com.aliyun.oss.model.OSSObjectSummary; +import com.google.common.base.Predicate; +import com.google.common.base.Strings; +import com.google.common.io.ByteSource; +import com.google.common.io.Files; +import com.google.inject.Inject; import org.apache.druid.data.input.impl.CloudObjectLocation; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.IAE; @@ -43,15 +41,15 @@ import org.apache.druid.segment.loading.URIDataPuller; import org.apache.druid.utils.CompressionUtils; -import com.aliyun.oss.OSS; -import com.aliyun.oss.OSSException; -import com.aliyun.oss.model.OSSObject; -import com.aliyun.oss.model.OSSObjectSummary; -import com.google.common.base.Predicate; -import com.google.common.base.Strings; -import com.google.common.io.ByteSource; -import com.google.common.io.Files; -import com.google.inject.Inject; +import javax.tools.FileObject; +import java.io.File; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Reader; +import java.io.Writer; +import java.net.URI; /** * A data segment puller that also hanldes URI data pulls. @@ -71,7 +69,8 @@ public OssDataSegmentPuller(OSS s3Client) this.client = s3Client; } - FileUtils.FileCopyResult getSegmentFiles(final CloudObjectLocation ossCoords, final File outDir) throws SegmentLoadingException + FileUtils.FileCopyResult getSegmentFiles(final CloudObjectLocation ossCoords, final File outDir) + throws SegmentLoadingException { log.info("Pulling index at path[%s] to outDir[%s]", ossCoords, outDir); @@ -270,9 +269,7 @@ public boolean apply(Throwable e) * Returns the "version" (aka last modified timestamp) of the URI * * @param uri The URI to check the last timestamp - * * @return The time in ms of the last modification of the URI in String format - * * @throws IOException */ @Override diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java index 065c47b73deb..8429433bdca7 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java @@ -19,12 +19,11 @@ package org.apache.druid.storage.aliyun; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.util.List; -import java.util.Map; - +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.segment.SegmentUtils; @@ -32,11 +31,11 @@ import org.apache.druid.timeline.DataSegment; import org.apache.druid.utils.CompressionUtils; -import com.aliyun.oss.OSS; -import com.aliyun.oss.OSSException; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.inject.Inject; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.List; +import java.util.Map; public class OssDataSegmentPusher implements DataSegmentPusher { @@ -58,7 +57,7 @@ public OssDataSegmentPusher( @Override public String getPathForHadoop() { - return StringUtils.format("%s/%s", config.getBucket(), config.getBaseKey()); + return StringUtils.format("%s/%s", config.getBucket(), config.getBaseKey()); } @Deprecated diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfig.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfig.java index 7d051d72136f..908586571051 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfig.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfig.java @@ -24,6 +24,7 @@ import javax.validation.constraints.Min; /** + * */ public class OssDataSegmentPusherConfig { @@ -39,6 +40,7 @@ public class OssDataSegmentPusherConfig @JsonProperty @Min(1) private int maxListingLength = 1024; + // use s3n by default for backward compatibility public void setBucket(String bucket) { diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java index 95f83f0a87d0..8afa754a3357 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java @@ -19,19 +19,22 @@ package org.apache.druid.storage.aliyun; -import java.util.List; - +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import com.fasterxml.jackson.core.Version; +import com.fasterxml.jackson.databind.Module; +import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; +import com.google.inject.Provides; +import com.google.inject.multibindings.MapBinder; import org.apache.druid.data.SearchableVersionedDataFinder; +import org.apache.druid.data.input.aliyun.OssInputSourceConfig; import org.apache.druid.guice.Binders; import org.apache.druid.guice.JsonConfigProvider; import org.apache.druid.guice.LazySingleton; import org.apache.druid.initialization.DruidModule; -import com.fasterxml.jackson.core.Version; -import com.fasterxml.jackson.databind.Module; -import com.google.common.collect.ImmutableList; -import com.google.inject.Binder; -import com.google.inject.multibindings.MapBinder; +import java.util.List; /** * @@ -80,14 +83,21 @@ public void configure(Binder binder) .addBinding(SCHEME_S3N) .to(OssTimestampVersionedDataFinder.class) .in(LazySingleton.class); - Binders.dataSegmentKillerBinder(binder).addBinding(SCHEME_S3_ZIP).to(OssDataSegmentKiller.class).in(LazySingleton.class); - Binders.dataSegmentMoverBinder(binder).addBinding(SCHEME_S3_ZIP).to(OssDataSegmentMover.class).in(LazySingleton.class); + Binders.dataSegmentKillerBinder(binder) + .addBinding(SCHEME_S3_ZIP) + .to(OssDataSegmentKiller.class) + .in(LazySingleton.class); + Binders.dataSegmentMoverBinder(binder) + .addBinding(SCHEME_S3_ZIP) + .to(OssDataSegmentMover.class) + .in(LazySingleton.class); Binders.dataSegmentArchiverBinder(binder) .addBinding(SCHEME_S3_ZIP) .to(OssDataSegmentArchiver.class) .in(LazySingleton.class); Binders.dataSegmentPusherBinder(binder).addBinding(SCHEME).to(OssDataSegmentPusher.class).in(LazySingleton.class); JsonConfigProvider.bind(binder, "druid.storage", OssInputDataConfig.class); + JsonConfigProvider.bind(binder, "druid.storage", OssInputSourceConfig.class); JsonConfigProvider.bind(binder, "druid.storage", OssDataSegmentPusherConfig.class); JsonConfigProvider.bind(binder, "druid.storage", OssDataSegmentArchiverConfig.class); JsonConfigProvider.bind(binder, "druid.storage", OssStorageConfig.class); @@ -96,4 +106,14 @@ public void configure(Binder binder) JsonConfigProvider.bind(binder, "druid.indexer.logs", OssTaskLogsConfig.class); binder.bind(OssTaskLogs.class).in(LazySingleton.class); } + + @Provides + @LazySingleton + public OSS getOSS(OssInputSourceConfig inputSourceConfig) + { + return new OSSClientBuilder().build(inputSourceConfig.getEndpoint(), + inputSourceConfig.getAccessKeyId().getPassword(), + inputSourceConfig.getSecretAccessKey().getPassword() + ); + } } diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java index 6ee55f3adda1..0d4fb3626d20 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java @@ -19,18 +19,6 @@ package org.apache.druid.storage.aliyun; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.Date; - -import org.apache.druid.common.utils.CurrentTimeMillisSupplier; -import org.apache.druid.java.util.common.IOE; -import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.tasklogs.TaskLogs; - import com.aliyun.oss.OSS; import com.aliyun.oss.OSSException; import com.aliyun.oss.model.GetObjectRequest; @@ -39,6 +27,17 @@ import com.google.common.base.Throwables; import com.google.common.io.ByteSource; import com.google.inject.Inject; +import org.apache.druid.common.utils.CurrentTimeMillisSupplier; +import org.apache.druid.java.util.common.IOE; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.tasklogs.TaskLogs; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Date; /** * Provides task logs archived on S3. @@ -117,7 +116,7 @@ public InputStream openStream() throws IOException ); } catch (OSSException e) { - if ( "NoSuchKey".equals(e.getErrorCode()) + if ("NoSuchKey".equals(e.getErrorCode()) || "NoSuchBucket".equals(e.getErrorCode())) { return Optional.absent(); } else { diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java index e2be7478132c..e8a5f36a199e 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java @@ -25,6 +25,7 @@ import javax.validation.constraints.NotNull; /** + * */ public class OssTaskLogsConfig { diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java index bf884f0decf0..f443e7396e0d 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java @@ -19,20 +19,18 @@ package org.apache.druid.storage.aliyun; -import java.net.URI; -import java.util.Collections; -import java.util.Iterator; -import java.util.regex.Pattern; - -import javax.annotation.Nullable; - +import com.aliyun.oss.OSS; +import com.aliyun.oss.model.OSSObjectSummary; +import com.google.inject.Inject; import org.apache.druid.data.SearchableVersionedDataFinder; import org.apache.druid.data.input.impl.CloudObjectLocation; import org.apache.druid.java.util.common.StringUtils; -import com.aliyun.oss.OSS; -import com.aliyun.oss.model.OSSObjectSummary; -import com.google.inject.Inject; +import javax.annotation.Nullable; +import java.net.URI; +import java.util.Collections; +import java.util.Iterator; +import java.util.regex.Pattern; public class OssTimestampVersionedDataFinder extends OssDataSegmentPuller implements SearchableVersionedDataFinder { @@ -54,7 +52,6 @@ public OssTimestampVersionedDataFinder(OSS client) * * @param uri The URI of in the form of `s3://some_bucket/some_key` * @param pattern The pattern matcher to determine if a *key* is of interest, or `null` to match everything. - * * @return A URI to the most recently modified object which matched the pattern. */ @Override diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java index 4ef3c394c742..691b0251ce2b 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java +++ b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java @@ -19,20 +19,6 @@ package org.apache.druid.storage.aliyun; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.druid.data.input.impl.CloudObjectLocation; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.RetryUtils; -import org.apache.druid.java.util.common.RetryUtils.Task; -import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.logger.Logger; - import com.aliyun.oss.OSS; import com.aliyun.oss.OSSException; import com.aliyun.oss.model.AccessControlList; @@ -45,6 +31,19 @@ import com.google.common.base.Joiner; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; +import org.apache.druid.data.input.impl.CloudObjectLocation; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.RetryUtils; +import org.apache.druid.java.util.common.RetryUtils.Task; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; /** * @@ -60,7 +59,7 @@ static boolean isServiceExceptionRecoverable(OSSException ex) { final boolean isIOException = ex.getCause() instanceof IOException; final boolean isTimeout = "RequestTimeout".equals(ex.getErrorCode()); - final boolean badStatusCode = false;//ex. == 400 || ex.getStatusCode() == 403 || ex.getStatusCode() == 404; + final boolean badStatusCode = false; //ex. == 400 || ex.getStatusCode() == 403 || ex.getStatusCode() == 404; return !badStatusCode && (isIOException || isTimeout); } @@ -111,7 +110,7 @@ static boolean isObjectInBucketIgnoringPermission( /** * Create an iterator over a set of S3 objects specified by a set of prefixes. - * + *

* For each provided prefix URI, the iterator will walk through all objects that are in the same bucket as the * provided URI and whose keys start with that URI's path, except for directory placeholders (which will be * ignored). The iterator is computed incrementally by calling {@link OssClientHelper#listObjectsV2} for @@ -178,8 +177,8 @@ public static URI checkURI(URI uri) * in the given bucket. * * @param client s3 client - * @param bucket s3 bucket - * @param key unique key for the object to be retrieved + * @param bucket s3 bucket + * @param key unique key for the object to be retrieved */ public static OSSObjectSummary getSingleObjectSummary(OSS client, String bucket, String key) { @@ -205,11 +204,12 @@ public static OSSObjectSummary getSingleObjectSummary(OSS client, String bucket, /** * Delete the files from S3 in a specified bucket, matching a specified prefix and filter + * * @param client s3 client - * @param config specifies the configuration to use when finding matching files in S3 to delete - * @param bucket s3 bucket - * @param prefix the file prefix - * @param filter function which returns true if the prefix file found should be deleted and false otherwise. + * @param config specifies the configuration to use when finding matching files in S3 to delete + * @param bucket s3 bucket + * @param prefix the file prefix + * @param filter function which returns true if the prefix file found should be deleted and false otherwise. * @throws Exception */ public static void deleteObjectsInPath( @@ -263,7 +263,7 @@ private static void deleteBucketKeys( /** * Uploads a file to S3 if possible. First trying to set ACL to give the bucket owner full control of the file before uploading. * - * @param client aliyun OSS client + * @param client aliyun OSS client * @param disableAcl true if ACL shouldn't be set for the file * @param key The key under which to store the new object. * @param file The path of the file to upload to Amazon S3. @@ -280,9 +280,9 @@ static void uploadFileIfPossible( log.info("Pushing [%s] to bucket[%s] and key[%s].", file, bucket, key); client.putObject(putObjectRequest); - + if (!disableAcl) { - client.setObjectAcl(bucket, key, OssUtils.grantFullControlToBucketOwner(client, bucket)); + client.setObjectAcl(bucket, key, OssUtils.grantFullControlToBucketOwner(client, bucket)); } } } diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java index 7252b8c267cc..3f1f08242422 100644 --- a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java @@ -19,14 +19,6 @@ package org.apache.druid.storage.aliyun; -import java.net.URI; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import org.junit.Assert; -import org.junit.Test; - import com.aliyun.oss.OSS; import com.aliyun.oss.OSSClient; import com.aliyun.oss.model.ListObjectsRequest; @@ -34,6 +26,13 @@ import com.aliyun.oss.model.ObjectListing; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import org.junit.Assert; +import org.junit.Test; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; public class ObjectSummaryIteratorTest { @@ -65,7 +64,13 @@ public void testSingleObject() public void testMultiObjectOneKeyAtATime() { test( - ImmutableList.of("oss://b/foo/bar1", "oss://b/foo/bar2", "oss://b/foo/bar3", "oss://b/foo/bar4", "oss://b/foo/baz"), + ImmutableList.of( + "oss://b/foo/bar1", + "oss://b/foo/bar2", + "oss://b/foo/bar3", + "oss://b/foo/bar4", + "oss://b/foo/baz" + ), ImmutableList.of("oss://b/foo/"), 1 ); @@ -75,7 +80,13 @@ public void testMultiObjectOneKeyAtATime() public void testMultiObjectTwoKeysAtATime() { test( - ImmutableList.of("oss://b/foo/bar1", "oss://b/foo/bar2", "oss://b/foo/bar3", "oss://b/foo/bar4", "oss://b/foo/baz"), + ImmutableList.of( + "oss://b/foo/bar1", + "oss://b/foo/bar2", + "oss://b/foo/bar3", + "oss://b/foo/bar4", + "oss://b/foo/baz" + ), ImmutableList.of("oss://b/foo/"), 2 ); @@ -85,7 +96,13 @@ public void testMultiObjectTwoKeysAtATime() public void testMultiObjectTenKeysAtATime() { test( - ImmutableList.of("oss://b/foo/bar1", "oss://b/foo/bar2", "oss://b/foo/bar3", "oss://b/foo/bar4", "oss://b/foo/baz"), + ImmutableList.of( + "oss://b/foo/bar1", + "oss://b/foo/bar2", + "oss://b/foo/bar3", + "oss://b/foo/bar4", + "oss://b/foo/baz" + ), ImmutableList.of("oss://b/foo/"), 10 ); @@ -149,7 +166,13 @@ public void testDifferentBucket() public void testWithMultiplePrefixesReturningAllNonEmptyObjectsStartingWithOneOfPrefixes() { test( - ImmutableList.of("oss://b/foo/bar1", "oss://b/foo/bar2", "oss://b/foo/bar3", "oss://b/foo/bar4", "oss://b/foo/baz"), + ImmutableList.of( + "oss://b/foo/bar1", + "oss://b/foo/bar2", + "oss://b/foo/bar3", + "oss://b/foo/bar4", + "oss://b/foo/baz" + ), ImmutableList.of("oss://b/foo/bar", "oss://b/foo/baz"), 10 ); @@ -166,11 +189,11 @@ private static void test( // O(N^2) but who cares -- the list is short. for (final String uri : expectedUris) { final List matches = TEST_OBJECTS.stream() - .filter( - summary -> - OssUtils.summaryToUri(summary).toString().equals(uri) - ) - .collect(Collectors.toList()); + .filter( + summary -> + OssUtils.summaryToUri(summary).toString().equals(uri) + ) + .collect(Collectors.toList()); expectedObjects.add(Iterables.getOnlyElement(matches)); } diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java index 7bc09214c965..7a2b1db9c412 100644 --- a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java @@ -19,16 +19,6 @@ package org.apache.druid.storage.aliyun; -import java.util.Map; - -import org.apache.druid.jackson.DefaultObjectMapper; -import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.timeline.DataSegment; -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - import com.aliyun.oss.OSS; import com.aliyun.oss.OSSClient; import com.fasterxml.jackson.databind.BeanProperty; @@ -38,6 +28,15 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.timeline.DataSegment; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Map; public class OssDataSegmentArchiverTest { @@ -112,7 +111,12 @@ public void testSimpleArchive() throws Exception OssDataSegmentPuller.KEY, ARCHIVER_CONFIG.getArchiveBaseKey() + "archived" )); - final OssDataSegmentArchiver archiver = new OssDataSegmentArchiver(MAPPER, OSS_CLIENT, ARCHIVER_CONFIG, PUSHER_CONFIG) + final OssDataSegmentArchiver archiver = new OssDataSegmentArchiver( + MAPPER, + OSS_CLIENT, + ARCHIVER_CONFIG, + PUSHER_CONFIG + ) { @Override public DataSegment move(DataSegment segment, Map targetLoadSpec) @@ -126,7 +130,12 @@ public DataSegment move(DataSegment segment, Map targetLoadSpec) @Test public void testSimpleArchiveDoesntMove() throws Exception { - final OssDataSegmentArchiver archiver = new OssDataSegmentArchiver(MAPPER, OSS_CLIENT, ARCHIVER_CONFIG, PUSHER_CONFIG) + final OssDataSegmentArchiver archiver = new OssDataSegmentArchiver( + MAPPER, + OSS_CLIENT, + ARCHIVER_CONFIG, + PUSHER_CONFIG + ) { @Override public DataSegment move(DataSegment segment, Map targetLoadSpec) @@ -149,7 +158,12 @@ public void testSimpleRestore() throws Exception OssDataSegmentPuller.KEY, ARCHIVER_CONFIG.getArchiveBaseKey() + "archived" )); - final OssDataSegmentArchiver archiver = new OssDataSegmentArchiver(MAPPER, OSS_CLIENT, ARCHIVER_CONFIG, PUSHER_CONFIG) + final OssDataSegmentArchiver archiver = new OssDataSegmentArchiver( + MAPPER, + OSS_CLIENT, + ARCHIVER_CONFIG, + PUSHER_CONFIG + ) { @Override public DataSegment move(DataSegment segment, Map targetLoadSpec) @@ -163,7 +177,12 @@ public DataSegment move(DataSegment segment, Map targetLoadSpec) @Test public void testSimpleRestoreDoesntMove() throws Exception { - final OssDataSegmentArchiver archiver = new OssDataSegmentArchiver(MAPPER, OSS_CLIENT, ARCHIVER_CONFIG, PUSHER_CONFIG) + final OssDataSegmentArchiver archiver = new OssDataSegmentArchiver( + MAPPER, + OSS_CLIENT, + ARCHIVER_CONFIG, + PUSHER_CONFIG + ) { @Override public DataSegment move(DataSegment segment, Map targetLoadSpec) diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java index fd854ec2c256..0547f3895377 100644 --- a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java @@ -19,10 +19,12 @@ package org.apache.druid.storage.aliyun; -import java.io.IOException; -import java.net.URI; -import java.util.Arrays; - +import com.aliyun.oss.OSS; +import com.aliyun.oss.model.DeleteObjectsRequest; +import com.aliyun.oss.model.OSSObjectSummary; +import com.amazonaws.SdkClientException; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; import org.easymock.EasyMock; @@ -33,12 +35,9 @@ import org.junit.Test; import org.junit.runner.RunWith; -import com.aliyun.oss.OSS; -import com.aliyun.oss.model.DeleteObjectsRequest; -import com.aliyun.oss.model.OSSObjectSummary; -import com.amazonaws.SdkClientException; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.net.URI; +import java.util.Arrays; @RunWith(EasyMockRunner.class) public class OssDataSegmentKillerTest extends EasyMockSupport diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java index 110b4ad7962b..c8d8cf4a6fc5 100644 --- a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java @@ -19,20 +19,6 @@ package org.apache.druid.storage.aliyun; -import java.io.File; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.common.MapUtils; -import org.apache.druid.segment.loading.SegmentLoadingException; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.partition.NoneShardSpec; -import org.junit.Assert; -import org.junit.Test; - import com.aliyun.oss.OSSClient; import com.aliyun.oss.OSSException; import com.aliyun.oss.model.CopyObjectRequest; @@ -44,6 +30,19 @@ import com.aliyun.oss.model.StorageClass; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.MapUtils; +import org.apache.druid.segment.loading.SegmentLoadingException; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.NoneShardSpec; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; public class OssDataSegmentMoverTest { @@ -126,7 +125,7 @@ public void testMoveException() throws Exception ImmutableMap.of("baseKey", "targetBaseKey", "bucket", "archive") ); } - + @Test public void testIgnoresGoneButAlreadyMoved() throws Exception { @@ -181,7 +180,7 @@ private static class MockClient extends OSSClient private MockClient() { - super("endpoint","accessKeyId","keySecret"); + super("endpoint", "accessKeyId", "keySecret"); } public boolean didMove() @@ -189,15 +188,6 @@ public boolean didMove() return copied && deletedOld; } -// @Override -// public AccessControlList getBucketAcl(String bucketName) -// { -// final AccessControlList acl = new AccessControlList(); -// acl.setOwner(new Owner("ownerId", "owner")); -// acl.grantAllPermissions(new Grant(new CanonicalGrantee(acl.getOwner().getId()), Permission.FullControl)); -// return acl; -// } - @Override public boolean doesObjectExist(String bucketName, String objectKey) { @@ -241,7 +231,15 @@ public CopyObjectResult copyObject(CopyObjectRequest copyObjectRequest) .add(destinationObjectKey); return new CopyObjectResult(); } else { - final OSSException exception = new OSSException("OssDataSegmentMoverTest", "NoSuchKey",null, null, null, null, null); + final OSSException exception = new OSSException( + "OssDataSegmentMoverTest", + "NoSuchKey", + null, + null, + null, + null, + null + ); throw exception; } } diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java index 60e0b1070071..1c1a1a853273 100644 --- a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java @@ -19,16 +19,12 @@ package org.apache.druid.storage.aliyun; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.util.Date; -import java.util.zip.GZIPOutputStream; - +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.OSSObject; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.ObjectListing; import org.apache.druid.data.input.impl.CloudObjectLocation; import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.StringUtils; @@ -39,12 +35,15 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; -import com.aliyun.oss.OSS; -import com.aliyun.oss.OSSException; -import com.aliyun.oss.model.ListObjectsRequest; -import com.aliyun.oss.model.OSSObject; -import com.aliyun.oss.model.OSSObjectSummary; -import com.aliyun.oss.model.ObjectListing; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.Date; +import java.util.zip.GZIPOutputStream; /** * @@ -170,7 +169,7 @@ public void testGZUncompressRetries() throws IOException, SegmentLoadingExceptio File tmpDir = temporaryFolder.newFolder("gzTestDir"); - OSSException exception = new OSSException("OssDataSegmentPullerTest","NoSuchKey",null,null,null,null,null); + OSSException exception = new OSSException("OssDataSegmentPullerTest", "NoSuchKey", null, null, null, null, null); EasyMock.expect(s3Client.doesObjectExist(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) .andReturn(true) .once(); diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java index 09fd47f40094..e462a5c6d68b 100644 --- a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java @@ -19,20 +19,18 @@ package org.apache.druid.storage.aliyun; -import java.io.IOException; -import java.util.Locale; -import java.util.Set; - -import javax.validation.ConstraintViolation; -import javax.validation.Validation; -import javax.validation.Validator; - +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Iterators; import org.apache.druid.jackson.DefaultObjectMapper; import org.junit.Assert; import org.junit.Test; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Iterators; +import javax.validation.ConstraintViolation; +import javax.validation.Validation; +import javax.validation.Validator; +import java.io.IOException; +import java.util.Locale; +import java.util.Set; public class OssDataSegmentPusherConfigTest { @@ -62,20 +60,21 @@ public void testSerializationWithDefaults() throws IOException @Test public void testSerializationValidatingMaxListingLength() throws IOException { - Locale old = Locale.getDefault(); - Locale.setDefault(Locale.ENGLISH); - try { - String jsonConfig = "{\"bucket\":\"bucket1\",\"baseKey\":\"dataSource1\"," - + "\"disableAcl\":false,\"maxListingLength\":-1}"; - Validator validator = Validation.buildDefaultValidatorFactory().getValidator(); - - OssDataSegmentPusherConfig config = JSON_MAPPER.readValue(jsonConfig, OssDataSegmentPusherConfig.class); - Set> violations = validator.validate(config); - Assert.assertEquals(1, violations.size()); - ConstraintViolation violation = Iterators.getOnlyElement(violations.iterator()); - Assert.assertEquals("must be greater than or equal to 1", violation.getMessage()); - }finally { - Locale.setDefault(old); - } + Locale old = Locale.getDefault(); + Locale.setDefault(Locale.ENGLISH); + try { + String jsonConfig = "{\"bucket\":\"bucket1\",\"baseKey\":\"dataSource1\"," + + "\"disableAcl\":false,\"maxListingLength\":-1}"; + Validator validator = Validation.buildDefaultValidatorFactory().getValidator(); + + OssDataSegmentPusherConfig config = JSON_MAPPER.readValue(jsonConfig, OssDataSegmentPusherConfig.class); + Set> violations = validator.validate(config); + Assert.assertEquals(1, violations.size()); + ConstraintViolation violation = Iterators.getOnlyElement(violations.iterator()); + Assert.assertEquals("must be greater than or equal to 1", violation.getMessage()); + } + finally { + Locale.setDefault(old); + } } } diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java index 00a1f2e15e98..e413e49529e7 100644 --- a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java @@ -19,11 +19,9 @@ package org.apache.druid.storage.aliyun; -import java.io.File; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.regex.Pattern; - +import com.aliyun.oss.OSS; +import com.aliyun.oss.model.PutObjectResult; +import com.google.common.io.Files; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.NoneShardSpec; @@ -33,11 +31,13 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; -import com.aliyun.oss.OSS; -import com.aliyun.oss.model.PutObjectResult; -import com.google.common.io.Files; +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.regex.Pattern; /** + * */ public class OssDataSegmentPusherTest { @@ -68,7 +68,10 @@ public void testPush() throws Exception @Test public void testPushUseUniquePath() throws Exception { - testPushInternal(true, "key/foo/2015-01-01T00:00:00\\.000Z_2016-01-01T00:00:00\\.000Z/0/0/[A-Za-z0-9-]{36}/index\\.zip"); + testPushInternal( + true, + "key/foo/2015-01-01T00:00:00\\.000Z_2016-01-01T00:00:00\\.000Z/0/0/[A-Za-z0-9-]{36}/index\\.zip" + ); } private void testPushInternal(boolean useUniquePath, String matcher) throws Exception diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java index 90850ee27879..5c770ecd20d0 100644 --- a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java @@ -19,13 +19,17 @@ package org.apache.druid.storage.aliyun; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - +import com.aliyun.oss.ClientException; +import com.aliyun.oss.OSS; +import com.aliyun.oss.model.AccessControlList; +import com.aliyun.oss.model.DeleteObjectsRequest; +import com.aliyun.oss.model.Grant; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.Owner; +import com.aliyun.oss.model.PutObjectRequest; +import com.aliyun.oss.model.PutObjectResult; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.apache.druid.common.utils.CurrentTimeMillisSupplier; import org.apache.druid.java.util.common.StringUtils; import org.easymock.EasyMock; @@ -38,19 +42,12 @@ import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; -import com.aliyun.oss.ClientException; -import com.aliyun.oss.OSS; -import com.aliyun.oss.model.AccessControlList; -import com.aliyun.oss.model.DeleteObjectsRequest; -import com.aliyun.oss.model.Grant; -import com.aliyun.oss.model.OSSObjectSummary; -import com.aliyun.oss.model.Owner; -import com.aliyun.oss.model.Permission; -import com.aliyun.oss.model.PutObjectRequest; -import com.aliyun.oss.model.PutObjectResult; -import com.aliyun.oss.model.SetObjectAclRequest; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; @RunWith(EasyMockRunner.class) public class OssTaskLogsTest extends EasyMockSupport @@ -76,7 +73,7 @@ public class OssTaskLogsTest extends EasyMockSupport @Rule public final TemporaryFolder tempFolder = new TemporaryFolder(); - + @Test public void testTaskLogsPushWithAclDisabled() throws Exception { @@ -89,25 +86,6 @@ public void testTaskLogsPushWithAclDisabled() throws Exception Assert.assertEquals("Grant list should be empty as ACL is disabled", 0, grantList.size()); } -// @Test -// public void testTaskLogsPushWithAclEnabled() throws Exception -// { -// String ownerId = "test_owner"; -// String ownerDisplayName = "test_owner"; -// -// List grantList = testPushInternal(false, ownerId, ownerDisplayName); -// -// Assert.assertNotNull("Grant list should not be null", grantList); -// Assert.assertEquals("Grant list size should be equal to 1", 1, grantList.size()); -// Grant grant = grantList.get(0); -// Assert.assertEquals( -// "The Grantee identifier should be test_owner", -// "test_owner", -// grant.getGrantee().getIdentifier() -// ); -// Assert.assertEquals("The Grant should have full control permission", Permission.FullControl, grant.getPermission()); -// } - @Test public void test_killAll_noException_deletesAllTaskLogs() throws IOException { @@ -123,9 +101,9 @@ public void test_killAll_noException_deletesAllTaskLogs() throws IOException ); DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET); - deleteRequest1.setKeys(Arrays.asList(KEY_1)); + deleteRequest1.setKeys(Arrays.asList(KEY_1)); DeleteObjectsRequest deleteRequest2 = new DeleteObjectsRequest(TEST_BUCKET); - deleteRequest2.setKeys(Arrays.asList(KEY_2)); + deleteRequest2.setKeys(Arrays.asList(KEY_2)); OssTestUtils.mockClientDeleteObjects( ossClient, @@ -140,8 +118,8 @@ public void test_killAll_noException_deletesAllTaskLogs() throws IOException config.setPrefix(TEST_PREFIX); OssInputDataConfig inputDataConfig = new OssInputDataConfig(); inputDataConfig.setMaxListingLength(MAX_KEYS); - OssTaskLogs OssTaskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); - OssTaskLogs.killAll(); + OssTaskLogs taskLogs = new taskLogs(ossClient, config, inputDataConfig, timeSupplier); + taskLogs.killAll(); EasyMock.verify(ossClient, timeSupplier); } @@ -174,8 +152,8 @@ public void test_killAll_recoverableExceptionWhenDeletingObjects_deletesAllTaskL config.setPrefix(TEST_PREFIX); OssInputDataConfig inputDataConfig = new OssInputDataConfig(); inputDataConfig.setMaxListingLength(MAX_KEYS); - OssTaskLogs OssTaskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); - OssTaskLogs.killAll(); + OssTaskLogs taskLogs = new taskLogs(ossClient, config, inputDataConfig, timeSupplier); + taskLogs.killAll(); EasyMock.verify(ossClient, timeSupplier); } @@ -208,8 +186,8 @@ public void test_killAll_nonrecoverableExceptionWhenListingObjects_doesntDeleteA config.setPrefix(TEST_PREFIX); OssInputDataConfig inputDataConfig = new OssInputDataConfig(); inputDataConfig.setMaxListingLength(MAX_KEYS); - OssTaskLogs OssTaskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); - OssTaskLogs.killAll(); + OssTaskLogs taskLogs = new taskLogs(ossClient, config, inputDataConfig, timeSupplier); + taskLogs.killAll(); } catch (IOException e) { ioExceptionThrown = true; @@ -244,8 +222,8 @@ public void test_killOlderThan_noException_deletesOnlyTaskLogsOlderThan() throws config.setPrefix(TEST_PREFIX); OssInputDataConfig inputDataConfig = new OssInputDataConfig(); inputDataConfig.setMaxListingLength(MAX_KEYS); - OssTaskLogs OssTaskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); - OssTaskLogs.killOlderThan(TIME_NOW); + OssTaskLogs taskLogs = new taskLogs(ossClient, config, inputDataConfig, timeSupplier); + taskLogs.killOlderThan(TIME_NOW); EasyMock.verify(ossClient, timeSupplier); } @@ -277,8 +255,8 @@ public void test_killOlderThan_recoverableExceptionWhenListingObjects_deletesAll config.setPrefix(TEST_PREFIX); OssInputDataConfig inputDataConfig = new OssInputDataConfig(); inputDataConfig.setMaxListingLength(MAX_KEYS); - OssTaskLogs OssTaskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); - OssTaskLogs.killOlderThan(TIME_NOW); + OssTaskLogs taskLogs = new taskLogs(ossClient, config, inputDataConfig, timeSupplier); + taskLogs.killOlderThan(TIME_NOW); EasyMock.verify(ossClient, timeSupplier); } @@ -310,8 +288,8 @@ public void test_killOlderThan_nonrecoverableExceptionWhenListingObjects_doesntD config.setPrefix(TEST_PREFIX); OssInputDataConfig inputDataConfig = new OssInputDataConfig(); inputDataConfig.setMaxListingLength(MAX_KEYS); - OssTaskLogs OssTaskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); - OssTaskLogs.killOlderThan(TIME_NOW); + OssTaskLogs taskLogs = new taskLogs(ossClient, config, inputDataConfig, timeSupplier); + taskLogs.killOlderThan(TIME_NOW); } catch (IOException e) { ioExceptionThrown = true; @@ -346,12 +324,12 @@ private List testPushInternal(boolean disableAcl, String ownerId, String config.setBucket(TEST_BUCKET); CurrentTimeMillisSupplier timeSupplier = new CurrentTimeMillisSupplier(); OssInputDataConfig inputDataConfig = new OssInputDataConfig(); - OssTaskLogs OssTaskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); + OssTaskLogs taskLogs = new taskLogs(ossClient, config, inputDataConfig, timeSupplier); String taskId = "index_test-datasource_2019-06-18T13:30:28.887Z"; File logFile = tempFolder.newFile("test_log_file"); - OssTaskLogs.pushTaskLog(taskId, logFile); + taskLogs.pushTaskLog(taskId, logFile); return aclExpected.getGrants().stream().collect(Collectors.toList()); } diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTestUtils.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTestUtils.java index a67230103575..35ef96663e07 100644 --- a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTestUtils.java +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTestUtils.java @@ -19,12 +19,12 @@ package org.apache.druid.storage.aliyun; -import java.net.URI; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - +import com.aliyun.oss.OSS; +import com.aliyun.oss.model.DeleteObjectsRequest; +import com.aliyun.oss.model.DeleteObjectsResult; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.ObjectListing; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.StringUtils; import org.easymock.EasyMock; @@ -33,12 +33,11 @@ import org.easymock.IExpectationSetters; import org.joda.time.DateTime; -import com.aliyun.oss.OSS; -import com.aliyun.oss.model.DeleteObjectsRequest; -import com.aliyun.oss.model.DeleteObjectsResult; -import com.aliyun.oss.model.ListObjectsRequest; -import com.aliyun.oss.model.OSSObjectSummary; -import com.aliyun.oss.model.ObjectListing; +import java.net.URI; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class OssTestUtils extends EasyMockSupport { @@ -84,7 +83,8 @@ public void appendTo(StringBuffer buffer) public static void expectListObjects( OSS client, URI prefix, - List objectSummaries) + List objectSummaries + ) { final ObjectListing result = new ObjectListing(); result.setBucketName(prefix.getAuthority()); @@ -163,7 +163,8 @@ public void appendTo(StringBuffer buffer) public static OSSObjectSummary newOSSObjectSummary( String bucket, String key, - long lastModifiedTimestamp) + long lastModifiedTimestamp + ) { OSSObjectSummary objectSummary = new OSSObjectSummary(); objectSummary.setBucketName(bucket); diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java index 8c9f5a325c91..048b5db94e52 100644 --- a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java +++ b/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java @@ -19,19 +19,18 @@ package org.apache.druid.storage.aliyun; -import java.net.URI; -import java.util.Date; -import java.util.regex.Pattern; - +import com.aliyun.oss.OSS; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.ObjectListing; import org.apache.druid.java.util.common.StringUtils; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Test; -import com.aliyun.oss.OSS; -import com.aliyun.oss.model.ListObjectsRequest; -import com.aliyun.oss.model.OSSObjectSummary; -import com.aliyun.oss.model.ObjectListing; +import java.net.URI; +import java.util.Date; +import java.util.regex.Pattern; public class OssTimestampVersionedDataFinderTest { From 8d3de81ff4d0578623a84bf8f959cc9034344f0c Mon Sep 17 00:00:00 2001 From: frank chen Date: Sat, 16 May 2020 13:39:38 +0800 Subject: [PATCH 03/24] data stored successfully --- .../storage/aliyun/OssStorageConfig.java | 28 ------ .../pom.xml | 28 ++++-- .../druid/data/input/aliyun/OssEntity.java | 0 .../data/input/aliyun/OssInputSource.java | 24 ++---- .../input/aliyun/OssInputSourceConfig.java | 63 +++++++------- .../aliyun/OssInputSourceDruidModule.java | 2 +- .../aliyun/OssFirehoseDruidModule.java | 2 +- .../aliyun/StaticOssFirehoseFactory.java | 12 +-- .../storage/aliyun/ObjectSummaryIterator.java | 9 -- .../aliyun/OssDataSegmentArchiver.java | 16 ++-- .../aliyun/OssDataSegmentArchiverConfig.java | 0 .../storage/aliyun/OssDataSegmentKiller.java | 24 +++--- .../storage/aliyun/OssDataSegmentMover.java | 50 +++++------ .../storage/aliyun/OssDataSegmentPuller.java | 14 +-- .../storage/aliyun/OssDataSegmentPusher.java | 8 +- .../aliyun/OssDataSegmentPusherConfig.java | 0 .../storage/aliyun/OssInputDataConfig.java | 4 +- .../druid/storage/aliyun/OssLoadSpec.java | 0 .../storage/aliyun/OssStorageDruidModule.java | 17 ++-- .../druid/storage/aliyun/OssTaskLogs.java | 10 +-- .../storage/aliyun/OssTaskLogsConfig.java | 0 .../OssTimestampVersionedDataFinder.java | 8 +- .../apache/druid/storage/aliyun/OssUtils.java | 26 +++--- ...rg.apache.druid.initialization.DruidModule | 0 .../aliyun/ObjectSummaryIteratorTest.java | 86 +++++++++---------- .../aliyun/OssDataSegmentArchiverTest.java | 2 +- .../aliyun/OssDataSegmentKillerTest.java | 10 +-- .../aliyun/OssDataSegmentMoverTest.java | 0 .../aliyun/OssDataSegmentPullerTest.java | 2 +- .../OssDataSegmentPusherConfigTest.java | 0 .../aliyun/OssDataSegmentPusherTest.java | 0 .../druid/storage/aliyun/OssTaskLogsTest.java | 14 +-- .../druid/storage/aliyun/OssTestUtils.java | 0 .../OssTimestampVersionedDataFinderTest.java | 14 +-- 34 files changed, 218 insertions(+), 255 deletions(-) delete mode 100644 extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/pom.xml (89%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java (100%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java (81%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java (60%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java (95%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java (96%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java (92%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java (92%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java (88%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java (100%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java (77%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java (82%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java (95%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java (92%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfig.java (100%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java (95%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java (100%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java (84%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java (94%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java (100%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java (87%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java (90%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule (100%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java (77%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java (98%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java (96%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java (100%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java (99%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java (100%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java (100%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java (94%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/test/java/org/apache/druid/storage/aliyun/OssTestUtils.java (100%) rename extensions-contrib/{aliyun-extensions => aliyun-oss-extensions}/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java (90%) diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java b/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java deleted file mode 100644 index cf0f9e3ecc7e..000000000000 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.storage.aliyun; - -/** - * General configurations for Amazon S3 storage. - */ -public class OssStorageConfig -{ - -} diff --git a/extensions-contrib/aliyun-extensions/pom.xml b/extensions-contrib/aliyun-oss-extensions/pom.xml similarity index 89% rename from extensions-contrib/aliyun-extensions/pom.xml rename to extensions-contrib/aliyun-oss-extensions/pom.xml index d8400964e26a..6a01b9dd370e 100644 --- a/extensions-contrib/aliyun-extensions/pom.xml +++ b/extensions-contrib/aliyun-oss-extensions/pom.xml @@ -23,9 +23,9 @@ 4.0.0 org.apache.druid.extensions - druid-aliyun-extensions - druid-aliyun-extensions - druid-aliyun-extensions + druid-aliyun-oss-extensions + druid-aliyun-oss-extensions + druid-aliyun-oss-extensions org.apache.druid @@ -143,7 +143,6 @@ test - + diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java similarity index 100% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java similarity index 81% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java index 355110cea7f1..45b3e4c9a33e 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java @@ -20,7 +20,6 @@ package org.apache.druid.data.input.aliyun; import com.aliyun.oss.OSS; -import com.aliyun.oss.OSSClientBuilder; import com.aliyun.oss.model.OSSObjectSummary; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; @@ -51,29 +50,22 @@ public class OssInputSource extends CloudObjectInputSource { - // We lazily initialize ServerSideEncryptingAmazonS3 to avoid costly s3 operation when we only need S3InputSource - // for stored information (such as for task logs) and not for ingestion. - // (This cost only applies for new ServerSideEncryptingAmazonS3 created with s3InputSourceConfig given). private final Supplier clientSupplier; @JsonProperty("properties") private final OssInputSourceConfig inputSourceConfig; private final OssInputDataConfig inputDataConfig; /** - * Constructor for S3InputSource + * Constructor for OssInputSource * - * @param client The default ServerSideEncryptingAmazonS3 client built with all default configs - * from Guice. This injected singleton client is use when {@param s3InputSourceConfig} - * is not provided and hence, we can skip building a new client from - * {@param s3ClientBuilder} - * @param clientBulider Use for building a new s3Client to use instead of the default injected - * {@param s3Client}. The configurations of the client can be changed - * before being built + * @param client The default client built with all default configs + * from Guice. This injected singleton client is used when {@param inputSourceConfig} + * is not provided and hence * @param inputDataConfig Stores the configuration for options related to reading input data * @param uris User provided uris to read input data * @param prefixes User provided prefixes to read input data * @param objects User provided cloud objects values to read input data - * @param inputSourceConfig User provided properties for overriding the default S3 configuration + * @param inputSourceConfig User provided properties for overriding the default aliyun-oss configuration */ @JsonCreator public OssInputSource( @@ -86,15 +78,13 @@ public OssInputSource( ) { super(OssStorageDruidModule.SCHEME, uris, prefixes, objects); - this.inputDataConfig = Preconditions.checkNotNull(inputDataConfig, "S3DataSegmentPusherConfig"); + this.inputDataConfig = Preconditions.checkNotNull(inputDataConfig, "inputDataConfig"); Preconditions.checkNotNull(client, "client"); this.inputSourceConfig = inputSourceConfig; this.clientSupplier = Suppliers.memoize( () -> { if (inputSourceConfig != null) { - String accessKeyId = inputSourceConfig.getAccessKeyId().getPassword(); - String secretAccessKey = inputSourceConfig.getSecretAccessKey().getPassword(); - return new OSSClientBuilder().build(inputSourceConfig.getEndpoint(), accessKeyId, secretAccessKey); + return inputSourceConfig.buildClient(); } else { return client; } diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java similarity index 60% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java index 4ed95e85808a..0c3e1fb06c12 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java @@ -19,17 +19,17 @@ package org.apache.druid.data.input.aliyun; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import org.apache.druid.metadata.PasswordProvider; -import javax.annotation.Nullable; import java.util.Objects; /** - * Contains properties for s3 input source. + * Contains properties for aliyun-oss input source. * Properties can be specified by ingestionSpec which will override system default. */ public class OssInputSourceConfig @@ -37,51 +37,43 @@ public class OssInputSourceConfig @JsonCreator public OssInputSourceConfig( @JsonProperty("endpoint") String endpoint, - @JsonProperty("accessKeyId") @Nullable PasswordProvider accessKeyId, - @JsonProperty("secretAccessKey") @Nullable PasswordProvider secretAccessKey + @JsonProperty("accessKey") PasswordProvider accessKey, + @JsonProperty("secretKey") PasswordProvider secretKey ) { - if (accessKeyId != null || secretAccessKey != null) { - this.accessKeyId = Preconditions.checkNotNull( - accessKeyId, - "accessKeyId cannot be null if secretAccessKey is given" - ); - this.secretAccessKey = Preconditions.checkNotNull( - secretAccessKey, - "secretAccessKey cannot be null if accessKeyId is given" - ); - } + this.accessKey = Preconditions.checkNotNull( + accessKey, + "accessKey cannot be null" + ); + this.secretKey = Preconditions.checkNotNull( + secretKey, + "secretKey cannot be null" + ); + this.endpoint = endpoint; } @JsonProperty private String endpoint; @JsonProperty - private PasswordProvider accessKeyId; + private PasswordProvider accessKey; @JsonProperty - private PasswordProvider secretAccessKey; + private PasswordProvider secretKey; public String getEndpoint() { return endpoint; } - public PasswordProvider getAccessKeyId() - { - return accessKeyId; - } - - public PasswordProvider getSecretAccessKey() + public PasswordProvider getAccessKey() { - return secretAccessKey; + return accessKey; } - @JsonIgnore - public boolean isCredentialsConfigured() + public PasswordProvider getSecretKey() { - return accessKeyId != null && - secretAccessKey != null; + return secretKey; } @Override @@ -89,8 +81,8 @@ public String toString() { return "OssInputSourceConfig{" + "endpoint=" + endpoint + - "accessKeyId=" + accessKeyId + - ", secretAccessKey=" + secretAccessKey + + "accessKeyId=" + accessKey + + ", secretAccessKey=" + secretKey + '}'; } @@ -104,14 +96,19 @@ public boolean equals(Object o) return false; } OssInputSourceConfig that = (OssInputSourceConfig) o; - return Objects.equals(accessKeyId, that.accessKeyId) && - Objects.equals(secretAccessKey, that.secretAccessKey) && + return Objects.equals(accessKey, that.accessKey) && + Objects.equals(secretKey, that.secretKey) && Objects.equals(endpoint, that.endpoint); } @Override public int hashCode() { - return Objects.hash(accessKeyId, secretAccessKey, endpoint); + return Objects.hash(accessKey, secretKey, endpoint); + } + + public OSS buildClient() + { + return new OSSClientBuilder().build(endpoint, accessKey.getPassword(), secretKey.getPassword()); } } diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java similarity index 95% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java index a1a9bf29fc74..024a0c532d2d 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java @@ -30,7 +30,7 @@ import java.util.List; /** - * Druid module to wire up native batch support for S3 input + * Druid module to wire up native batch support for aliyun-oss input */ public class OssInputSourceDruidModule implements DruidModule { diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java similarity index 96% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java index 1382f17153b2..527b87378ff6 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java @@ -37,7 +37,7 @@ public class OssFirehoseDruidModule implements DruidModule public List getJacksonModules() { return ImmutableList.of( - new SimpleModule().registerSubtypes(new NamedType(StaticOssFirehoseFactory.class, "static-s3")) + new SimpleModule().registerSubtypes(new NamedType(StaticOssFirehoseFactory.class, "static-aliyun-oss")) ); } diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java similarity index 92% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java index c6e9ffda973f..c7cc821ba94b 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java @@ -51,7 +51,7 @@ import java.util.stream.Collectors; /** - * Builds firehoses that read from a predefined list of S3 objects and then dry up. + * Builds firehoses that read from a predefined list of aliyun-oss objects and then dry up. */ public class StaticOssFirehoseFactory extends PrefetchableTextFilesFirehoseFactory { @@ -75,7 +75,7 @@ public StaticOssFirehoseFactory( ) { super(maxCacheCapacityBytes, maxFetchCapacityBytes, prefetchTriggerBytes, fetchTimeout, maxFetchRetry); - this.client = Preconditions.checkNotNull(client, "s3Client"); + this.client = Preconditions.checkNotNull(client, "client"); this.uris = uris == null ? new ArrayList<>() : uris; this.prefixes = prefixes == null ? new ArrayList<>() : prefixes; @@ -88,11 +88,11 @@ public StaticOssFirehoseFactory( } for (final URI inputURI : this.uris) { - Preconditions.checkArgument("s3".equals(inputURI.getScheme()), "input uri scheme == s3 (%s)", inputURI); + Preconditions.checkArgument("aliyun-oss".equals(inputURI.getScheme()), "input uri scheme == aliyun-oss (%s)", inputURI); } for (final URI inputURI : this.prefixes) { - Preconditions.checkArgument("s3".equals(inputURI.getScheme()), "input uri scheme == s3 (%s)", inputURI); + Preconditions.checkArgument("aliyun-oss".equals(inputURI.getScheme()), "input uri scheme == aliyun-oss (%s)", inputURI); } } @@ -138,7 +138,7 @@ protected InputStream openObjectStream(URI object) throws IOException final OSSObject ossObject = client.getObject(bucket, key); if (ossObject == null) { - throw new ISE("Failed to get an s3 object for bucket[%s] and key[%s]", bucket, key); + throw new ISE("Failed to get an aliyun-oss object for bucket[%s] and key[%s]", bucket, key); } return ossObject.getObjectContent(); } @@ -158,7 +158,7 @@ protected InputStream openObjectStream(URI object, long start) throws IOExceptio final OSSObject ossObject = client.getObject(request); if (ossObject == null) { throw new ISE( - "Failed to get an s3 object for bucket[%s], key[%s], and start[%d]", + "Failed to get an aliyun-oss object for bucket[%s], key[%s], and start[%d]", bucket, key, start diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java similarity index 92% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java index 8c5ae866c1cf..0c4dd2200aa2 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java @@ -144,10 +144,6 @@ private void advanceObjectSummary() /** * Checks if a given object is a directory placeholder and should be ignored. - *

- * Adapted from org.jets3t.service.model.StorageObject.isDirectoryPlaceholder(). Does not include the check for - * legacy JetS3t directory placeholder objects, since it is based on content-type, which isn't available in an - * S3ObjectSummary. */ private static boolean isDirectoryPlaceholder(final OSSObjectSummary objectSummary) { @@ -156,11 +152,6 @@ private static boolean isDirectoryPlaceholder(final OSSObjectSummary objectSumma return true; } - // Recognize s3sync.rb directory placeholders by MD5/ETag value. - if ("d66759af42f282e1ba19144df2d405d0".equals(objectSummary.getETag())) { - return true; - } - // Recognize place-holder objects created by the Google Storage console or S3 Organizer Firefox extension. if (objectSummary.getKey().endsWith("_$folder$") && objectSummary.getSize() == 0) { return true; diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java similarity index 88% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java index 6b5c9b625635..3114ff9ab468 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java @@ -54,14 +54,14 @@ public OssDataSegmentArchiver( @Override public DataSegment archive(DataSegment segment) throws SegmentLoadingException { - String targetS3Bucket = archiveConfig.getArchiveBucket(); - String targetS3BaseKey = archiveConfig.getArchiveBaseKey(); + String targetBucket = archiveConfig.getArchiveBucket(); + String targetKey = archiveConfig.getArchiveBaseKey(); final DataSegment archived = move( segment, ImmutableMap.of( - "bucket", targetS3Bucket, - "baseKey", targetS3BaseKey + "bucket", targetBucket, + "baseKey", targetKey ) ); if (sameLoadSpec(segment, archived)) { @@ -73,14 +73,14 @@ public DataSegment archive(DataSegment segment) throws SegmentLoadingException @Override public DataSegment restore(DataSegment segment) throws SegmentLoadingException { - String targetS3Bucket = restoreConfig.getBucket(); - String targetS3BaseKey = restoreConfig.getBaseKey(); + String targetBucket = restoreConfig.getBucket(); + String targetKey = restoreConfig.getBaseKey(); final DataSegment restored = move( segment, ImmutableMap.of( - "bucket", targetS3Bucket, - "baseKey", targetS3BaseKey + "bucket", targetBucket, + "baseKey", targetKey ) ); diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java similarity index 100% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java similarity index 77% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java index d2abd3ce6176..c31d89557551 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java @@ -61,19 +61,19 @@ public void kill(DataSegment segment) throws SegmentLoadingException { try { Map loadSpec = segment.getLoadSpec(); - String s3Bucket = MapUtils.getString(loadSpec, "bucket"); - String s3Path = MapUtils.getString(loadSpec, "key"); - String s3DescriptorPath = DataSegmentKiller.descriptorPath(s3Path); + String bucket = MapUtils.getString(loadSpec, "bucket"); + String path = MapUtils.getString(loadSpec, "key"); + String descriptorPath = DataSegmentKiller.descriptorPath(path); - if (client.doesObjectExist(s3Bucket, s3Path)) { - log.info("Removing index file[s3://%s/%s] from s3!", s3Bucket, s3Path); - client.deleteObject(s3Bucket, s3Path); + if (client.doesObjectExist(bucket, path)) { + log.info("Removing index file[aliyun-oss://%s/%s] from aliyun OSS!", bucket, path); + client.deleteObject(bucket, path); } // descriptor.json is a file to store segment metadata in deep storage. This file is deprecated and not stored // anymore, but we still delete them if exists. - if (client.doesObjectExist(s3Bucket, s3DescriptorPath)) { - log.info("Removing descriptor file[s3://%s/%s] from s3!", s3Bucket, s3DescriptorPath); - client.deleteObject(s3Bucket, s3DescriptorPath); + if (client.doesObjectExist(bucket, descriptorPath)) { + log.info("Removing descriptor file[aliyun-oss://%s/%s] from aliyun OSS!", bucket, descriptorPath); + client.deleteObject(bucket, descriptorPath); } } catch (OSSException e) { @@ -86,9 +86,9 @@ public void killAll() throws IOException { if (segmentPusherConfig.getBucket() == null || segmentPusherConfig.getBaseKey() == null) { throw new ISE( - "Cannot delete all segment from S3 Deep Storage since druid.storage.bucket and druid.storage.baseKey are not both set."); + "Cannot delete all segment from aliyun OSS Deep Storage since druid.storage.bucket and druid.storage.baseKey are not both set."); } - log.info("Deleting all segment files from s3 location [bucket: '%s' prefix: '%s']", + log.info("Deleting all segment files from aliyun OSS location [bucket: '%s' prefix: '%s']", segmentPusherConfig.getBucket(), segmentPusherConfig.getBaseKey() ); try { @@ -101,7 +101,7 @@ public void killAll() throws IOException ); } catch (Exception e) { - log.error("Error occurred while deleting segment files from s3. Error: %s", e.getMessage()); + log.error("Error occurred while deleting segment files from aliyun OSS. Error: %s", e.getMessage()); throw new IOException(e); } } diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java similarity index 82% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java index 05f96a38ce8b..c92052085f0a 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java @@ -73,7 +73,7 @@ public DataSegment move(DataSegment segment, Map targetLoadSpec) final String targetBucket = MapUtils.getString(targetLoadSpec, "bucket"); final String targetKey = MapUtils.getString(targetLoadSpec, "baseKey"); - final String targetS3Path = OssUtils.constructSegmentPath( + final String targetPath = OssUtils.constructSegmentPath( targetKey, DataSegmentPusher.getDefaultStorageDir(segment, false) ); @@ -81,11 +81,11 @@ public DataSegment move(DataSegment segment, Map targetLoadSpec) if (targetBucket.isEmpty()) { throw new SegmentLoadingException("Target OSS bucket is not specified"); } - if (targetS3Path.isEmpty()) { + if (targetPath.isEmpty()) { throw new SegmentLoadingException("Target OSS baseKey is not specified"); } - safeMove(bucket, key, targetBucket, targetS3Path); + safeMove(bucket, key, targetBucket, targetPath); return segment.withLoadSpec( ImmutableMap.builder() @@ -103,7 +103,7 @@ public boolean apply(String input) ) ) .put("bucket", targetBucket) - .put("key", targetS3Path) + .put("key", targetPath) .build() ); } @@ -113,24 +113,24 @@ public boolean apply(String input) } private void safeMove( - final String s3Bucket, - final String s3Path, - final String targetS3Bucket, - final String targetS3Path + final String srcBucket, + final String srcPath, + final String targetBucket, + final String targetPath ) throws SegmentLoadingException { try { OssUtils.retry( () -> { final String copyMsg = StringUtils.format( - "[s3://%s/%s] to [s3://%s/%s]", - s3Bucket, - s3Path, - targetS3Bucket, - targetS3Path + "[aliyun-oss://%s/%s] to [aliyun-oss://%s/%s]", + srcBucket, + srcPath, + targetBucket, + targetPath ); try { - selfCheckingMove(s3Bucket, targetS3Bucket, s3Path, targetS3Path, copyMsg); + selfCheckingMove(srcBucket, targetBucket, srcPath, targetPath, copyMsg); return null; } catch (OSSException | IOException | SegmentLoadingException e) { @@ -150,7 +150,7 @@ private void safeMove( /** * Copies an object and after that checks that the object is present at the target location, via a separate API call. * If it is not, an exception is thrown, and the object is not deleted at the old location. This "paranoic" check - * is added after it was observed that S3 may report a successful move, and the object is not found at the target + * is added after it was observed that oss may report a successful move, and the object is not found at the target * location. */ private void selfCheckingMove( @@ -162,7 +162,7 @@ private void selfCheckingMove( ) throws IOException, SegmentLoadingException { if (srcBucket.equals(dstBucket) && srcPath.equals(dstPath)) { - log.info("No need to move file[s3://%s/%s] onto itself", srcBucket, srcPath); + log.info("No need to move file[aliyun-oss://%s/%s] onto itself", srcBucket, srcPath); return; } if (client.doesObjectExist(srcBucket, srcPath)) { @@ -174,14 +174,14 @@ private void selfCheckingMove( // keyCount is still zero. if (listResult.getObjectSummaries().size() == 0) { // should never happen - throw new ISE("Unable to list object [s3://%s/%s]", srcBucket, srcPath); + throw new ISE("Unable to list object [aliyun-oss://%s/%s]", srcBucket, srcPath); } final OSSObjectSummary objectSummary = listResult.getObjectSummaries().get(0); if (objectSummary.getStorageClass() != null && objectSummary.getStorageClass().equals(StorageClass.IA.name())) { throw new OSSException( StringUtils.format( - "Cannot move file[oss://%s/%s] of storage class glacier, skipping.", + "Cannot move file[aliyun-oss://%s/%s] of storage class glacier, skipping.", srcBucket, srcPath ) @@ -206,7 +206,7 @@ private void selfCheckingMove( // ensure object exists in target location if (client.doesObjectExist(dstBucket, dstPath)) { log.info( - "Not moving file [s3://%s/%s], already present in target location [s3://%s/%s]", + "Not moving file [aliyun-oss://%s/%s], already present in target location [aliyun-oss://%s/%s]", srcBucket, srcPath, dstBucket, @@ -221,26 +221,26 @@ private void selfCheckingMove( } } - private void deleteWithRetriesSilent(final String s3Bucket, final String s3Path) + private void deleteWithRetriesSilent(final String bucket, final String path) { try { - deleteWithRetries(s3Bucket, s3Path); + deleteWithRetries(bucket, path); } catch (Exception e) { - log.error(e, "Failed to delete file [s3://%s/%s], giving up", s3Bucket, s3Path); + log.error(e, "Failed to delete file [aliyun-oss://%s/%s], giving up", bucket, path); } } - private void deleteWithRetries(final String s3Bucket, final String s3Path) throws Exception + private void deleteWithRetries(final String bucket, final String path) throws Exception { RetryUtils.retry( () -> { try { - client.deleteObject(s3Bucket, s3Path); + client.deleteObject(bucket, path); return null; } catch (Exception e) { - log.info(e, "Error while trying to delete [s3://%s/%s]", s3Bucket, s3Path); + log.info(e, "Error while trying to delete [aliyun-oss://%s/%s]", bucket, path); throw e; } }, diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java similarity index 95% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java index b4e1cfb94b1d..c76fc7dbe9ff 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java @@ -64,9 +64,9 @@ public class OssDataSegmentPuller implements URIDataPuller protected final OSS client; @Inject - public OssDataSegmentPuller(OSS s3Client) + public OssDataSegmentPuller(OSS client) { - this.client = s3Client; + this.client = client; } FileUtils.FileCopyResult getSegmentFiles(final CloudObjectLocation ossCoords, final File outDir) @@ -173,7 +173,7 @@ public String getName() } /** - * Returns an input stream for a s3 object. The returned input stream is not thread-safe. + * Returns an input stream for an OSS object. The returned input stream is not thread-safe. */ @Override public InputStream openInputStream() throws IOException @@ -199,14 +199,14 @@ public void close() throws IOException }; } catch (OSSException e) { - throw new IOE(e, "Could not load S3 URI [%s]", uri); + throw new IOE(e, "Could not load OSS URI [%s]", uri); } } @Override public OutputStream openOutputStream() { - throw new UOE("Cannot stream S3 output"); + throw new UOE("Cannot stream OSS output"); } @Override @@ -236,7 +236,7 @@ public long getLastModified() @Override public boolean delete() { - throw new UOE("Cannot delete S3 items anonymously. jetS3t doesn't support authenticated deletes easily."); + throw new UOE("Cannot delete OSS items anonymously. jetS3t doesn't support authenticated deletes easily."); } }; } @@ -299,7 +299,7 @@ private boolean isObjectInBucket(final CloudObjectLocation coords) throws Segmen ); } catch (OSSException | IOException e) { - throw new SegmentLoadingException(e, "S3 fail! Key[%s]", coords); + throw new SegmentLoadingException(e, "aliyun-oss fail! Key[%s]", coords); } catch (Exception e) { throw new RuntimeException(e); diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java similarity index 92% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java index 8429433bdca7..022121631653 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java @@ -77,21 +77,21 @@ public List getAllowedPropertyPrefixesForHadoop() public DataSegment push(final File indexFilesDir, final DataSegment inSegment, final boolean useUniquePath) throws IOException { - final String s3Path = OssUtils.constructSegmentPath(config.getBaseKey(), getStorageDir(inSegment, useUniquePath)); + final String path = OssUtils.constructSegmentPath(config.getBaseKey(), getStorageDir(inSegment, useUniquePath)); - log.debug("Copying segment[%s] to S3 at location[%s]", inSegment.getId(), s3Path); + log.debug("Copying segment[%s] to OSS at location[%s]", inSegment.getId(), path); final File zipOutFile = File.createTempFile("druid", "index.zip"); final long indexSize = CompressionUtils.zip(indexFilesDir, zipOutFile); final DataSegment outSegment = inSegment.withSize(indexSize) - .withLoadSpec(makeLoadSpec(config.getBucket(), s3Path)) + .withLoadSpec(makeLoadSpec(config.getBucket(), path)) .withBinaryVersion(SegmentUtils.getVersionFromDir(indexFilesDir)); try { return OssUtils.retry( () -> { - OssUtils.uploadFileIfPossible(client, config.getDisableAcl(), config.getBucket(), s3Path, zipOutFile); + OssUtils.uploadFileIfPossible(client, config.getDisableAcl(), config.getBucket(), path, zipOutFile); return outSegment; } diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfig.java similarity index 100% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfig.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfig.java diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java similarity index 95% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java index 5b917084db3a..e701349f7356 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java @@ -25,13 +25,13 @@ /** * Stores the configuration for options related to reading - * input data from Amazon S3 into Druid + * input data from aliyun-oss into Druid */ public class OssInputDataConfig { /** * The maximum number of input files matching a given prefix to retrieve - * from Amazon S3 at a time. + * from aliyun-oss at a time. */ @JsonProperty @Min(1) diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java similarity index 100% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java similarity index 84% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java index 8afa754a3357..39eb48d15d6a 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java @@ -20,7 +20,6 @@ package org.apache.druid.storage.aliyun; import com.aliyun.oss.OSS; -import com.aliyun.oss.OSSClientBuilder; import com.fasterxml.jackson.core.Version; import com.fasterxml.jackson.databind.Module; import com.google.common.collect.ImmutableList; @@ -41,9 +40,9 @@ */ public class OssStorageDruidModule implements DruidModule { - public static final String SCHEME = "oss"; - public static final String SCHEME_S3N = "oss_3n"; - public static final String SCHEME_S3_ZIP = "oss_zip"; + public static final String SCHEME = "aliyun-oss"; + public static final String SCHEME_S3N = "aliyun-oss_3n"; + public static final String SCHEME_S3_ZIP = "aliyun-oss_zip"; @Override public List getJacksonModules() @@ -97,10 +96,9 @@ public void configure(Binder binder) .in(LazySingleton.class); Binders.dataSegmentPusherBinder(binder).addBinding(SCHEME).to(OssDataSegmentPusher.class).in(LazySingleton.class); JsonConfigProvider.bind(binder, "druid.storage", OssInputDataConfig.class); - JsonConfigProvider.bind(binder, "druid.storage", OssInputSourceConfig.class); + JsonConfigProvider.bind(binder, "druid.storage.aliyun-oss", OssInputSourceConfig.class); JsonConfigProvider.bind(binder, "druid.storage", OssDataSegmentPusherConfig.class); JsonConfigProvider.bind(binder, "druid.storage", OssDataSegmentArchiverConfig.class); - JsonConfigProvider.bind(binder, "druid.storage", OssStorageConfig.class); Binders.taskLogsBinder(binder).addBinding(SCHEME).to(OssTaskLogs.class); JsonConfigProvider.bind(binder, "druid.indexer.logs", OssTaskLogsConfig.class); @@ -109,11 +107,8 @@ public void configure(Binder binder) @Provides @LazySingleton - public OSS getOSS(OssInputSourceConfig inputSourceConfig) + public OSS initializeOssClient(OssInputSourceConfig inputSourceConfig) { - return new OSSClientBuilder().build(inputSourceConfig.getEndpoint(), - inputSourceConfig.getAccessKeyId().getPassword(), - inputSourceConfig.getSecretAccessKey().getPassword() - ); + return inputSourceConfig.buildClient(); } } diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java similarity index 94% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java index 0d4fb3626d20..a145af65d03e 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java @@ -36,11 +36,11 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.util.Arrays; +import java.util.Collections; import java.util.Date; /** - * Provides task logs archived on S3. + * Provides task logs archived on aliyun-oss. */ public class OssTaskLogs implements TaskLogs { @@ -103,7 +103,7 @@ public InputStream openStream() throws IOException } final GetObjectRequest request = new GetObjectRequest(config.getBucket(), taskKey); - request.setMatchingETagConstraints(Arrays.asList(objectMetadata.getETag())); + request.setMatchingETagConstraints(Collections.singletonList(objectMetadata.getETag())); request.setRange(start, end); return client.getObject(request).getObjectContent(); @@ -166,7 +166,7 @@ String getTaskLogKey(String taskid, String filename) public void killAll() throws IOException { log.info( - "Deleting all task logs from s3 location [bucket: '%s' prefix: '%s'].", + "Deleting all task logs from aliyun-oss location [bucket: '%s' prefix: '%s'].", config.getBucket(), config.getPrefix() ); @@ -179,7 +179,7 @@ public void killAll() throws IOException public void killOlderThan(long timestamp) throws IOException { log.info( - "Deleting all task logs from s3 location [bucket: '%s' prefix: '%s'] older than %s.", + "Deleting all task logs from aliyun-oss location [bucket: '%s' prefix: '%s'] older than %s.", config.getBucket(), config.getPrefix(), new Date(timestamp) diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java similarity index 100% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java similarity index 87% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java index f443e7396e0d..f19d9eea340e 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java @@ -46,11 +46,11 @@ public OssTimestampVersionedDataFinder(OSS client) * Gets the key with the most recently modified timestamp. * `pattern` is evaluated against the entire key AFTER the path given in `uri`. * The substring `pattern` is matched against will have a leading `/` removed. - * For example `s3://some_bucket/some_prefix/some_key` with a URI of `s3://some_bucket/some_prefix` will match against `some_key`. - * `s3://some_bucket/some_prefixsome_key` with a URI of `s3://some_bucket/some_prefix` will match against `some_key` - * `s3://some_bucket/some_prefix//some_key` with a URI of `s3://some_bucket/some_prefix` will match against `/some_key` + * For example `aliyun-oss://some_bucket/some_prefix/some_key` with a URI of `aliyun-oss://some_bucket/some_prefix` will match against `some_key`. + * `aliyun-oss://some_bucket/some_prefixsome_key` with a URI of `aliyun-oss://some_bucket/some_prefix` will match against `some_key` + * `aliyun-oss://some_bucket/some_prefix//some_key` with a URI of `aliyun-oss://some_bucket/some_prefix` will match against `/some_key` * - * @param uri The URI of in the form of `s3://some_bucket/some_key` + * @param uri The URI of in the form of `aliyun-oss://some_bucket/some_key` * @param pattern The pattern matcher to determine if a *key* is of interest, or `null` to match everything. * @return A URI to the most recently modified object which matched the pattern. */ diff --git a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java similarity index 90% rename from extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java index 691b0251ce2b..c06282a64788 100644 --- a/extensions-contrib/aliyun-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java @@ -81,7 +81,7 @@ public boolean apply(Throwable e) }; /** - * Retries S3 operations that fail due to io-related exceptions. Service-level exceptions (access denied, file not + * Retries aliyun-oss operations that fail due to io-related exceptions. Service-level exceptions (access denied, file not * found, etc) are not retried. */ static T retry(Task f) throws Exception @@ -109,7 +109,7 @@ static boolean isObjectInBucketIgnoringPermission( } /** - * Create an iterator over a set of S3 objects specified by a set of prefixes. + * Create an iterator over a set of aliyun-oss objects specified by a set of prefixes. *

* For each provided prefix URI, the iterator will walk through all objects that are in the same bucket as the * provided URI and whose keys start with that URI's path, except for directory placeholders (which will be @@ -127,10 +127,10 @@ public static Iterator objectSummaryIterator( } /** - * Create an {@link URI} from the given {@link S3ObjectSummary}. The result URI is composed as below. + * Create an {@link URI} from the given {@link OSSObjectSummary}. The result URI is composed as below. * * 

-   * {@code s3://{BUCKET_NAME}/{OBJECT_KEY}}
+   * {@code aliyun-oss://{BUCKET_NAME}/{OBJECT_KEY}}
    *
*/ public static URI summaryToUri(OSSObjectSummary object) @@ -172,12 +172,12 @@ public static URI checkURI(URI uri) } /** - * Gets a single {@link S3ObjectSummary} from s3. Since this method might return a wrong object if there are multiple + * Gets a single {@link OSSObjectSummary} from aliyun-oss. Since this method might return a wrong object if there are multiple * objects that match the given key, this method should be used only when it's guaranteed that the given key is unique * in the given bucket. * - * @param client s3 client - * @param bucket s3 bucket + * @param client aliyun-oss client + * @param bucket aliyun-oss bucket * @param key unique key for the object to be retrieved */ public static OSSObjectSummary getSingleObjectSummary(OSS client, String bucket, String key) @@ -203,11 +203,11 @@ public static OSSObjectSummary getSingleObjectSummary(OSS client, String bucket, } /** - * Delete the files from S3 in a specified bucket, matching a specified prefix and filter + * Delete the files from aliyun-oss in a specified bucket, matching a specified prefix and filter * - * @param client s3 client - * @param config specifies the configuration to use when finding matching files in S3 to delete - * @param bucket s3 bucket + * @param client aliyun-oss client + * @param config specifies the configuration to use when finding matching files in aliyun-oss to delete + * @param bucket aliyun-oss bucket * @param prefix the file prefix * @param filter function which returns true if the prefix file found should be deleted and false otherwise. * @throws Exception @@ -261,12 +261,12 @@ private static void deleteBucketKeys( } /** - * Uploads a file to S3 if possible. First trying to set ACL to give the bucket owner full control of the file before uploading. + * Uploads a file to aliyun-oss if possible. First trying to set ACL to give the bucket owner full control of the file before uploading. * * @param client aliyun OSS client * @param disableAcl true if ACL shouldn't be set for the file * @param key The key under which to store the new object. - * @param file The path of the file to upload to Amazon S3. + * @param file The path of the file to upload to aliyun-oss. */ static void uploadFileIfPossible( OSS client, diff --git a/extensions-contrib/aliyun-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/aliyun-oss-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule similarity index 100% rename from extensions-contrib/aliyun-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule rename to extensions-contrib/aliyun-oss-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java similarity index 77% rename from extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java rename to extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java index 3f1f08242422..fba1d494ba5c 100644 --- a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java @@ -54,8 +54,8 @@ public class ObjectSummaryIteratorTest public void testSingleObject() { test( - ImmutableList.of("oss://b/foo/baz"), - ImmutableList.of("oss://b/foo/baz"), + ImmutableList.of("aliyun-oss://b/foo/baz"), + ImmutableList.of("aliyun-oss://b/foo/baz"), 5 ); } @@ -65,13 +65,13 @@ public void testMultiObjectOneKeyAtATime() { test( ImmutableList.of( - "oss://b/foo/bar1", - "oss://b/foo/bar2", - "oss://b/foo/bar3", - "oss://b/foo/bar4", - "oss://b/foo/baz" + "aliyun-oss://b/foo/bar1", + "aliyun-oss://b/foo/bar2", + "aliyun-oss://b/foo/bar3", + "aliyun-oss://b/foo/bar4", + "aliyun-oss://b/foo/baz" ), - ImmutableList.of("oss://b/foo/"), + ImmutableList.of("aliyun-oss://b/foo/"), 1 ); } @@ -81,13 +81,13 @@ public void testMultiObjectTwoKeysAtATime() { test( ImmutableList.of( - "oss://b/foo/bar1", - "oss://b/foo/bar2", - "oss://b/foo/bar3", - "oss://b/foo/bar4", - "oss://b/foo/baz" + "aliyun-oss://b/foo/bar1", + "aliyun-oss://b/foo/bar2", + "aliyun-oss://b/foo/bar3", + "aliyun-oss://b/foo/bar4", + "aliyun-oss://b/foo/baz" ), - ImmutableList.of("oss://b/foo/"), + ImmutableList.of("aliyun-oss://b/foo/"), 2 ); } @@ -97,13 +97,13 @@ public void testMultiObjectTenKeysAtATime() { test( ImmutableList.of( - "oss://b/foo/bar1", - "oss://b/foo/bar2", - "oss://b/foo/bar3", - "oss://b/foo/bar4", - "oss://b/foo/baz" + "aliyun-oss://b/foo/bar1", + "aliyun-oss://b/foo/bar2", + "aliyun-oss://b/foo/bar3", + "aliyun-oss://b/foo/bar4", + "aliyun-oss://b/foo/baz" ), - ImmutableList.of("oss://b/foo/"), + ImmutableList.of("aliyun-oss://b/foo/"), 10 ); } @@ -112,8 +112,8 @@ public void testMultiObjectTenKeysAtATime() public void testPrefixInMiddleOfKey() { test( - ImmutableList.of("oss://b/foo/bar1", "oss://b/foo/bar2", "oss://b/foo/bar3", "oss://b/foo/bar4"), - ImmutableList.of("oss://b/foo/bar"), + ImmutableList.of("aliyun-oss://b/foo/bar1", "aliyun-oss://b/foo/bar2", "aliyun-oss://b/foo/bar3", "aliyun-oss://b/foo/bar4"), + ImmutableList.of("aliyun-oss://b/foo/bar"), 10 ); } @@ -123,14 +123,14 @@ public void testNoPath() { test( ImmutableList.of( - "oss://b/foo", - "oss://b/foo/bar1", - "oss://b/foo/bar2", - "oss://b/foo/bar3", - "oss://b/foo/bar4", - "oss://b/foo/baz" + "aliyun-oss://b/foo", + "aliyun-oss://b/foo/bar1", + "aliyun-oss://b/foo/bar2", + "aliyun-oss://b/foo/bar3", + "aliyun-oss://b/foo/bar4", + "aliyun-oss://b/foo/baz" ), - ImmutableList.of("oss://b"), + ImmutableList.of("aliyun-oss://b"), 10 ); } @@ -140,14 +140,14 @@ public void testSlashPath() { test( ImmutableList.of( - "oss://b/foo", - "oss://b/foo/bar1", - "oss://b/foo/bar2", - "oss://b/foo/bar3", - "oss://b/foo/bar4", - "oss://b/foo/baz" + "aliyun-oss://b/foo", + "aliyun-oss://b/foo/bar1", + "aliyun-oss://b/foo/bar2", + "aliyun-oss://b/foo/bar3", + "aliyun-oss://b/foo/bar4", + "aliyun-oss://b/foo/baz" ), - ImmutableList.of("oss://b/"), + ImmutableList.of("aliyun-oss://b/"), 10 ); } @@ -157,7 +157,7 @@ public void testDifferentBucket() { test( ImmutableList.of(), - ImmutableList.of("oss://bx/foo/"), + ImmutableList.of("aliyun-oss://bx/foo/"), 10 ); } @@ -167,13 +167,13 @@ public void testWithMultiplePrefixesReturningAllNonEmptyObjectsStartingWithOneOf { test( ImmutableList.of( - "oss://b/foo/bar1", - "oss://b/foo/bar2", - "oss://b/foo/bar3", - "oss://b/foo/bar4", - "oss://b/foo/baz" + "aliyun-oss://b/foo/bar1", + "aliyun-oss://b/foo/bar2", + "aliyun-oss://b/foo/bar3", + "aliyun-oss://b/foo/bar4", + "aliyun-oss://b/foo/baz" ), - ImmutableList.of("oss://b/foo/bar", "oss://b/foo/baz"), + ImmutableList.of("aliyun-oss://b/foo/bar", "aliyun-oss://b/foo/baz"), 10 ); } diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java similarity index 98% rename from extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java rename to extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java index 7a2b1db9c412..ae44c7e6a4c2 100644 --- a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java @@ -56,7 +56,7 @@ public Object findInjectableValue( } } ) - .registerModule(new SimpleModule("s3-archive-test-module").registerSubtypes(OssLoadSpec.class)); + .registerModule(new SimpleModule("aliyun-oss-archive-test-module").registerSubtypes(OssLoadSpec.class)); private static final OssDataSegmentArchiverConfig ARCHIVER_CONFIG = new OssDataSegmentArchiverConfig() { @Override diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java similarity index 96% rename from extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java rename to extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java index 0547f3895377..82c6617a4c95 100644 --- a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java @@ -37,7 +37,7 @@ import java.io.IOException; import java.net.URI; -import java.util.Arrays; +import java.util.Collections; @RunWith(EasyMockRunner.class) public class OssDataSegmentKillerTest extends EasyMockSupport @@ -46,7 +46,7 @@ public class OssDataSegmentKillerTest extends EasyMockSupport private static final String KEY_2 = "key2"; private static final String TEST_BUCKET = "test_bucket"; private static final String TEST_PREFIX = "test_prefix"; - private static final URI PREFIX_URI = URI.create(StringUtils.format("s3://%s/%s", TEST_BUCKET, TEST_PREFIX)); + private static final URI PREFIX_URI = URI.create(StringUtils.format("aliyun-oss://%s/%s", TEST_BUCKET, TEST_PREFIX)); private static final long TIME_0 = 0L; private static final long TIME_1 = 1L; private static final int MAX_KEYS = 1; @@ -99,9 +99,9 @@ public void test_killAll_noException_deletesAllSegments() throws IOException ); DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET); - deleteRequest1.setKeys(Arrays.asList(KEY_1)); + deleteRequest1.setKeys(Collections.singletonList(KEY_1)); DeleteObjectsRequest deleteRequest2 = new DeleteObjectsRequest(TEST_BUCKET); - deleteRequest2.setKeys(Arrays.asList(KEY_2)); + deleteRequest2.setKeys(Collections.singletonList(KEY_2)); OssTestUtils.mockClientDeleteObjects( client, @@ -136,7 +136,7 @@ public void test_killAll_recoverableExceptionWhenListingObjects_deletesAllSegmen ); DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET); - deleteRequest1.setKeys(Arrays.asList(KEY_1)); + deleteRequest1.setKeys(Collections.singletonList(KEY_1)); OssTestUtils.mockClientDeleteObjects( client, diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java similarity index 100% rename from extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java rename to extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java similarity index 99% rename from extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java rename to extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java index 1c1a1a853273..726d551b970d 100644 --- a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java @@ -75,7 +75,7 @@ public void testSimpleGetVersion() throws IOException EasyMock.replay(s3Client); - String version = puller.getVersion(URI.create(StringUtils.format("oss://%s/%s", bucket, objectSummary.getKey()))); + String version = puller.getVersion(URI.create(StringUtils.format("aliyun-oss://%s/%s", bucket, objectSummary.getKey()))); EasyMock.verify(s3Client); diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java similarity index 100% rename from extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java rename to extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java similarity index 100% rename from extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java rename to extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java similarity index 94% rename from extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java rename to extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java index 5c770ecd20d0..ba3f08e84bbd 100644 --- a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java @@ -118,7 +118,7 @@ public void test_killAll_noException_deletesAllTaskLogs() throws IOException config.setPrefix(TEST_PREFIX); OssInputDataConfig inputDataConfig = new OssInputDataConfig(); inputDataConfig.setMaxListingLength(MAX_KEYS); - OssTaskLogs taskLogs = new taskLogs(ossClient, config, inputDataConfig, timeSupplier); + OssTaskLogs taskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); taskLogs.killAll(); EasyMock.verify(ossClient, timeSupplier); @@ -152,7 +152,7 @@ public void test_killAll_recoverableExceptionWhenDeletingObjects_deletesAllTaskL config.setPrefix(TEST_PREFIX); OssInputDataConfig inputDataConfig = new OssInputDataConfig(); inputDataConfig.setMaxListingLength(MAX_KEYS); - OssTaskLogs taskLogs = new taskLogs(ossClient, config, inputDataConfig, timeSupplier); + OssTaskLogs taskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); taskLogs.killAll(); EasyMock.verify(ossClient, timeSupplier); @@ -186,7 +186,7 @@ public void test_killAll_nonrecoverableExceptionWhenListingObjects_doesntDeleteA config.setPrefix(TEST_PREFIX); OssInputDataConfig inputDataConfig = new OssInputDataConfig(); inputDataConfig.setMaxListingLength(MAX_KEYS); - OssTaskLogs taskLogs = new taskLogs(ossClient, config, inputDataConfig, timeSupplier); + OssTaskLogs taskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); taskLogs.killAll(); } catch (IOException e) { @@ -222,7 +222,7 @@ public void test_killOlderThan_noException_deletesOnlyTaskLogsOlderThan() throws config.setPrefix(TEST_PREFIX); OssInputDataConfig inputDataConfig = new OssInputDataConfig(); inputDataConfig.setMaxListingLength(MAX_KEYS); - OssTaskLogs taskLogs = new taskLogs(ossClient, config, inputDataConfig, timeSupplier); + OssTaskLogs taskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); taskLogs.killOlderThan(TIME_NOW); EasyMock.verify(ossClient, timeSupplier); @@ -255,7 +255,7 @@ public void test_killOlderThan_recoverableExceptionWhenListingObjects_deletesAll config.setPrefix(TEST_PREFIX); OssInputDataConfig inputDataConfig = new OssInputDataConfig(); inputDataConfig.setMaxListingLength(MAX_KEYS); - OssTaskLogs taskLogs = new taskLogs(ossClient, config, inputDataConfig, timeSupplier); + OssTaskLogs taskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); taskLogs.killOlderThan(TIME_NOW); EasyMock.verify(ossClient, timeSupplier); @@ -288,7 +288,7 @@ public void test_killOlderThan_nonrecoverableExceptionWhenListingObjects_doesntD config.setPrefix(TEST_PREFIX); OssInputDataConfig inputDataConfig = new OssInputDataConfig(); inputDataConfig.setMaxListingLength(MAX_KEYS); - OssTaskLogs taskLogs = new taskLogs(ossClient, config, inputDataConfig, timeSupplier); + OssTaskLogs taskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); taskLogs.killOlderThan(TIME_NOW); } catch (IOException e) { @@ -324,7 +324,7 @@ private List testPushInternal(boolean disableAcl, String ownerId, String config.setBucket(TEST_BUCKET); CurrentTimeMillisSupplier timeSupplier = new CurrentTimeMillisSupplier(); OssInputDataConfig inputDataConfig = new OssInputDataConfig(); - OssTaskLogs taskLogs = new taskLogs(ossClient, config, inputDataConfig, timeSupplier); + OssTaskLogs taskLogs = new OssTaskLogs(ossClient, config, inputDataConfig, timeSupplier); String taskId = "index_test-datasource_2019-06-18T13:30:28.887Z"; File logFile = tempFolder.newFile("test_log_file"); diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTestUtils.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTestUtils.java similarity index 100% rename from extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTestUtils.java rename to extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTestUtils.java diff --git a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java similarity index 90% rename from extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java rename to extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java index 048b5db94e52..3cebb8fcb394 100644 --- a/extensions-contrib/aliyun-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java @@ -69,11 +69,11 @@ public void testSimpleLatestVersion() EasyMock.replay(client); - URI latest = finder.getLatestVersion(URI.create(StringUtils.format("oss://%s/%s", bucket, keyPrefix)), pattern); + URI latest = finder.getLatestVersion(URI.create(StringUtils.format("aliyun-oss://%s/%s", bucket, keyPrefix)), pattern); EasyMock.verify(client); - URI expected = URI.create(StringUtils.format("oss://%s/%s", bucket, object1.getKey())); + URI expected = URI.create(StringUtils.format("aliyun-oss://%s/%s", bucket, object1.getKey())); Assert.assertEquals(expected, latest); } @@ -98,7 +98,7 @@ public void testMissing() EasyMock.replay(oss); - URI latest = finder.getLatestVersion(URI.create(StringUtils.format("oss://%s/%s", bucket, keyPrefix)), pattern); + URI latest = finder.getLatestVersion(URI.create(StringUtils.format("aliyun-oss://%s/%s", bucket, keyPrefix)), pattern); EasyMock.verify(oss); @@ -133,11 +133,11 @@ public void testFindSelf() EasyMock.replay(s3Client); - URI latest = finder.getLatestVersion(URI.create(StringUtils.format("oss://%s/%s", bucket, keyPrefix)), pattern); + URI latest = finder.getLatestVersion(URI.create(StringUtils.format("aliyun-oss://%s/%s", bucket, keyPrefix)), pattern); EasyMock.verify(s3Client); - URI expected = URI.create(StringUtils.format("oss://%s/%s", bucket, object0.getKey())); + URI expected = URI.create(StringUtils.format("aliyun-oss://%s/%s", bucket, object0.getKey())); Assert.assertEquals(expected, latest); } @@ -167,11 +167,11 @@ public void testFindExact() EasyMock.replay(s3Client); - URI latest = finder.getLatestVersion(URI.create(StringUtils.format("oss://%s/%s", bucket, object0.getKey())), null); + URI latest = finder.getLatestVersion(URI.create(StringUtils.format("aliyun-oss://%s/%s", bucket, object0.getKey())), null); EasyMock.verify(s3Client); - URI expected = URI.create(StringUtils.format("oss://%s/%s", bucket, object0.getKey())); + URI expected = URI.create(StringUtils.format("aliyun-oss://%s/%s", bucket, object0.getKey())); Assert.assertEquals(expected, latest); } From 23cd7a1275e07924189ce4858e6e425898fae699 Mon Sep 17 00:00:00 2001 From: frank chen Date: Sat, 16 May 2020 21:38:51 +0800 Subject: [PATCH 04/24] modify config path --- ...SourceConfig.java => OssClientConfig.java} | 6 ++--- .../data/input/aliyun/OssInputSource.java | 6 ++--- .../aliyun/OssDataSegmentArchiver.java | 6 ++--- .../storage/aliyun/OssDataSegmentKiller.java | 10 +++---- .../storage/aliyun/OssDataSegmentMover.java | 7 ++--- .../storage/aliyun/OssDataSegmentPusher.java | 12 ++++----- .../druid/storage/aliyun/OssLoadSpec.java | 2 +- ...usherConfig.java => OssStorageConfig.java} | 27 +++++-------------- .../storage/aliyun/OssStorageDruidModule.java | 22 +++++++-------- .../druid/storage/aliyun/OssTaskLogs.java | 2 +- .../apache/druid/storage/aliyun/OssUtils.java | 9 ++----- .../aliyun/OssDataSegmentArchiverTest.java | 10 +++---- .../aliyun/OssDataSegmentKillerTest.java | 10 +++---- .../aliyun/OssDataSegmentMoverTest.java | 10 +++---- .../OssDataSegmentPusherConfigTest.java | 22 +++++++-------- .../aliyun/OssDataSegmentPusherTest.java | 6 ++--- 16 files changed, 72 insertions(+), 95 deletions(-) rename extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/{OssInputSourceConfig.java => OssClientConfig.java} (95%) rename extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/{OssDataSegmentPusherConfig.java => OssStorageConfig.java} (75%) diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java similarity index 95% rename from extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java index 0c3e1fb06c12..839ffcb94ba4 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java @@ -32,10 +32,10 @@ * Contains properties for aliyun-oss input source. * Properties can be specified by ingestionSpec which will override system default. */ -public class OssInputSourceConfig +public class OssClientConfig { @JsonCreator - public OssInputSourceConfig( + public OssClientConfig( @JsonProperty("endpoint") String endpoint, @JsonProperty("accessKey") PasswordProvider accessKey, @JsonProperty("secretKey") PasswordProvider secretKey @@ -95,7 +95,7 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - OssInputSourceConfig that = (OssInputSourceConfig) o; + OssClientConfig that = (OssClientConfig) o; return Objects.equals(accessKey, that.accessKey) && Objects.equals(secretKey, that.secretKey) && Objects.equals(endpoint, that.endpoint); diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java index 45b3e4c9a33e..8a187b808c39 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java @@ -52,7 +52,7 @@ public class OssInputSource extends CloudObjectInputSource { private final Supplier clientSupplier; @JsonProperty("properties") - private final OssInputSourceConfig inputSourceConfig; + private final OssClientConfig inputSourceConfig; private final OssInputDataConfig inputDataConfig; /** @@ -74,7 +74,7 @@ public OssInputSource( @JsonProperty("uris") @Nullable List uris, @JsonProperty("prefixes") @Nullable List prefixes, @JsonProperty("objects") @Nullable List objects, - @JsonProperty("properties") @Nullable OssInputSourceConfig inputSourceConfig + @JsonProperty("properties") @Nullable OssClientConfig inputSourceConfig ) { super(OssStorageDruidModule.SCHEME, uris, prefixes, objects); @@ -95,7 +95,7 @@ public OssInputSource( @Nullable @JsonProperty("properties") - public OssInputSourceConfig getOssInputSourceConfig() + public OssClientConfig getOssInputSourceConfig() { return inputSourceConfig; } diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java index 3114ff9ab468..6bc063824700 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java @@ -34,7 +34,7 @@ public class OssDataSegmentArchiver extends OssDataSegmentMover implements DataSegmentArchiver { private final OssDataSegmentArchiverConfig archiveConfig; - private final OssDataSegmentPusherConfig restoreConfig; + private final OssStorageConfig restoreConfig; private final ObjectMapper mapper; @Inject @@ -42,7 +42,7 @@ public OssDataSegmentArchiver( @Json ObjectMapper mapper, OSS client, OssDataSegmentArchiverConfig archiveConfig, - OssDataSegmentPusherConfig restoreConfig + OssStorageConfig restoreConfig ) { super(client, restoreConfig); @@ -74,7 +74,7 @@ public DataSegment archive(DataSegment segment) throws SegmentLoadingException public DataSegment restore(DataSegment segment) throws SegmentLoadingException { String targetBucket = restoreConfig.getBucket(); - String targetKey = restoreConfig.getBaseKey(); + String targetKey = restoreConfig.getPrefix(); final DataSegment restored = move( segment, diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java index c31d89557551..4635eb419f88 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java @@ -41,13 +41,13 @@ public class OssDataSegmentKiller implements DataSegmentKiller private static final Logger log = new Logger(OssDataSegmentKiller.class); private final OSS client; - private final OssDataSegmentPusherConfig segmentPusherConfig; + private final OssStorageConfig segmentPusherConfig; private final OssInputDataConfig inputDataConfig; @Inject public OssDataSegmentKiller( OSS client, - OssDataSegmentPusherConfig segmentPusherConfig, + OssStorageConfig segmentPusherConfig, OssInputDataConfig inputDataConfig ) { @@ -84,19 +84,19 @@ public void kill(DataSegment segment) throws SegmentLoadingException @Override public void killAll() throws IOException { - if (segmentPusherConfig.getBucket() == null || segmentPusherConfig.getBaseKey() == null) { + if (segmentPusherConfig.getBucket() == null || segmentPusherConfig.getPrefix() == null) { throw new ISE( "Cannot delete all segment from aliyun OSS Deep Storage since druid.storage.bucket and druid.storage.baseKey are not both set."); } log.info("Deleting all segment files from aliyun OSS location [bucket: '%s' prefix: '%s']", - segmentPusherConfig.getBucket(), segmentPusherConfig.getBaseKey() + segmentPusherConfig.getBucket(), segmentPusherConfig.getPrefix() ); try { OssUtils.deleteObjectsInPath( client, inputDataConfig, segmentPusherConfig.getBucket(), - segmentPusherConfig.getBaseKey(), + segmentPusherConfig.getPrefix(), Predicates.alwaysTrue() ); } diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java index c92052085f0a..b0fd110bb402 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java @@ -50,12 +50,12 @@ public class OssDataSegmentMover implements DataSegmentMover private static final Logger log = new Logger(OssDataSegmentMover.class); private final OSS client; - private final OssDataSegmentPusherConfig config; + private final OssStorageConfig config; @Inject public OssDataSegmentMover( OSS client, - OssDataSegmentPusherConfig config + OssStorageConfig config ) { this.client = client; @@ -189,9 +189,6 @@ private void selfCheckingMove( } else { log.info("Moving file %s", copyMsg); final CopyObjectRequest copyRequest = new CopyObjectRequest(srcBucket, srcPath, dstBucket, dstPath); - if (!config.getDisableAcl()) { - //copyRequest.set(OssUtils.grantFullControlToBucketOwner(client, dstBucket)); - } client.copyObject(copyRequest); if (!client.doesObjectExist(dstBucket, dstPath)) { throw new IOE( diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java index 022121631653..e01fcfbbcdc4 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java @@ -42,12 +42,12 @@ public class OssDataSegmentPusher implements DataSegmentPusher private static final EmittingLogger log = new EmittingLogger(OssDataSegmentPusher.class); private final OSS client; - private final OssDataSegmentPusherConfig config; + private final OssStorageConfig config; @Inject public OssDataSegmentPusher( OSS client, - OssDataSegmentPusherConfig config + OssStorageConfig config ) { this.client = client; @@ -57,7 +57,7 @@ public OssDataSegmentPusher( @Override public String getPathForHadoop() { - return StringUtils.format("%s/%s", config.getBucket(), config.getBaseKey()); + return StringUtils.format("%s/%s", config.getBucket(), config.getPrefix()); } @Deprecated @@ -77,7 +77,7 @@ public List getAllowedPropertyPrefixesForHadoop() public DataSegment push(final File indexFilesDir, final DataSegment inSegment, final boolean useUniquePath) throws IOException { - final String path = OssUtils.constructSegmentPath(config.getBaseKey(), getStorageDir(inSegment, useUniquePath)); + final String path = OssUtils.constructSegmentPath(config.getPrefix(), getStorageDir(inSegment, useUniquePath)); log.debug("Copying segment[%s] to OSS at location[%s]", inSegment.getId(), path); @@ -91,7 +91,7 @@ public DataSegment push(final File indexFilesDir, final DataSegment inSegment, f try { return OssUtils.retry( () -> { - OssUtils.uploadFileIfPossible(client, config.getDisableAcl(), config.getBucket(), path, zipOutFile); + OssUtils.uploadFileIfPossible(client, config.getBucket(), path, zipOutFile); return outSegment; } @@ -123,7 +123,7 @@ private Map makeLoadSpec(String bucket, String key) { return ImmutableMap.of( "type", - "oss_zip", + OssStorageDruidModule.SCHEME_ZIP, "bucket", bucket, "key", diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java index e35a48e34499..e0f5299a6123 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java @@ -33,7 +33,7 @@ /** * */ -@JsonTypeName(OssStorageDruidModule.SCHEME_S3_ZIP) +@JsonTypeName(OssStorageDruidModule.SCHEME_ZIP) public class OssLoadSpec implements LoadSpec { private final String bucket; diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java similarity index 75% rename from extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfig.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java index 908586571051..d863a06b49cd 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java @@ -26,35 +26,25 @@ /** * */ -public class OssDataSegmentPusherConfig +public class OssStorageConfig { @JsonProperty private String bucket = ""; @JsonProperty - private String baseKey = ""; - - @JsonProperty - private boolean disableAcl = true; + private String prefix = ""; @JsonProperty @Min(1) private int maxListingLength = 1024; - // use s3n by default for backward compatibility public void setBucket(String bucket) { this.bucket = bucket; } - - public void setBaseKey(String baseKey) + public void setPrefix(String prefix) { - this.baseKey = baseKey; - } - - public void setDisableAcl(boolean disableAcl) - { - this.disableAcl = disableAcl; + this.prefix = prefix; } public void setMaxListingLength(int maxListingLength) @@ -67,14 +57,9 @@ public String getBucket() return bucket; } - public String getBaseKey() - { - return baseKey; - } - - public boolean getDisableAcl() + public String getPrefix() { - return disableAcl; + return prefix; } public int getMaxListingLength() diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java index 39eb48d15d6a..36244f2b8dc7 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java @@ -27,7 +27,7 @@ import com.google.inject.Provides; import com.google.inject.multibindings.MapBinder; import org.apache.druid.data.SearchableVersionedDataFinder; -import org.apache.druid.data.input.aliyun.OssInputSourceConfig; +import org.apache.druid.data.input.aliyun.OssClientConfig; import org.apache.druid.guice.Binders; import org.apache.druid.guice.JsonConfigProvider; import org.apache.druid.guice.LazySingleton; @@ -42,7 +42,7 @@ public class OssStorageDruidModule implements DruidModule { public static final String SCHEME = "aliyun-oss"; public static final String SCHEME_S3N = "aliyun-oss_3n"; - public static final String SCHEME_S3_ZIP = "aliyun-oss_zip"; + public static final String SCHEME_ZIP = "aliyun-oss_zip"; @Override public List getJacksonModules() @@ -83,31 +83,31 @@ public void configure(Binder binder) .to(OssTimestampVersionedDataFinder.class) .in(LazySingleton.class); Binders.dataSegmentKillerBinder(binder) - .addBinding(SCHEME_S3_ZIP) + .addBinding(SCHEME_ZIP) .to(OssDataSegmentKiller.class) .in(LazySingleton.class); Binders.dataSegmentMoverBinder(binder) - .addBinding(SCHEME_S3_ZIP) + .addBinding(SCHEME_ZIP) .to(OssDataSegmentMover.class) .in(LazySingleton.class); Binders.dataSegmentArchiverBinder(binder) - .addBinding(SCHEME_S3_ZIP) + .addBinding(SCHEME_ZIP) .to(OssDataSegmentArchiver.class) .in(LazySingleton.class); Binders.dataSegmentPusherBinder(binder).addBinding(SCHEME).to(OssDataSegmentPusher.class).in(LazySingleton.class); - JsonConfigProvider.bind(binder, "druid.storage", OssInputDataConfig.class); - JsonConfigProvider.bind(binder, "druid.storage.aliyun-oss", OssInputSourceConfig.class); - JsonConfigProvider.bind(binder, "druid.storage", OssDataSegmentPusherConfig.class); - JsonConfigProvider.bind(binder, "druid.storage", OssDataSegmentArchiverConfig.class); + JsonConfigProvider.bind(binder, "druid.aliyun-oss", OssClientConfig.class); + JsonConfigProvider.bind(binder, "druid.aliyun-oss.storage", OssInputDataConfig.class); + JsonConfigProvider.bind(binder, "druid.aliyun-oss.storage", OssStorageConfig.class); + JsonConfigProvider.bind(binder, "druid.aliyun-oss.storage", OssDataSegmentArchiverConfig.class); Binders.taskLogsBinder(binder).addBinding(SCHEME).to(OssTaskLogs.class); - JsonConfigProvider.bind(binder, "druid.indexer.logs", OssTaskLogsConfig.class); + JsonConfigProvider.bind(binder, "druid.indexer.logs.aliyun-oss", OssTaskLogsConfig.class); binder.bind(OssTaskLogs.class).in(LazySingleton.class); } @Provides @LazySingleton - public OSS initializeOssClient(OssInputSourceConfig inputSourceConfig) + public OSS initializeOssClient(OssClientConfig inputSourceConfig) { return inputSourceConfig.buildClient(); } diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java index a145af65d03e..953a533a4bc9 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java @@ -146,7 +146,7 @@ private void pushTaskFile(final File logFile, String taskKey) throws IOException try { OssUtils.retry( () -> { - OssUtils.uploadFileIfPossible(client, config.getDisableAcl(), config.getBucket(), taskKey, logFile); + OssUtils.uploadFileIfPossible(client, config.getBucket(), taskKey, logFile); return null; } ); diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java index c06282a64788..4b371940896d 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java @@ -165,8 +165,8 @@ public static String extractKey(URI uri) public static URI checkURI(URI uri) { - if (uri.getScheme().equalsIgnoreCase(OssStorageDruidModule.SCHEME_S3_ZIP)) { - uri = URI.create(SCHEME + uri.toString().substring(OssStorageDruidModule.SCHEME_S3_ZIP.length())); + if (uri.getScheme().equalsIgnoreCase(OssStorageDruidModule.SCHEME_ZIP)) { + uri = URI.create(SCHEME + uri.toString().substring(OssStorageDruidModule.SCHEME_ZIP.length())); } return CloudObjectLocation.validateUriScheme(SCHEME, uri); } @@ -270,7 +270,6 @@ private static void deleteBucketKeys( */ static void uploadFileIfPossible( OSS client, - boolean disableAcl, String bucket, String key, File file @@ -280,9 +279,5 @@ static void uploadFileIfPossible( log.info("Pushing [%s] to bucket[%s] and key[%s].", file, bucket, key); client.putObject(putObjectRequest); - - if (!disableAcl) { - client.setObjectAcl(bucket, key, OssUtils.grantFullControlToBucketOwner(client, bucket)); - } } } diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java index ae44c7e6a4c2..02a88ec885c9 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverTest.java @@ -71,7 +71,7 @@ public String getArchiveBaseKey() return "archive_base_key"; } }; - private static final OssDataSegmentPusherConfig PUSHER_CONFIG = new OssDataSegmentPusherConfig(); + private static final OssStorageConfig PUSHER_CONFIG = new OssStorageConfig(); private static final OSS OSS_CLIENT = EasyMock.createStrictMock(OSSClient.class); private static final OssDataSegmentPuller PULLER = new OssDataSegmentPuller(OSS_CLIENT); private static final DataSegment SOURCE_SEGMENT = DataSegment @@ -83,7 +83,7 @@ public String getArchiveBaseKey() .version("version") .loadSpec(ImmutableMap.of( "type", - OssStorageDruidModule.SCHEME_S3_ZIP, + OssStorageDruidModule.SCHEME_ZIP, OssDataSegmentPuller.BUCKET, "source_bucket", OssDataSegmentPuller.KEY, @@ -95,7 +95,7 @@ public String getArchiveBaseKey() @BeforeClass public static void setUpStatic() { - PUSHER_CONFIG.setBaseKey("push_base"); + PUSHER_CONFIG.setPrefix("push_base"); PUSHER_CONFIG.setBucket("push_bucket"); } @@ -105,7 +105,7 @@ public void testSimpleArchive() throws Exception final DataSegment archivedSegment = SOURCE_SEGMENT .withLoadSpec(ImmutableMap.of( "type", - OssStorageDruidModule.SCHEME_S3_ZIP, + OssStorageDruidModule.SCHEME_ZIP, OssDataSegmentPuller.BUCKET, ARCHIVER_CONFIG.getArchiveBucket(), OssDataSegmentPuller.KEY, @@ -152,7 +152,7 @@ public void testSimpleRestore() throws Exception final DataSegment archivedSegment = SOURCE_SEGMENT .withLoadSpec(ImmutableMap.of( "type", - OssStorageDruidModule.SCHEME_S3_ZIP, + OssStorageDruidModule.SCHEME_ZIP, OssDataSegmentPuller.BUCKET, ARCHIVER_CONFIG.getArchiveBucket(), OssDataSegmentPuller.KEY, diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java index 82c6617a4c95..f7183c6a752d 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java @@ -56,7 +56,7 @@ public class OssDataSegmentKillerTest extends EasyMockSupport @Mock private OSS client; @Mock - private OssDataSegmentPusherConfig segmentPusherConfig; + private OssStorageConfig segmentPusherConfig; @Mock private OssInputDataConfig inputDataConfig; @@ -67,7 +67,7 @@ public void test_killAll_accountConfigWithNullBucketAndBaseKey_throwsISEExceptio { EasyMock.expect(segmentPusherConfig.getBucket()).andReturn(null); EasyMock.expectLastCall().atLeastOnce(); - EasyMock.expect(segmentPusherConfig.getBaseKey()).andReturn(null); + EasyMock.expect(segmentPusherConfig.getPrefix()).andReturn(null); EasyMock.expectLastCall().anyTimes(); boolean thrownISEException = false; @@ -111,7 +111,7 @@ public void test_killAll_noException_deletesAllSegments() throws IOException EasyMock.expect(segmentPusherConfig.getBucket()).andReturn(TEST_BUCKET); EasyMock.expectLastCall().anyTimes(); - EasyMock.expect(segmentPusherConfig.getBaseKey()).andReturn(TEST_PREFIX); + EasyMock.expect(segmentPusherConfig.getPrefix()).andReturn(TEST_PREFIX); EasyMock.expectLastCall().anyTimes(); EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS); @@ -146,7 +146,7 @@ public void test_killAll_recoverableExceptionWhenListingObjects_deletesAllSegmen EasyMock.expect(segmentPusherConfig.getBucket()).andReturn(TEST_BUCKET); EasyMock.expectLastCall().anyTimes(); - EasyMock.expect(segmentPusherConfig.getBaseKey()).andReturn(TEST_PREFIX); + EasyMock.expect(segmentPusherConfig.getPrefix()).andReturn(TEST_PREFIX); EasyMock.expectLastCall().anyTimes(); EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS); @@ -184,7 +184,7 @@ public void test_killAll_nonrecoverableExceptionWhenListingObjects_deletesAllSeg EasyMock.expect(segmentPusherConfig.getBucket()).andReturn(TEST_BUCKET); EasyMock.expectLastCall().anyTimes(); - EasyMock.expect(segmentPusherConfig.getBaseKey()).andReturn(TEST_PREFIX); + EasyMock.expect(segmentPusherConfig.getPrefix()).andReturn(TEST_PREFIX); EasyMock.expectLastCall().anyTimes(); EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS); diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java index c8d8cf4a6fc5..a1ff64950406 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java @@ -67,7 +67,7 @@ public class OssDataSegmentMoverTest public void testMove() throws Exception { MockClient mockClient = new MockClient(); - OssDataSegmentMover mover = new OssDataSegmentMover(mockClient, new OssDataSegmentPusherConfig()); + OssDataSegmentMover mover = new OssDataSegmentMover(mockClient, new OssStorageConfig()); mockClient.putObject( "main", @@ -92,7 +92,7 @@ public void testMove() throws Exception public void testMoveNoop() throws Exception { MockClient mockS3Client = new MockClient(); - OssDataSegmentMover mover = new OssDataSegmentMover(mockS3Client, new OssDataSegmentPusherConfig()); + OssDataSegmentMover mover = new OssDataSegmentMover(mockS3Client, new OssStorageConfig()); mockS3Client.putObject( "archive", @@ -118,7 +118,7 @@ public void testMoveNoop() throws Exception public void testMoveException() throws Exception { MockClient mockS3Client = new MockClient(); - OssDataSegmentMover mover = new OssDataSegmentMover(mockS3Client, new OssDataSegmentPusherConfig()); + OssDataSegmentMover mover = new OssDataSegmentMover(mockS3Client, new OssStorageConfig()); mover.move( SOURCE_SEGMENT, @@ -130,7 +130,7 @@ public void testMoveException() throws Exception public void testIgnoresGoneButAlreadyMoved() throws Exception { MockClient mockS3Client = new MockClient(); - OssDataSegmentMover mover = new OssDataSegmentMover(mockS3Client, new OssDataSegmentPusherConfig()); + OssDataSegmentMover mover = new OssDataSegmentMover(mockS3Client, new OssStorageConfig()); mover.move(new DataSegment( "test", Intervals.of("2013-01-01/2013-01-02"), @@ -153,7 +153,7 @@ public void testIgnoresGoneButAlreadyMoved() throws Exception public void testFailsToMoveMissing() throws Exception { MockClient client = new MockClient(); - OssDataSegmentMover mover = new OssDataSegmentMover(client, new OssDataSegmentPusherConfig()); + OssDataSegmentMover mover = new OssDataSegmentMover(client, new OssStorageConfig()); mover.move(new DataSegment( "test", Intervals.of("2013-01-01/2013-01-02"), diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java index e462a5c6d68b..c2a23fd3bbc4 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java @@ -39,21 +39,21 @@ public class OssDataSegmentPusherConfigTest @Test public void testSerialization() throws IOException { - String jsonConfig = "{\"bucket\":\"bucket1\",\"baseKey\":\"dataSource1\"," - + "\"disableAcl\":false,\"maxListingLength\":2000}"; + String jsonConfig = "{\"bucket\":\"bucket1\",\"prefix\":\"dataSource1\"," + + "\"maxListingLength\":2000}"; - OssDataSegmentPusherConfig config = JSON_MAPPER.readValue(jsonConfig, OssDataSegmentPusherConfig.class); + OssStorageConfig config = JSON_MAPPER.readValue(jsonConfig, OssStorageConfig.class); Assert.assertEquals(jsonConfig, JSON_MAPPER.writeValueAsString(config)); } @Test public void testSerializationWithDefaults() throws IOException { - String jsonConfig = "{\"bucket\":\"bucket1\",\"baseKey\":\"dataSource1\"}"; - String expectedJsonConfig = "{\"bucket\":\"bucket1\",\"baseKey\":\"dataSource1\"," - + "\"disableAcl\":true,\"maxListingLength\":1024}"; + String jsonConfig = "{\"bucket\":\"bucket1\",\"prefix\":\"dataSource1\"}"; + String expectedJsonConfig = "{\"bucket\":\"bucket1\",\"prefix\":\"dataSource1\"," + + "\"maxListingLength\":1024}"; - OssDataSegmentPusherConfig config = JSON_MAPPER.readValue(jsonConfig, OssDataSegmentPusherConfig.class); + OssStorageConfig config = JSON_MAPPER.readValue(jsonConfig, OssStorageConfig.class); Assert.assertEquals(expectedJsonConfig, JSON_MAPPER.writeValueAsString(config)); } @@ -63,12 +63,12 @@ public void testSerializationValidatingMaxListingLength() throws IOException Locale old = Locale.getDefault(); Locale.setDefault(Locale.ENGLISH); try { - String jsonConfig = "{\"bucket\":\"bucket1\",\"baseKey\":\"dataSource1\"," - + "\"disableAcl\":false,\"maxListingLength\":-1}"; + String jsonConfig = "{\"bucket\":\"bucket1\",\"prefix\":\"dataSource1\"," + + "\"maxListingLength\":-1}"; Validator validator = Validation.buildDefaultValidatorFactory().getValidator(); - OssDataSegmentPusherConfig config = JSON_MAPPER.readValue(jsonConfig, OssDataSegmentPusherConfig.class); - Set> violations = validator.validate(config); + OssStorageConfig config = JSON_MAPPER.readValue(jsonConfig, OssStorageConfig.class); + Set> violations = validator.validate(config); Assert.assertEquals(1, violations.size()); ConstraintViolation violation = Iterators.getOnlyElement(violations.iterator()); Assert.assertEquals("must be greater than or equal to 1", violation.getMessage()); diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java index e413e49529e7..1c8c1159a0ff 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java @@ -84,9 +84,9 @@ private void testPushInternal(boolean useUniquePath, String matcher) throws Exce EasyMock.replay(client); - OssDataSegmentPusherConfig config = new OssDataSegmentPusherConfig(); + OssStorageConfig config = new OssStorageConfig(); config.setBucket("bucket"); - config.setBaseKey("key"); + config.setPrefix("key"); OssDataSegmentPusher pusher = new OssDataSegmentPusher(client, config); @@ -118,7 +118,7 @@ private void testPushInternal(boolean useUniquePath, String matcher) throws Exce segment.getLoadSpec().get("key").toString(), Pattern.compile(matcher).matcher(segment.getLoadSpec().get("key").toString()).matches() ); - Assert.assertEquals("oss_zip", segment.getLoadSpec().get("type")); + Assert.assertEquals("aliyun-oss_zip", segment.getLoadSpec().get("type")); EasyMock.verify(client); } From 399d53623b413910cc1377371a58008f96dc9eee Mon Sep 17 00:00:00 2001 From: frank chen Date: Sat, 16 May 2020 21:49:29 +0800 Subject: [PATCH 05/24] add doc --- .../aliyun-oss-extensions.md | 54 +++++++++++++++++++ .../data/input/aliyun/OssClientConfig.java | 1 + .../druid/data/input/aliyun/OssEntity.java | 3 ++ .../data/input/aliyun/OssInputSource.java | 3 ++ .../aliyun/OssInputSourceDruidModule.java | 1 + .../aliyun/OssFirehoseDruidModule.java | 2 +- .../aliyun/StaticOssFirehoseFactory.java | 1 + .../storage/aliyun/ObjectSummaryIterator.java | 2 + .../aliyun/OssDataSegmentArchiver.java | 4 +- .../aliyun/OssDataSegmentArchiverConfig.java | 3 ++ .../storage/aliyun/OssDataSegmentKiller.java | 2 +- .../storage/aliyun/OssDataSegmentMover.java | 3 ++ .../storage/aliyun/OssDataSegmentPuller.java | 1 + .../storage/aliyun/OssDataSegmentPusher.java | 3 ++ .../storage/aliyun/OssInputDataConfig.java | 1 + .../druid/storage/aliyun/OssLoadSpec.java | 2 +- .../storage/aliyun/OssStorageConfig.java | 2 +- .../storage/aliyun/OssStorageDruidModule.java | 2 +- .../druid/storage/aliyun/OssTaskLogs.java | 1 + .../storage/aliyun/OssTaskLogsConfig.java | 2 +- .../OssTimestampVersionedDataFinder.java | 3 ++ .../apache/druid/storage/aliyun/OssUtils.java | 2 +- 22 files changed, 90 insertions(+), 8 deletions(-) create mode 100644 docs/development/extensions-contrib/aliyun-oss-extensions.md diff --git a/docs/development/extensions-contrib/aliyun-oss-extensions.md b/docs/development/extensions-contrib/aliyun-oss-extensions.md new file mode 100644 index 000000000000..1d6076f3d0cb --- /dev/null +++ b/docs/development/extensions-contrib/aliyun-oss-extensions.md @@ -0,0 +1,54 @@ +--- +id: aliyun-oss +title: "Aliyun OSS" +--- + + + + +To use this Apache Druid extension, make sure to [include](../../development/extensions.md#loading-extensions) `druid-aliyun-oss-extensions` extension. + +## Deep Storage + +[Aliyun](https://www.aliyun.com) is the 3rd largest cloud infrasture provider in the world. It also provides its own storage solution, that is OSS, [Object Storage Service](https://www.aliyun.com/product/oss). + +To use aliyun OSS, first config the OSS as below + +|Property|Description|Possible Values|Default| +|--------|---------------|-----------|-------| +|`druid.aliyun-oss.accessKey`|the 'AccessKey ID' of your account which can be used to access the bucket||Must be set.| +|`druid.aliyun-oss.secretKey`|the 'AccessKey Secret' of your account which can be used to access the bucket| |Must be set. | +|`druid.aliyun-oss.endpoint`|the endpoint url of your oss storage| |Must be set.| + +if you want to use OSS as deep storage, use the configurations below + +|Property|Description|Possible Values|Default| +|--------|---------------|-----------|-------| +|`druid.storage.type`| | aliyun-oss|Must be set.| +|`druid.aliyun-oss.storage.bucket`||storage bucket name.|Must be set.| +|`druid.aliyun-oss.storage.prefix`|a prefix string prepended to the file names for the segments published to aliyun oss deep storage| druid/segments | | + +To save index logs to OSS, apply the configurations below: + +|Property|Description|Possible Values|Default| +|--------|---------------|-----------|-------| +|`druid.indexer.logs.type`| | aliyun-oss|Must be set.| +|`druid.indexer.logs.aliyun-oss.bucket`|the bucket used to keep logs||Must be set.| +|`druid.indexer.logs.aliyun-oss.prefix`|a prefix string prepended to the log files| || diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java index 839ffcb94ba4..21b93eec8034 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java @@ -31,6 +31,7 @@ /** * Contains properties for aliyun-oss input source. * Properties can be specified by ingestionSpec which will override system default. + * @author frank.chen021@outlook.com */ public class OssClientConfig { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java index aacaec3d3eb1..6aa7e86081ba 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java @@ -34,6 +34,9 @@ import java.io.InputStream; import java.net.URI; +/** + * @author frank.chen021@outlook.com + */ public class OssEntity extends RetryingInputEntity { private final OSS ossClient; diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java index 8a187b808c39..bb10b689ce76 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java @@ -48,6 +48,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +/** + * @author frank.chen021@outlook.com + */ public class OssInputSource extends CloudObjectInputSource { private final Supplier clientSupplier; diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java index 024a0c532d2d..d771c9bc0fff 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java @@ -31,6 +31,7 @@ /** * Druid module to wire up native batch support for aliyun-oss input + * @author frank.chen021@outlook.com */ public class OssInputSourceDruidModule implements DruidModule { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java index 527b87378ff6..1589a2a7a89c 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java @@ -29,7 +29,7 @@ import java.util.List; /** - * + * @author frank.chen021@outlook.com */ public class OssFirehoseDruidModule implements DruidModule { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java index c7cc821ba94b..8ffe472dde6b 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java @@ -52,6 +52,7 @@ /** * Builds firehoses that read from a predefined list of aliyun-oss objects and then dry up. + * @author frank.chen021@outlook.com */ public class StaticOssFirehoseFactory extends PrefetchableTextFilesFirehoseFactory { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java index 0c4dd2200aa2..79f616ff30f8 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java @@ -35,6 +35,8 @@ *

* As required by the specification of that method, this iterator is computed incrementally in batches of * {@code maxListLength}. The first call is made at the same time the iterator is constructed. + * + * @author frank.chen021@outlook.com */ public class ObjectSummaryIterator implements Iterator { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java index 6bc063824700..8b215574e665 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java @@ -30,7 +30,9 @@ import org.apache.druid.segment.loading.SegmentLoadingException; import org.apache.druid.timeline.DataSegment; - +/** + * @author frank.chen021@outlook.com + */ public class OssDataSegmentArchiver extends OssDataSegmentMover implements DataSegmentArchiver { private final OssDataSegmentArchiverConfig archiveConfig; diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java index dd659044e9fe..c6efc3a89bfb 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java @@ -21,6 +21,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; +/** + * @author frank.chen021@outlook.com + */ public class OssDataSegmentArchiverConfig { @JsonProperty diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java index 4635eb419f88..c6c957885c3b 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java @@ -34,7 +34,7 @@ import java.util.Map; /** - * + * @author frank.chen021@outlook.com */ public class OssDataSegmentKiller implements DataSegmentKiller { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java index b0fd110bb402..bd49ef55d170 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java @@ -45,6 +45,9 @@ import java.io.IOException; import java.util.Map; +/** + * @author frank.chen021@outlook.com + */ public class OssDataSegmentMover implements DataSegmentMover { private static final Logger log = new Logger(OssDataSegmentMover.class); diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java index c76fc7dbe9ff..6d32eae4f05d 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java @@ -53,6 +53,7 @@ /** * A data segment puller that also hanldes URI data pulls. + * @author frank.chen021@outlook.com */ public class OssDataSegmentPuller implements URIDataPuller { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java index e01fcfbbcdc4..a9d52b45747b 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java @@ -37,6 +37,9 @@ import java.util.List; import java.util.Map; +/** + * @author frank.chen021@outlook.com + */ public class OssDataSegmentPusher implements DataSegmentPusher { private static final EmittingLogger log = new EmittingLogger(OssDataSegmentPusher.class); diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java index e701349f7356..e82112d6a3f8 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java @@ -26,6 +26,7 @@ /** * Stores the configuration for options related to reading * input data from aliyun-oss into Druid + * @author frank.chen021@outlook.com */ public class OssInputDataConfig { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java index e0f5299a6123..7027a5294b4c 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java @@ -31,7 +31,7 @@ import java.io.File; /** - * + * @author frank.chen021@outlook.com */ @JsonTypeName(OssStorageDruidModule.SCHEME_ZIP) public class OssLoadSpec implements LoadSpec diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java index d863a06b49cd..cf731360ea23 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java @@ -24,7 +24,7 @@ import javax.validation.constraints.Min; /** - * + * @author frank.chen021@outlook.com */ public class OssStorageConfig { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java index 36244f2b8dc7..6d6406641655 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java @@ -36,7 +36,7 @@ import java.util.List; /** - * + * @author frank.chen021@outlook.com */ public class OssStorageDruidModule implements DruidModule { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java index 953a533a4bc9..e706f6d09a81 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java @@ -41,6 +41,7 @@ /** * Provides task logs archived on aliyun-oss. + * @author frank.chen021@outlook.com */ public class OssTaskLogs implements TaskLogs { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java index e8a5f36a199e..91a9814ce675 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java @@ -25,7 +25,7 @@ import javax.validation.constraints.NotNull; /** - * + * @author frank.chen021@outlook.com */ public class OssTaskLogsConfig { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java index f19d9eea340e..151ec994be66 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java @@ -32,6 +32,9 @@ import java.util.Iterator; import java.util.regex.Pattern; +/** + * @author frank.chen021@outlook.com + */ public class OssTimestampVersionedDataFinder extends OssDataSegmentPuller implements SearchableVersionedDataFinder { private static final int MAX_LISTING_KEYS = 1000; diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java index 4b371940896d..72ef586b7c5c 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java @@ -46,7 +46,7 @@ import java.util.List; /** - * + * @author frank.chen021@outlook.com */ public class OssUtils { From 38a893ab53494cc3582ed2413aa99a0fb1314eaa Mon Sep 17 00:00:00 2001 From: frank chen Date: Sat, 16 May 2020 23:56:54 +0800 Subject: [PATCH 06/24] add aliyun-oss extension to project --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 5c4cf156d105..6d1686f498b2 100644 --- a/pom.xml +++ b/pom.xml @@ -189,6 +189,7 @@ extensions-contrib/tdigestsketch extensions-contrib/influxdb-emitter extensions-contrib/gce-extensions + extensions-contrib/aliyun-oss-extensions distribution From 2860f94b6304fb349b94b2da24cea6f57466bb60 Mon Sep 17 00:00:00 2001 From: frank chen Date: Wed, 20 May 2020 11:35:25 +0800 Subject: [PATCH 07/24] remove descriptor deletion code to avoid warning message output by aliyun client --- .../apache/druid/storage/aliyun/OssDataSegmentKiller.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java index c6c957885c3b..3de8d41e3eb9 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java @@ -63,18 +63,11 @@ public void kill(DataSegment segment) throws SegmentLoadingException Map loadSpec = segment.getLoadSpec(); String bucket = MapUtils.getString(loadSpec, "bucket"); String path = MapUtils.getString(loadSpec, "key"); - String descriptorPath = DataSegmentKiller.descriptorPath(path); if (client.doesObjectExist(bucket, path)) { log.info("Removing index file[aliyun-oss://%s/%s] from aliyun OSS!", bucket, path); client.deleteObject(bucket, path); } - // descriptor.json is a file to store segment metadata in deep storage. This file is deprecated and not stored - // anymore, but we still delete them if exists. - if (client.doesObjectExist(bucket, descriptorPath)) { - log.info("Removing descriptor file[aliyun-oss://%s/%s] from aliyun OSS!", bucket, descriptorPath); - client.deleteObject(bucket, descriptorPath); - } } catch (OSSException e) { throw new SegmentLoadingException(e, "Couldn't kill segment[%s]: [%s]", segment.getId(), e); From a97f6cacdb36d5eb3a0bc92a00a54584862fa955 Mon Sep 17 00:00:00 2001 From: frank chen Date: Wed, 20 May 2020 20:50:45 +0800 Subject: [PATCH 08/24] fix warnings reported by lgtm-com --- .../src/main/java/org/apache/druid/storage/aliyun/OssUtils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java index 72ef586b7c5c..797f5c8a055f 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java @@ -264,7 +264,6 @@ private static void deleteBucketKeys( * Uploads a file to aliyun-oss if possible. First trying to set ACL to give the bucket owner full control of the file before uploading. * * @param client aliyun OSS client - * @param disableAcl true if ACL shouldn't be set for the file * @param key The key under which to store the new object. * @param file The path of the file to upload to aliyun-oss. */ From 57597f6c8b4cac7803fb27f56d679af8b9b623a5 Mon Sep 17 00:00:00 2001 From: frank chen Date: Wed, 20 May 2020 23:04:34 +0800 Subject: [PATCH 09/24] fix ci warnings Signed-off-by: frank chen --- .../org/apache/druid/data/input/aliyun/OssClientConfig.java | 1 - .../java/org/apache/druid/data/input/aliyun/OssEntity.java | 3 --- .../org/apache/druid/data/input/aliyun/OssInputSource.java | 3 --- .../druid/data/input/aliyun/OssInputSourceDruidModule.java | 1 - .../apache/druid/firehose/aliyun/OssFirehoseDruidModule.java | 3 --- .../apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java | 1 - .../org/apache/druid/storage/aliyun/ObjectSummaryIterator.java | 1 - .../apache/druid/storage/aliyun/OssDataSegmentArchiver.java | 3 --- .../druid/storage/aliyun/OssDataSegmentArchiverConfig.java | 3 --- .../org/apache/druid/storage/aliyun/OssDataSegmentKiller.java | 3 --- .../org/apache/druid/storage/aliyun/OssDataSegmentMover.java | 3 --- .../org/apache/druid/storage/aliyun/OssDataSegmentPuller.java | 1 - .../org/apache/druid/storage/aliyun/OssDataSegmentPusher.java | 3 --- .../org/apache/druid/storage/aliyun/OssInputDataConfig.java | 1 - .../main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java | 3 --- .../java/org/apache/druid/storage/aliyun/OssStorageConfig.java | 3 --- .../org/apache/druid/storage/aliyun/OssStorageDruidModule.java | 3 --- .../main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java | 1 - .../org/apache/druid/storage/aliyun/OssTaskLogsConfig.java | 3 --- .../druid/storage/aliyun/OssTimestampVersionedDataFinder.java | 3 --- .../main/java/org/apache/druid/storage/aliyun/OssUtils.java | 3 --- 21 files changed, 49 deletions(-) diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java index 21b93eec8034..839ffcb94ba4 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java @@ -31,7 +31,6 @@ /** * Contains properties for aliyun-oss input source. * Properties can be specified by ingestionSpec which will override system default. - * @author frank.chen021@outlook.com */ public class OssClientConfig { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java index 6aa7e86081ba..aacaec3d3eb1 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java @@ -34,9 +34,6 @@ import java.io.InputStream; import java.net.URI; -/** - * @author frank.chen021@outlook.com - */ public class OssEntity extends RetryingInputEntity { private final OSS ossClient; diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java index bb10b689ce76..8a187b808c39 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java @@ -48,9 +48,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -/** - * @author frank.chen021@outlook.com - */ public class OssInputSource extends CloudObjectInputSource { private final Supplier clientSupplier; diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java index d771c9bc0fff..024a0c532d2d 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java @@ -31,7 +31,6 @@ /** * Druid module to wire up native batch support for aliyun-oss input - * @author frank.chen021@outlook.com */ public class OssInputSourceDruidModule implements DruidModule { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java index 1589a2a7a89c..864717657fe8 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/OssFirehoseDruidModule.java @@ -28,9 +28,6 @@ import java.util.List; -/** - * @author frank.chen021@outlook.com - */ public class OssFirehoseDruidModule implements DruidModule { @Override diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java index 8ffe472dde6b..c7cc821ba94b 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java @@ -52,7 +52,6 @@ /** * Builds firehoses that read from a predefined list of aliyun-oss objects and then dry up. - * @author frank.chen021@outlook.com */ public class StaticOssFirehoseFactory extends PrefetchableTextFilesFirehoseFactory { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java index 79f616ff30f8..a9e6b21a92a3 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java @@ -36,7 +36,6 @@ * As required by the specification of that method, this iterator is computed incrementally in batches of * {@code maxListLength}. The first call is made at the same time the iterator is constructed. * - * @author frank.chen021@outlook.com */ public class ObjectSummaryIterator implements Iterator { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java index 8b215574e665..b836876af987 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiver.java @@ -30,9 +30,6 @@ import org.apache.druid.segment.loading.SegmentLoadingException; import org.apache.druid.timeline.DataSegment; -/** - * @author frank.chen021@outlook.com - */ public class OssDataSegmentArchiver extends OssDataSegmentMover implements DataSegmentArchiver { private final OssDataSegmentArchiverConfig archiveConfig; diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java index c6efc3a89bfb..dd659044e9fe 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentArchiverConfig.java @@ -21,9 +21,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; -/** - * @author frank.chen021@outlook.com - */ public class OssDataSegmentArchiverConfig { @JsonProperty diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java index 3de8d41e3eb9..9959b1215c22 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java @@ -33,9 +33,6 @@ import java.io.IOException; import java.util.Map; -/** - * @author frank.chen021@outlook.com - */ public class OssDataSegmentKiller implements DataSegmentKiller { private static final Logger log = new Logger(OssDataSegmentKiller.class); diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java index bd49ef55d170..b0fd110bb402 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java @@ -45,9 +45,6 @@ import java.io.IOException; import java.util.Map; -/** - * @author frank.chen021@outlook.com - */ public class OssDataSegmentMover implements DataSegmentMover { private static final Logger log = new Logger(OssDataSegmentMover.class); diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java index 6d32eae4f05d..c76fc7dbe9ff 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java @@ -53,7 +53,6 @@ /** * A data segment puller that also hanldes URI data pulls. - * @author frank.chen021@outlook.com */ public class OssDataSegmentPuller implements URIDataPuller { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java index a9d52b45747b..e01fcfbbcdc4 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java @@ -37,9 +37,6 @@ import java.util.List; import java.util.Map; -/** - * @author frank.chen021@outlook.com - */ public class OssDataSegmentPusher implements DataSegmentPusher { private static final EmittingLogger log = new EmittingLogger(OssDataSegmentPusher.class); diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java index e82112d6a3f8..e701349f7356 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java @@ -26,7 +26,6 @@ /** * Stores the configuration for options related to reading * input data from aliyun-oss into Druid - * @author frank.chen021@outlook.com */ public class OssInputDataConfig { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java index 7027a5294b4c..155c26fbf3ca 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssLoadSpec.java @@ -30,9 +30,6 @@ import java.io.File; -/** - * @author frank.chen021@outlook.com - */ @JsonTypeName(OssStorageDruidModule.SCHEME_ZIP) public class OssLoadSpec implements LoadSpec { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java index cf731360ea23..e0f930bd958f 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java @@ -23,9 +23,6 @@ import javax.validation.constraints.Min; -/** - * @author frank.chen021@outlook.com - */ public class OssStorageConfig { @JsonProperty diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java index 6d6406641655..a744688b13ed 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java @@ -35,9 +35,6 @@ import java.util.List; -/** - * @author frank.chen021@outlook.com - */ public class OssStorageDruidModule implements DruidModule { public static final String SCHEME = "aliyun-oss"; diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java index e706f6d09a81..953a533a4bc9 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java @@ -41,7 +41,6 @@ /** * Provides task logs archived on aliyun-oss. - * @author frank.chen021@outlook.com */ public class OssTaskLogs implements TaskLogs { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java index 91a9814ce675..3a3e5c37c1b8 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogsConfig.java @@ -24,9 +24,6 @@ import javax.validation.constraints.NotNull; -/** - * @author frank.chen021@outlook.com - */ public class OssTaskLogsConfig { @JsonProperty diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java index 151ec994be66..f19d9eea340e 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java @@ -32,9 +32,6 @@ import java.util.Iterator; import java.util.regex.Pattern; -/** - * @author frank.chen021@outlook.com - */ public class OssTimestampVersionedDataFinder extends OssDataSegmentPuller implements SearchableVersionedDataFinder { private static final int MAX_LISTING_KEYS = 1000; diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java index 797f5c8a055f..8bc13fbd0c19 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java @@ -45,9 +45,6 @@ import java.util.Iterator; import java.util.List; -/** - * @author frank.chen021@outlook.com - */ public class OssUtils { private static final String SCHEME = OssStorageDruidModule.SCHEME; From be7f522c7ada8d6a297885c598a3a3083d956660 Mon Sep 17 00:00:00 2001 From: frank chen Date: Wed, 20 May 2020 23:47:56 +0800 Subject: [PATCH 10/24] fix errors reported by intellj inspection check Signed-off-by: frank chen --- .../druid/storage/aliyun/OssTaskLogsTest.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java index ba3f08e84bbd..7b7bdac86634 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java @@ -46,6 +46,7 @@ import java.io.IOException; import java.net.URI; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -101,9 +102,9 @@ public void test_killAll_noException_deletesAllTaskLogs() throws IOException ); DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET); - deleteRequest1.setKeys(Arrays.asList(KEY_1)); + deleteRequest1.setKeys(Collections.singletonList(KEY_1)); DeleteObjectsRequest deleteRequest2 = new DeleteObjectsRequest(TEST_BUCKET); - deleteRequest2.setKeys(Arrays.asList(KEY_2)); + deleteRequest2.setKeys(Collections.singletonList(KEY_2)); OssTestUtils.mockClientDeleteObjects( ossClient, @@ -138,7 +139,7 @@ public void test_killAll_recoverableExceptionWhenDeletingObjects_deletesAllTaskL ); DeleteObjectsRequest expectedRequest = new DeleteObjectsRequest(TEST_BUCKET); - expectedRequest.setKeys(Arrays.asList(KEY_1)); + expectedRequest.setKeys(Collections.singletonList(KEY_1)); OssTestUtils.mockClientDeleteObjects( ossClient, ImmutableList.of(expectedRequest), @@ -172,7 +173,7 @@ public void test_killAll_nonrecoverableExceptionWhenListingObjects_doesntDeleteA ); DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET); - deleteRequest1.setKeys(Arrays.asList(KEY_1)); + deleteRequest1.setKeys(Collections.singletonList(KEY_1)); OssTestUtils.mockClientDeleteObjects( ossClient, ImmutableList.of(), @@ -211,7 +212,7 @@ public void test_killOlderThan_noException_deletesOnlyTaskLogsOlderThan() throws ); DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET); - deleteRequest1.setKeys(Arrays.asList(KEY_1)); + deleteRequest1.setKeys(Collections.singletonList(KEY_1)); OssTestUtils.mockClientDeleteObjects(ossClient, ImmutableList.of(deleteRequest1), ImmutableMap.of()); @@ -240,7 +241,7 @@ public void test_killOlderThan_recoverableExceptionWhenListingObjects_deletesAll ); DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET); - deleteRequest1.setKeys(Arrays.asList(KEY_1)); + deleteRequest1.setKeys(Collections.singletonList(KEY_1)); OssTestUtils.mockClientDeleteObjects( ossClient, @@ -274,7 +275,7 @@ public void test_killOlderThan_nonrecoverableExceptionWhenListingObjects_doesntD ); DeleteObjectsRequest deleteRequest1 = new DeleteObjectsRequest(TEST_BUCKET); - deleteRequest1.setKeys(Arrays.asList(KEY_1)); + deleteRequest1.setKeys(Collections.singletonList(KEY_1)); OssTestUtils.mockClientDeleteObjects( ossClient, ImmutableList.of(), From 3ff0dcb132c661c998447017cc9c2d4badd9fa33 Mon Sep 17 00:00:00 2001 From: frank chen Date: Wed, 20 May 2020 23:51:35 +0800 Subject: [PATCH 11/24] fix doc spelling check Signed-off-by: frank chen --- docs/development/extensions-contrib/aliyun-oss-extensions.md | 2 +- website/.spelling | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/development/extensions-contrib/aliyun-oss-extensions.md b/docs/development/extensions-contrib/aliyun-oss-extensions.md index 1d6076f3d0cb..c655e42f8b5a 100644 --- a/docs/development/extensions-contrib/aliyun-oss-extensions.md +++ b/docs/development/extensions-contrib/aliyun-oss-extensions.md @@ -27,7 +27,7 @@ To use this Apache Druid extension, make sure to [include](../../development/ext ## Deep Storage -[Aliyun](https://www.aliyun.com) is the 3rd largest cloud infrasture provider in the world. It also provides its own storage solution, that is OSS, [Object Storage Service](https://www.aliyun.com/product/oss). +[Aliyun](https://www.aliyun.com) is the 3rd largest cloud infrastructure provider in the world. It also provides its own storage solution, that is OSS, [Object Storage Service](https://www.aliyun.com/product/oss). To use aliyun OSS, first config the OSS as below diff --git a/website/.spelling b/website/.spelling index 55d11778e6b3..bf2145452892 100644 --- a/website/.spelling +++ b/website/.spelling @@ -1750,3 +1750,8 @@ CVE-2019-17571 CVE-2019-12399 CVE-2018-17196 bin.tar.gz +Aliyun +aliyun +OSS +AccessKey +aliyun-oss \ No newline at end of file From 60c8b9c5456df9dca417df544914090fda098185 Mon Sep 17 00:00:00 2001 From: frank chen Date: Thu, 21 May 2020 00:00:28 +0800 Subject: [PATCH 12/24] fix dependency warnings reported by ci Signed-off-by: frank chen --- .../druid/storage/aliyun/OssDataSegmentKillerTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java index f7183c6a752d..df00b15792f4 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java @@ -19,10 +19,11 @@ package org.apache.druid.storage.aliyun; +import com.aliyun.oss.ClientException; import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSException; import com.aliyun.oss.model.DeleteObjectsRequest; import com.aliyun.oss.model.OSSObjectSummary; -import com.amazonaws.SdkClientException; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.druid.java.util.common.ISE; @@ -50,8 +51,8 @@ public class OssDataSegmentKillerTest extends EasyMockSupport private static final long TIME_0 = 0L; private static final long TIME_1 = 1L; private static final int MAX_KEYS = 1; - private static final Exception RECOVERABLE_EXCEPTION = new SdkClientException(new IOException()); - private static final Exception NON_RECOVERABLE_EXCEPTION = new SdkClientException(new NullPointerException()); + private static final Exception RECOVERABLE_EXCEPTION = new ClientException(new IOException()); + private static final Exception NON_RECOVERABLE_EXCEPTION = new ClientException(new NullPointerException()); @Mock private OSS client; From f931414a21dfcce601e2b6fb6d761f1be06bdfe3 Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 22 May 2020 23:52:56 +0800 Subject: [PATCH 13/24] fix warnings reported by CI Signed-off-by: frank chen --- .../extensions-contrib/aliyun-oss-extensions.md | 6 +++--- extensions-contrib/aliyun-oss-extensions/pom.xml | 5 ----- .../druid/storage/aliyun/OssDataSegmentPusher.java | 3 --- .../org/apache/druid/storage/aliyun/OssUtils.java | 2 +- .../storage/aliyun/OssDataSegmentKillerTest.java | 1 - .../druid/storage/aliyun/OssTaskLogsTest.java | 5 ++--- website/.spelling | 14 ++++++++------ 7 files changed, 14 insertions(+), 22 deletions(-) diff --git a/docs/development/extensions-contrib/aliyun-oss-extensions.md b/docs/development/extensions-contrib/aliyun-oss-extensions.md index c655e42f8b5a..4021d1339b0c 100644 --- a/docs/development/extensions-contrib/aliyun-oss-extensions.md +++ b/docs/development/extensions-contrib/aliyun-oss-extensions.md @@ -29,13 +29,13 @@ To use this Apache Druid extension, make sure to [include](../../development/ext [Aliyun](https://www.aliyun.com) is the 3rd largest cloud infrastructure provider in the world. It also provides its own storage solution, that is OSS, [Object Storage Service](https://www.aliyun.com/product/oss). -To use aliyun OSS, first config the OSS as below +To use aliyun OSS, first config as below |Property|Description|Possible Values|Default| |--------|---------------|-----------|-------| |`druid.aliyun-oss.accessKey`|the 'AccessKey ID' of your account which can be used to access the bucket||Must be set.| |`druid.aliyun-oss.secretKey`|the 'AccessKey Secret' of your account which can be used to access the bucket| |Must be set. | -|`druid.aliyun-oss.endpoint`|the endpoint url of your oss storage| |Must be set.| +|`druid.aliyun-oss.endpoint`|the endpoint url of your OSS storage| |Must be set.| if you want to use OSS as deep storage, use the configurations below @@ -43,7 +43,7 @@ if you want to use OSS as deep storage, use the configurations below |--------|---------------|-----------|-------| |`druid.storage.type`| | aliyun-oss|Must be set.| |`druid.aliyun-oss.storage.bucket`||storage bucket name.|Must be set.| -|`druid.aliyun-oss.storage.prefix`|a prefix string prepended to the file names for the segments published to aliyun oss deep storage| druid/segments | | +|`druid.aliyun-oss.storage.prefix`|a prefix string prepended to the file names for the segments published to Aliyun OSS deep storage| druid/segments | | To save index logs to OSS, apply the configurations below: diff --git a/extensions-contrib/aliyun-oss-extensions/pom.xml b/extensions-contrib/aliyun-oss-extensions/pom.xml index 6a01b9dd370e..0a3e48ed1461 100644 --- a/extensions-contrib/aliyun-oss-extensions/pom.xml +++ b/extensions-contrib/aliyun-oss-extensions/pom.xml @@ -73,11 +73,6 @@ guice provided - - com.google.inject.extensions - guice-assistedinject - ${guice.version} - com.fasterxml.jackson.core jackson-databind diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java index e01fcfbbcdc4..3f2fd2fde62d 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java @@ -116,9 +116,6 @@ public Map makeLoadSpec(URI finalIndexZipFilePath) return makeLoadSpec(finalIndexZipFilePath.getHost(), finalIndexZipFilePath.getPath().substring(1)); } - /** - * Any change in loadSpec need to be reflected {@link org.apache.druid.indexer.JobHelper#getURIFromSegment()} - */ private Map makeLoadSpec(String bucket, String key) { return ImmutableMap.of( diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java index 8bc13fbd0c19..f665c66857cf 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java @@ -110,7 +110,7 @@ static boolean isObjectInBucketIgnoringPermission( *

* For each provided prefix URI, the iterator will walk through all objects that are in the same bucket as the * provided URI and whose keys start with that URI's path, except for directory placeholders (which will be - * ignored). The iterator is computed incrementally by calling {@link OssClientHelper#listObjectsV2} for + * ignored). The iterator is computed incrementally by calling {@link OSS#listObjects} for * each prefix in batches of {@param maxListLength}. The first call is made at the same time the iterator is * constructed. */ diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java index df00b15792f4..deff2a45b2b2 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java @@ -21,7 +21,6 @@ import com.aliyun.oss.ClientException; import com.aliyun.oss.OSS; -import com.aliyun.oss.OSSException; import com.aliyun.oss.model.DeleteObjectsRequest; import com.aliyun.oss.model.OSSObjectSummary; import com.google.common.collect.ImmutableList; diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java index 7b7bdac86634..dc0debe539a0 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java @@ -45,10 +45,9 @@ import java.io.File; import java.io.IOException; import java.net.URI; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; @RunWith(EasyMockRunner.class) public class OssTaskLogsTest extends EasyMockSupport @@ -332,6 +331,6 @@ private List testPushInternal(boolean disableAcl, String ownerId, String taskLogs.pushTaskLog(taskId, logFile); - return aclExpected.getGrants().stream().collect(Collectors.toList()); + return new ArrayList<>(aclExpected.getGrants()); } } diff --git a/website/.spelling b/website/.spelling index bf2145452892..56f34a31afc3 100644 --- a/website/.spelling +++ b/website/.spelling @@ -567,6 +567,13 @@ thriftJar - ../docs/development/extensions-contrib/time-min-max.md timeMax timeMin + - ../docs/development/extensions-contrib/aliyun-oss-extensions.md +Aliyun +aliyun +OSS +AccessKey +aliyun-oss +url - ../docs/development/extensions-core/approximate-histograms.md approxHistogram approxHistogramFold @@ -1749,9 +1756,4 @@ UserGroupInformation CVE-2019-17571 CVE-2019-12399 CVE-2018-17196 -bin.tar.gz -Aliyun -aliyun -OSS -AccessKey -aliyun-oss \ No newline at end of file +bin.tar.gz \ No newline at end of file From 757e84e75035dc6cad76ebaf32cd2b45045e0f97 Mon Sep 17 00:00:00 2001 From: frank chen Date: Tue, 26 May 2020 10:32:21 +0800 Subject: [PATCH 14/24] add package configuration to support showing extension info Signed-off-by: frank chen --- .../aliyun-oss-extensions/pom.xml | 65 ++----------------- 1 file changed, 6 insertions(+), 59 deletions(-) diff --git a/extensions-contrib/aliyun-oss-extensions/pom.xml b/extensions-contrib/aliyun-oss-extensions/pom.xml index 0a3e48ed1461..a03d7a1aae39 100644 --- a/extensions-contrib/aliyun-oss-extensions/pom.xml +++ b/extensions-contrib/aliyun-oss-extensions/pom.xml @@ -140,65 +140,7 @@ - - org.jacoco - jacoco-maven-plugin - 0.8.4 - - - Ignore firehose code - org/apache/druid/firehose/azure/**/* integration-tests - - - - BUNDLE - - - INSTRUCTION - COVEREDRATIO - 0.86 - - - LINE - COVEREDRATIO - 0.85 - - - BRANCH - COVEREDRATIO - 0.88 - - - COMPLEXITY - COVEREDRATIO - 0.80 - - - METHOD - COVEREDRATIO - 0.78 - - - CLASS - COVEREDRATIO - 0.90 - - - - - - - - report - test - - report - check - - - - - + org.apache.maven.plugins maven-assembly-plugin @@ -206,6 +148,11 @@ jar-with-dependencies + + + true + + From b2a9e25e5a4397ec0bd43a547d87ba7357af6eb1 Mon Sep 17 00:00:00 2001 From: frank chen Date: Tue, 16 Jun 2020 11:18:30 +0800 Subject: [PATCH 15/24] add IT test cases and fix bugs Signed-off-by: frank chen --- .../aliyun-oss-extensions.md | 14 +- .../druid/data/input/aliyun/OssEntity.java | 2 +- .../storage/aliyun/OssInputDataConfig.java | 5 +- .../storage/aliyun/OssStorageConfig.java | 4 +- .../storage/aliyun/OssStorageDruidModule.java | 10 +- .../OssDataSegmentPusherConfigTest.java | 2 +- .../environment-configs/override-examples/oss | 30 ++++ .../org/apache/druid/tests/TestNGGroup.java | 7 + ...stractOssInputSourceParallelIndexTest.java | 132 ++++++++++++++++++ .../indexer/ITOssToOssParallelIndexTest.java | 49 +++++++ 10 files changed, 239 insertions(+), 16 deletions(-) create mode 100644 integration-tests/docker/environment-configs/override-examples/oss create mode 100644 integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractOssInputSourceParallelIndexTest.java create mode 100644 integration-tests/src/test/java/org/apache/druid/tests/indexer/ITOssToOssParallelIndexTest.java diff --git a/docs/development/extensions-contrib/aliyun-oss-extensions.md b/docs/development/extensions-contrib/aliyun-oss-extensions.md index 4021d1339b0c..e8d071cf13f6 100644 --- a/docs/development/extensions-contrib/aliyun-oss-extensions.md +++ b/docs/development/extensions-contrib/aliyun-oss-extensions.md @@ -33,22 +33,22 @@ To use aliyun OSS, first config as below |Property|Description|Possible Values|Default| |--------|---------------|-----------|-------| -|`druid.aliyun-oss.accessKey`|the 'AccessKey ID' of your account which can be used to access the bucket||Must be set.| -|`druid.aliyun-oss.secretKey`|the 'AccessKey Secret' of your account which can be used to access the bucket| |Must be set. | -|`druid.aliyun-oss.endpoint`|the endpoint url of your OSS storage| |Must be set.| +|`druid.aliyun.oss.accessKey`|the 'AccessKey ID' of your account which can be used to access the bucket||Must be set.| +|`druid.aliyun.oss.secretKey`|the 'AccessKey Secret' of your account which can be used to access the bucket| |Must be set. | +|`druid.aliyun.oss.endpoint`|the endpoint url of your OSS storage| |Must be set.| if you want to use OSS as deep storage, use the configurations below |Property|Description|Possible Values|Default| |--------|---------------|-----------|-------| |`druid.storage.type`| | aliyun-oss|Must be set.| -|`druid.aliyun-oss.storage.bucket`||storage bucket name.|Must be set.| -|`druid.aliyun-oss.storage.prefix`|a prefix string prepended to the file names for the segments published to Aliyun OSS deep storage| druid/segments | | +|`druid.storage.aliyun.oss.bucket`||storage bucket name.|Must be set.| +|`druid.storage.aliyun.oss.prefix`|a prefix string prepended to the file names for the segments published to Aliyun OSS deep storage| druid/segments | | To save index logs to OSS, apply the configurations below: |Property|Description|Possible Values|Default| |--------|---------------|-----------|-------| |`druid.indexer.logs.type`| | aliyun-oss|Must be set.| -|`druid.indexer.logs.aliyun-oss.bucket`|the bucket used to keep logs||Must be set.| -|`druid.indexer.logs.aliyun-oss.prefix`|a prefix string prepended to the log files| || +|`druid.indexer.logs.aliyun.oss.bucket`|the bucket used to keep logs||Must be set.| +|`druid.indexer.logs.aliyun.oss.prefix`|a prefix string prepended to the log files| || diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java index aacaec3d3eb1..114ae9248e21 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java @@ -55,7 +55,7 @@ public URI getUri() protected InputStream readFrom(long offset) throws IOException { final GetObjectRequest request = new GetObjectRequest(object.getBucket(), object.getPath()); - request.setRange(offset, request.getRange()[1]); + request.setRange(offset, -1 /*from offset to end*/); try { final OSSObject ossObject = ossClient.getObject(request); diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java index e701349f7356..38d304e9ea81 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; +import javax.validation.constraints.Max; import javax.validation.constraints.Min; /** @@ -32,10 +33,12 @@ public class OssInputDataConfig /** * The maximum number of input files matching a given prefix to retrieve * from aliyun-oss at a time. + * valid range is [1,1000] */ @JsonProperty @Min(1) - private int maxListingLength = 1024; + @Max(1000) + private int maxListingLength = 1000; public void setMaxListingLength(int maxListingLength) { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java index e0f930bd958f..2cb1d56d96fb 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; +import javax.validation.constraints.Max; import javax.validation.constraints.Min; public class OssStorageConfig @@ -33,7 +34,8 @@ public class OssStorageConfig @JsonProperty @Min(1) - private int maxListingLength = 1024; + @Max(1000) + private int maxListingLength = 1000; public void setBucket(String bucket) { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java index a744688b13ed..a347333d749d 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java @@ -92,13 +92,13 @@ public void configure(Binder binder) .to(OssDataSegmentArchiver.class) .in(LazySingleton.class); Binders.dataSegmentPusherBinder(binder).addBinding(SCHEME).to(OssDataSegmentPusher.class).in(LazySingleton.class); - JsonConfigProvider.bind(binder, "druid.aliyun-oss", OssClientConfig.class); - JsonConfigProvider.bind(binder, "druid.aliyun-oss.storage", OssInputDataConfig.class); - JsonConfigProvider.bind(binder, "druid.aliyun-oss.storage", OssStorageConfig.class); - JsonConfigProvider.bind(binder, "druid.aliyun-oss.storage", OssDataSegmentArchiverConfig.class); + JsonConfigProvider.bind(binder, "druid.aliyun.oss", OssClientConfig.class); + JsonConfigProvider.bind(binder, "druid.storage.aliyun.oss", OssInputDataConfig.class); + JsonConfigProvider.bind(binder, "druid.storage.aliyun.oss", OssStorageConfig.class); + JsonConfigProvider.bind(binder, "druid.storage.aliyun.oss", OssDataSegmentArchiverConfig.class); Binders.taskLogsBinder(binder).addBinding(SCHEME).to(OssTaskLogs.class); - JsonConfigProvider.bind(binder, "druid.indexer.logs.aliyun-oss", OssTaskLogsConfig.class); + JsonConfigProvider.bind(binder, "druid.indexer.logs.aliyun.oss", OssTaskLogsConfig.class); binder.bind(OssTaskLogs.class).in(LazySingleton.class); } diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java index c2a23fd3bbc4..a42d3e0dd163 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java @@ -51,7 +51,7 @@ public void testSerializationWithDefaults() throws IOException { String jsonConfig = "{\"bucket\":\"bucket1\",\"prefix\":\"dataSource1\"}"; String expectedJsonConfig = "{\"bucket\":\"bucket1\",\"prefix\":\"dataSource1\"," - + "\"maxListingLength\":1024}"; + + "\"maxListingLength\":1000}"; OssStorageConfig config = JSON_MAPPER.readValue(jsonConfig, OssStorageConfig.class); Assert.assertEquals(expectedJsonConfig, JSON_MAPPER.writeValueAsString(config)); diff --git a/integration-tests/docker/environment-configs/override-examples/oss b/integration-tests/docker/environment-configs/override-examples/oss new file mode 100644 index 000000000000..6ec534d3578d --- /dev/null +++ b/integration-tests/docker/environment-configs/override-examples/oss @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# +# Example of override config file to provide. +# Please replace with your cloud configs/credentials +# +druid_storage_type=aliyun-oss +druid_storage_aliyun_oss_bucket= +druid_storage_aliyun_oss_prefix= +druid_aliyun_oss_accessKey= +druid_aliyun_oss_secretKey= +druid_aliyun_oss_endpoint= +druid_extensions_loadList=["druid-aliyun-oss-extensions"] \ No newline at end of file diff --git a/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java b/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java index 58a3a1b07a1d..32a65d98ddfd 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java @@ -76,6 +76,13 @@ public class TestNGGroup */ public static final String AZURE_DEEP_STORAGE = "azure-deep-storage"; + /** + * This group is not part of CI. To run this group, azure configs/credentials for your azure must be provided in a file. + * The path of the file must then be pass to mvn with -Doverride.config.path= + * See integration-tests/docker/environment-configs/override-examples/azures for env vars to provide. + */ + public static final String ALIYUN_OSS_DEEP_STORAGE = "aliyun-oss-deep-storage"; + /** * This group is not part of CI. To run this group, hadoop configs must be provided in a file. The path of the file * must then be pass to mvn with -Doverride.config.path= diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractOssInputSourceParallelIndexTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractOssInputSourceParallelIndexTest.java new file mode 100644 index 000000000000..9e11f04b6c63 --- /dev/null +++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractOssInputSourceParallelIndexTest.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.tests.indexer; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.StringUtils; +import org.testng.annotations.DataProvider; + +import java.io.Closeable; +import java.util.List; +import java.util.UUID; +import java.util.function.Function; + +public abstract class AbstractOssInputSourceParallelIndexTest extends AbstractITBatchIndexTest +{ + private static final String INDEX_TASK = "/indexer/wikipedia_cloud_index_task.json"; + private static final String INDEX_QUERIES_RESOURCE = "/indexer/wikipedia_index_queries.json"; + private static final String INPUT_SOURCE_URIS_KEY = "uris"; + private static final String INPUT_SOURCE_PREFIXES_KEY = "prefixes"; + private static final String INPUT_SOURCE_OBJECTS_KEY = "objects"; + private static final String WIKIPEDIA_DATA_1 = "wikipedia_index_data1.json"; + private static final String WIKIPEDIA_DATA_2 = "wikipedia_index_data2.json"; + private static final String WIKIPEDIA_DATA_3 = "wikipedia_index_data3.json"; + + @DataProvider + public static Object[][] resources() + { + return new Object[][]{ + {new Pair<>(INPUT_SOURCE_URIS_KEY, + ImmutableList.of( + "aliyun-oss://%%BUCKET%%/%%PATH%%" + WIKIPEDIA_DATA_1, + "aliyun-oss://%%BUCKET%%/%%PATH%%" + WIKIPEDIA_DATA_2, + "aliyun-oss://%%BUCKET%%/%%PATH%%" + WIKIPEDIA_DATA_3 + ) + )}, + {new Pair<>(INPUT_SOURCE_PREFIXES_KEY, + ImmutableList.of( + "aliyun-oss://%%BUCKET%%/%%PATH%%" + ) + )}, + {new Pair<>(INPUT_SOURCE_OBJECTS_KEY, + ImmutableList.of( + ImmutableMap.of("bucket", "%%BUCKET%%", "path", "%%PATH%%" + WIKIPEDIA_DATA_1), + ImmutableMap.of("bucket", "%%BUCKET%%", "path", "%%PATH%%" + WIKIPEDIA_DATA_2), + ImmutableMap.of("bucket", "%%BUCKET%%", "path", "%%PATH%%" + WIKIPEDIA_DATA_3) + ) + )} + }; + } + + void doTest(Pair inputSource) throws Exception + { + final String indexDatasource = "wikipedia_index_test_" + UUID.randomUUID(); + try ( + final Closeable ignored1 = unloader(indexDatasource + config.getExtraDatasourceNameSuffix()); + ) { + final Function propsTransform = spec -> { + try { + String inputSourceValue = jsonMapper.writeValueAsString(inputSource.rhs); + inputSourceValue = StringUtils.replace( + inputSourceValue, + "%%BUCKET%%", + config.getCloudBucket() + ); + inputSourceValue = StringUtils.replace( + inputSourceValue, + "%%PATH%%", + config.getCloudPath() + ); + spec = StringUtils.replace( + spec, + "%%INPUT_FORMAT_TYPE%%", + InputFormatDetails.JSON.getInputFormatType() + ); + spec = StringUtils.replace( + spec, + "%%PARTITIONS_SPEC%%", + jsonMapper.writeValueAsString(new DynamicPartitionsSpec(null, null)) + ); + spec = StringUtils.replace( + spec, + "%%INPUT_SOURCE_TYPE%%", + "aliyun-oss" + ); + spec = StringUtils.replace( + spec, + "%%INPUT_SOURCE_PROPERTY_KEY%%", + inputSource.lhs + ); + return StringUtils.replace( + spec, + "%%INPUT_SOURCE_PROPERTY_VALUE%%", + inputSourceValue + ); + } + catch (Exception e) { + throw new RuntimeException(e); + } + }; + + doIndexTest( + indexDatasource, + INDEX_TASK, + propsTransform, + INDEX_QUERIES_RESOURCE, + false, + true, + true + ); + } + } +} diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITOssToOssParallelIndexTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITOssToOssParallelIndexTest.java new file mode 100644 index 000000000000..ea989598b4bd --- /dev/null +++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITOssToOssParallelIndexTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.tests.indexer; + +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.testing.guice.DruidTestModuleFactory; +import org.apache.druid.tests.TestNGGroup; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import java.util.List; + +/** + * IMPORTANT: + * To run this test, you must: + * 1) Set the bucket and path for your data. This can be done by setting -Ddruid.test.config.cloudBucket and + * -Ddruid.test.config.cloudPath or setting "cloud_bucket" and "cloud_path" in the config file. + * 2) Copy wikipedia_index_data1.json, wikipedia_index_data2.json, and wikipedia_index_data3.json + * located in integration-tests/src/test/resources/data/batch_index/json to your Aliyun OSS at the location set in step 1. + * 3) Provide -Doverride.config.path= with Aliyun OSS credentials/configs set. See + * integration-tests/docker/environment-configs/override-examples/oss for env vars to provide. + */ +@Test(groups = TestNGGroup.ALIYUN_OSS_DEEP_STORAGE) +@Guice(moduleFactory = DruidTestModuleFactory.class) +public class ITOssToOssParallelIndexTest extends AbstractOssInputSourceParallelIndexTest +{ + @Test(dataProvider = "resources") + public void testAliyunOssIndexData(Pair ossInputSource) throws Exception + { + doTest(ossInputSource); + } +} From 5b6d72a7530c80a075acdf651fbaff4027a58d82 Mon Sep 17 00:00:00 2001 From: frank chen Date: Tue, 23 Jun 2020 16:47:49 +0800 Subject: [PATCH 16/24] 1. code review comments adopted 2. change schema from 'aliyun-oss' to 'oss' Signed-off-by: frank chen --- .../aliyun-oss-extensions.md | 22 ++--- .../data/input/aliyun/OssClientConfig.java | 2 +- .../druid/data/input/aliyun/OssEntity.java | 2 +- .../data/input/aliyun/OssInputSource.java | 2 +- .../aliyun/OssInputSourceDruidModule.java | 3 +- .../aliyun/StaticOssFirehoseFactory.java | 20 ++-- .../storage/aliyun/OssDataSegmentKiller.java | 2 +- .../storage/aliyun/OssDataSegmentMover.java | 19 ++-- .../storage/aliyun/OssDataSegmentPuller.java | 2 +- .../storage/aliyun/OssInputDataConfig.java | 8 +- ...tor.java => OssObjectSummaryIterator.java} | 26 ++---- .../storage/aliyun/OssStorageConfig.java | 18 ---- .../storage/aliyun/OssStorageDruidModule.java | 19 ++-- .../druid/storage/aliyun/OssTaskLogs.java | 8 +- .../OssTimestampVersionedDataFinder.java | 12 +-- .../apache/druid/storage/aliyun/OssUtils.java | 40 ++++---- .../aliyun/OssDataSegmentKillerTest.java | 6 +- .../aliyun/OssDataSegmentPullerTest.java | 2 +- .../OssDataSegmentPusherConfigTest.java | 33 +------ .../aliyun/OssDataSegmentPusherTest.java | 2 +- ...java => OssObjectSummaryIteratorTest.java} | 92 +++++++++---------- .../OssTimestampVersionedDataFinderTest.java | 14 +-- ...stractOssInputSourceParallelIndexTest.java | 10 +- 23 files changed, 151 insertions(+), 213 deletions(-) rename extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/{ObjectSummaryIterator.java => OssObjectSummaryIterator.java} (81%) rename extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/{ObjectSummaryIteratorTest.java => OssObjectSummaryIteratorTest.java} (67%) diff --git a/docs/development/extensions-contrib/aliyun-oss-extensions.md b/docs/development/extensions-contrib/aliyun-oss-extensions.md index e8d071cf13f6..5930bb7da4c1 100644 --- a/docs/development/extensions-contrib/aliyun-oss-extensions.md +++ b/docs/development/extensions-contrib/aliyun-oss-extensions.md @@ -27,28 +27,28 @@ To use this Apache Druid extension, make sure to [include](../../development/ext ## Deep Storage -[Aliyun](https://www.aliyun.com) is the 3rd largest cloud infrastructure provider in the world. It also provides its own storage solution, that is OSS, [Object Storage Service](https://www.aliyun.com/product/oss). +[Aliyun](https://www.aliyun.com) is the 3rd largest cloud infrastructure provider in the world. It provides its own storage solution known as OSS, [Object Storage Service](https://www.aliyun.com/product/oss). -To use aliyun OSS, first config as below +To use aliyun OSS as deep storage, first config as below |Property|Description|Possible Values|Default| |--------|---------------|-----------|-------| -|`druid.aliyun.oss.accessKey`|the 'AccessKey ID' of your account which can be used to access the bucket||Must be set.| -|`druid.aliyun.oss.secretKey`|the 'AccessKey Secret' of your account which can be used to access the bucket| |Must be set. | -|`druid.aliyun.oss.endpoint`|the endpoint url of your OSS storage| |Must be set.| +|`druid.oss.accessKey`|the 'AccessKey ID' of your account which can be used to access the bucket||Must be set.| +|`druid.oss.secretKey`|the 'AccessKey Secret' of your account which can be used to access the bucket| |Must be set. | +|`druid.oss.endpoint`|the endpoint url of your OSS storage| |Must be set.| if you want to use OSS as deep storage, use the configurations below |Property|Description|Possible Values|Default| |--------|---------------|-----------|-------| -|`druid.storage.type`| | aliyun-oss|Must be set.| -|`druid.storage.aliyun.oss.bucket`||storage bucket name.|Must be set.| -|`druid.storage.aliyun.oss.prefix`|a prefix string prepended to the file names for the segments published to Aliyun OSS deep storage| druid/segments | | +|`druid.storage.type`| | oss|Must be set.| +|`druid.storage.oss.bucket`||storage bucket name.|Must be set.| +|`druid.storage.oss.prefix`|a prefix string prepended to the file names for the segments published to Aliyun OSS deep storage| druid/segments | | To save index logs to OSS, apply the configurations below: |Property|Description|Possible Values|Default| |--------|---------------|-----------|-------| -|`druid.indexer.logs.type`| | aliyun-oss|Must be set.| -|`druid.indexer.logs.aliyun.oss.bucket`|the bucket used to keep logs||Must be set.| -|`druid.indexer.logs.aliyun.oss.prefix`|a prefix string prepended to the log files| || +|`druid.indexer.logs.type`| | oss|Must be set.| +|`druid.indexer.logs.oss.bucket`|the bucket used to keep logs||Must be set.| +|`druid.indexer.logs.oss.prefix`|a prefix string prepended to the log files| || diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java index 839ffcb94ba4..0ad4539a0e66 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java @@ -29,7 +29,7 @@ import java.util.Objects; /** - * Contains properties for aliyun-oss input source. + * Contains properties for aliyun OSS input source. * Properties can be specified by ingestionSpec which will override system default. */ public class OssClientConfig diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java index 114ae9248e21..3f501f212f7b 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssEntity.java @@ -61,7 +61,7 @@ protected InputStream readFrom(long offset) throws IOException final OSSObject ossObject = ossClient.getObject(request); if (ossObject == null) { throw new ISE( - "Failed to get an s3 object for bucket[%s], key[%s], and start[%d]", + "Failed to get an Aliyun OSS object for bucket[%s], key[%s], and start[%d]", object.getBucket(), object.getPath(), offset diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java index 8a187b808c39..e8559e5606e8 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSource.java @@ -65,7 +65,7 @@ public class OssInputSource extends CloudObjectInputSource * @param uris User provided uris to read input data * @param prefixes User provided prefixes to read input data * @param objects User provided cloud objects values to read input data - * @param inputSourceConfig User provided properties for overriding the default aliyun-oss configuration + * @param inputSourceConfig User provided properties for overriding the default aliyun OSS configuration */ @JsonCreator public OssInputSource( diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java index 024a0c532d2d..b33a62e6c350 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssInputSourceDruidModule.java @@ -30,7 +30,7 @@ import java.util.List; /** - * Druid module to wire up native batch support for aliyun-oss input + * Druid module to wire up native batch support for aliyun OSS input */ public class OssInputSourceDruidModule implements DruidModule { @@ -45,6 +45,5 @@ public List getJacksonModules() @Override public void configure(Binder binder) { - } } diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java index c7cc821ba94b..d71198826dc7 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/firehose/aliyun/StaticOssFirehoseFactory.java @@ -36,6 +36,7 @@ import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.storage.aliyun.OssStorageDruidModule; import org.apache.druid.storage.aliyun.OssUtils; import org.apache.druid.utils.CompressionUtils; @@ -51,12 +52,11 @@ import java.util.stream.Collectors; /** - * Builds firehoses that read from a predefined list of aliyun-oss objects and then dry up. + * Builds firehoses that read from a predefined list of aliyun OSS objects and then dry up. */ public class StaticOssFirehoseFactory extends PrefetchableTextFilesFirehoseFactory { private static final Logger log = new Logger(StaticOssFirehoseFactory.class); - private static final int MAX_LISTING_LENGTH = 1024; private final OSS client; private final List uris; @@ -88,11 +88,17 @@ public StaticOssFirehoseFactory( } for (final URI inputURI : this.uris) { - Preconditions.checkArgument("aliyun-oss".equals(inputURI.getScheme()), "input uri scheme == aliyun-oss (%s)", inputURI); + Preconditions.checkArgument(OssStorageDruidModule.SCHEME.equals(inputURI.getScheme()), + "input uri scheme == %s (%s)", + OssStorageDruidModule.SCHEME, + inputURI); } for (final URI inputURI : this.prefixes) { - Preconditions.checkArgument("aliyun-oss".equals(inputURI.getScheme()), "input uri scheme == aliyun-oss (%s)", inputURI); + Preconditions.checkArgument(OssStorageDruidModule.SCHEME.equals(inputURI.getScheme()), + "input uri scheme == %s (%s)", + OssStorageDruidModule.SCHEME, + inputURI); } } @@ -119,7 +125,7 @@ protected Collection initObjects() final Iterator objectSummaryIterator = OssUtils.objectSummaryIterator( client, Collections.singletonList(prefix), - MAX_LISTING_LENGTH + OssUtils.MAX_LISTING_LENGTH ); objectSummaryIterator.forEachRemaining(objects::add); @@ -138,7 +144,7 @@ protected InputStream openObjectStream(URI object) throws IOException final OSSObject ossObject = client.getObject(bucket, key); if (ossObject == null) { - throw new ISE("Failed to get an aliyun-oss object for bucket[%s] and key[%s]", bucket, key); + throw new ISE("Failed to get an Aliyun OSS object for bucket[%s] and key[%s]", bucket, key); } return ossObject.getObjectContent(); } @@ -158,7 +164,7 @@ protected InputStream openObjectStream(URI object, long start) throws IOExceptio final OSSObject ossObject = client.getObject(request); if (ossObject == null) { throw new ISE( - "Failed to get an aliyun-oss object for bucket[%s], key[%s], and start[%d]", + "Failed to get an Aliyun OSS object for bucket[%s], key[%s], and start[%d]", bucket, key, start diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java index 9959b1215c22..4149d6457c0d 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentKiller.java @@ -62,7 +62,7 @@ public void kill(DataSegment segment) throws SegmentLoadingException String path = MapUtils.getString(loadSpec, "key"); if (client.doesObjectExist(bucket, path)) { - log.info("Removing index file[aliyun-oss://%s/%s] from aliyun OSS!", bucket, path); + log.info("Removing index file[%s://%s/%s] from aliyun OSS!", OssStorageDruidModule.SCHEME, bucket, path); client.deleteObject(bucket, path); } } diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java index b0fd110bb402..a93a2762f658 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentMover.java @@ -123,9 +123,11 @@ private void safeMove( OssUtils.retry( () -> { final String copyMsg = StringUtils.format( - "[aliyun-oss://%s/%s] to [aliyun-oss://%s/%s]", + "[%s://%s/%s] to [%s://%s/%s]", + OssStorageDruidModule.SCHEME, srcBucket, srcPath, + OssStorageDruidModule.SCHEME, targetBucket, targetPath ); @@ -162,7 +164,7 @@ private void selfCheckingMove( ) throws IOException, SegmentLoadingException { if (srcBucket.equals(dstBucket) && srcPath.equals(dstPath)) { - log.info("No need to move file[aliyun-oss://%s/%s] onto itself", srcBucket, srcPath); + log.info("No need to move file[%s://%s/%s] onto itself", OssStorageDruidModule.SCHEME, srcBucket, srcPath); return; } if (client.doesObjectExist(srcBucket, srcPath)) { @@ -174,14 +176,15 @@ private void selfCheckingMove( // keyCount is still zero. if (listResult.getObjectSummaries().size() == 0) { // should never happen - throw new ISE("Unable to list object [aliyun-oss://%s/%s]", srcBucket, srcPath); + throw new ISE("Unable to list object [%s://%s/%s]", OssStorageDruidModule.SCHEME, srcBucket, srcPath); } final OSSObjectSummary objectSummary = listResult.getObjectSummaries().get(0); if (objectSummary.getStorageClass() != null && objectSummary.getStorageClass().equals(StorageClass.IA.name())) { throw new OSSException( StringUtils.format( - "Cannot move file[aliyun-oss://%s/%s] of storage class glacier, skipping.", + "Cannot move file[%s://%s/%s] of storage class glacier, skipping.", + OssStorageDruidModule.SCHEME, srcBucket, srcPath ) @@ -203,9 +206,11 @@ private void selfCheckingMove( // ensure object exists in target location if (client.doesObjectExist(dstBucket, dstPath)) { log.info( - "Not moving file [aliyun-oss://%s/%s], already present in target location [aliyun-oss://%s/%s]", + "Not moving file [%s://%s/%s], already present in target location [%s://%s/%s]", + OssStorageDruidModule.SCHEME, srcBucket, srcPath, + OssStorageDruidModule.SCHEME, dstBucket, dstPath ); @@ -224,7 +229,7 @@ private void deleteWithRetriesSilent(final String bucket, final String path) deleteWithRetries(bucket, path); } catch (Exception e) { - log.error(e, "Failed to delete file [aliyun-oss://%s/%s], giving up", bucket, path); + log.error(e, "Failed to delete file [%s://%s/%s], giving up", OssStorageDruidModule.SCHEME, bucket, path); } } @@ -237,7 +242,7 @@ private void deleteWithRetries(final String bucket, final String path) throws Ex return null; } catch (Exception e) { - log.info(e, "Error while trying to delete [aliyun-oss://%s/%s]", bucket, path); + log.info(e, "Error while trying to delete [%s://%s/%s]", OssStorageDruidModule.SCHEME, bucket, path); throw e; } }, diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java index c76fc7dbe9ff..82c0f2ed8c24 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPuller.java @@ -299,7 +299,7 @@ private boolean isObjectInBucket(final CloudObjectLocation coords) throws Segmen ); } catch (OSSException | IOException e) { - throw new SegmentLoadingException(e, "aliyun-oss fail! Key[%s]", coords); + throw new SegmentLoadingException(e, "fail! Key[%s]", coords); } catch (Exception e) { throw new RuntimeException(e); diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java index 38d304e9ea81..c2ef2dfb465b 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssInputDataConfig.java @@ -26,19 +26,19 @@ /** * Stores the configuration for options related to reading - * input data from aliyun-oss into Druid + * input data from aliyun OSS into Druid */ public class OssInputDataConfig { /** * The maximum number of input files matching a given prefix to retrieve - * from aliyun-oss at a time. + * from aliyun OSS at a time. * valid range is [1,1000] */ @JsonProperty @Min(1) - @Max(1000) - private int maxListingLength = 1000; + @Max(OssUtils.MAX_LISTING_LENGTH) + private int maxListingLength = OssUtils.MAX_LISTING_LENGTH; public void setMaxListingLength(int maxListingLength) { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssObjectSummaryIterator.java similarity index 81% rename from extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java rename to extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssObjectSummaryIterator.java index a9e6b21a92a3..fb284a5c3c75 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/ObjectSummaryIterator.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssObjectSummaryIterator.java @@ -37,7 +37,7 @@ * {@code maxListLength}. The first call is made at the same time the iterator is constructed. * */ -public class ObjectSummaryIterator implements Iterator +public class OssObjectSummaryIterator implements Iterator { private final OSS client; private final Iterator prefixesIterator; @@ -48,7 +48,7 @@ public class ObjectSummaryIterator implements Iterator private Iterator objectSummaryIterator; private OSSObjectSummary currentObjectSummary; - ObjectSummaryIterator( + OssObjectSummaryIterator( final OSS client, final Iterable prefixes, final int maxListingLength @@ -56,7 +56,7 @@ public class ObjectSummaryIterator implements Iterator { this.client = client; this.prefixesIterator = prefixes.iterator(); - this.maxListingLength = maxListingLength; + this.maxListingLength = Math.min(OssUtils.MAX_LISTING_LENGTH, maxListingLength); prepareNextRequest(); fetchNextBatch(); @@ -100,7 +100,7 @@ private void fetchNextBatch() catch (OSSException e) { throw new RE( e, - "Failed to get object summaries from S3 bucket[%s], prefix[%s]; S3 error: %s", + "Failed to get object summaries from aliyun OSS bucket[%s], prefix[%s]; error: %s", request.getBucketName(), request.getPrefix(), e.getMessage() @@ -109,7 +109,7 @@ private void fetchNextBatch() catch (Exception e) { throw new RE( e, - "Failed to get object summaries from S3 bucket[%s], prefix[%s]", + "Failed to get object summaries from aliyun OSS bucket[%s], prefix[%s]", request.getBucketName(), request.getPrefix() ); @@ -125,7 +125,7 @@ private void advanceObjectSummary() while (objectSummaryIterator.hasNext()) { currentObjectSummary = objectSummaryIterator.next(); // skips directories and empty objects - if (!isDirectoryPlaceholder(currentObjectSummary) && currentObjectSummary.getSize() > 0) { + if (currentObjectSummary.getSize() > 0 && !isDirectory(currentObjectSummary)) { return; } } @@ -146,18 +146,8 @@ private void advanceObjectSummary() /** * Checks if a given object is a directory placeholder and should be ignored. */ - private static boolean isDirectoryPlaceholder(final OSSObjectSummary objectSummary) + private static boolean isDirectory(final OSSObjectSummary objectSummary) { - // Recognize "standard" directory place-holder indications used by Amazon's AWS Console and Panic's Transmit. - if (objectSummary.getKey().endsWith("/") && objectSummary.getSize() == 0) { - return true; - } - - // Recognize place-holder objects created by the Google Storage console or S3 Organizer Firefox extension. - if (objectSummary.getKey().endsWith("_$folder$") && objectSummary.getSize() == 0) { - return true; - } - - return false; + return objectSummary.getSize() == 0 && objectSummary.getKey().endsWith("/"); } } diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java index 2cb1d56d96fb..d3edcd6105c6 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageConfig.java @@ -21,9 +21,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; -import javax.validation.constraints.Max; -import javax.validation.constraints.Min; - public class OssStorageConfig { @JsonProperty @@ -32,11 +29,6 @@ public class OssStorageConfig @JsonProperty private String prefix = ""; - @JsonProperty - @Min(1) - @Max(1000) - private int maxListingLength = 1000; - public void setBucket(String bucket) { this.bucket = bucket; @@ -46,11 +38,6 @@ public void setPrefix(String prefix) this.prefix = prefix; } - public void setMaxListingLength(int maxListingLength) - { - this.maxListingLength = maxListingLength; - } - public String getBucket() { return bucket; @@ -60,9 +47,4 @@ public String getPrefix() { return prefix; } - - public int getMaxListingLength() - { - return maxListingLength; - } } diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java index a347333d749d..9ff7da4b381d 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java @@ -37,9 +37,8 @@ public class OssStorageDruidModule implements DruidModule { - public static final String SCHEME = "aliyun-oss"; - public static final String SCHEME_S3N = "aliyun-oss_3n"; - public static final String SCHEME_ZIP = "aliyun-oss_zip"; + public static final String SCHEME = "oss"; + public static final String SCHEME_ZIP = "oss_zip"; @Override public List getJacksonModules() @@ -75,10 +74,6 @@ public void configure(Binder binder) .addBinding(SCHEME) .to(OssTimestampVersionedDataFinder.class) .in(LazySingleton.class); - MapBinder.newMapBinder(binder, String.class, SearchableVersionedDataFinder.class) - .addBinding(SCHEME_S3N) - .to(OssTimestampVersionedDataFinder.class) - .in(LazySingleton.class); Binders.dataSegmentKillerBinder(binder) .addBinding(SCHEME_ZIP) .to(OssDataSegmentKiller.class) @@ -92,13 +87,13 @@ public void configure(Binder binder) .to(OssDataSegmentArchiver.class) .in(LazySingleton.class); Binders.dataSegmentPusherBinder(binder).addBinding(SCHEME).to(OssDataSegmentPusher.class).in(LazySingleton.class); - JsonConfigProvider.bind(binder, "druid.aliyun.oss", OssClientConfig.class); - JsonConfigProvider.bind(binder, "druid.storage.aliyun.oss", OssInputDataConfig.class); - JsonConfigProvider.bind(binder, "druid.storage.aliyun.oss", OssStorageConfig.class); - JsonConfigProvider.bind(binder, "druid.storage.aliyun.oss", OssDataSegmentArchiverConfig.class); + JsonConfigProvider.bind(binder, "druid.oss", OssClientConfig.class); + JsonConfigProvider.bind(binder, "druid.storage.oss", OssInputDataConfig.class); + JsonConfigProvider.bind(binder, "druid.storage.oss", OssStorageConfig.class); + JsonConfigProvider.bind(binder, "druid.storage.oss", OssDataSegmentArchiverConfig.class); Binders.taskLogsBinder(binder).addBinding(SCHEME).to(OssTaskLogs.class); - JsonConfigProvider.bind(binder, "druid.indexer.logs.aliyun.oss", OssTaskLogsConfig.class); + JsonConfigProvider.bind(binder, "druid.indexer.logs.oss", OssTaskLogsConfig.class); binder.bind(OssTaskLogs.class).in(LazySingleton.class); } diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java index 953a533a4bc9..515d85096e03 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTaskLogs.java @@ -40,7 +40,7 @@ import java.util.Date; /** - * Provides task logs archived on aliyun-oss. + * Provides task logs archived in aliyun OSS */ public class OssTaskLogs implements TaskLogs { @@ -166,7 +166,7 @@ String getTaskLogKey(String taskid, String filename) public void killAll() throws IOException { log.info( - "Deleting all task logs from aliyun-oss location [bucket: '%s' prefix: '%s'].", + "Deleting all task logs from aliyun OSS location [bucket: '%s' prefix: '%s'].", config.getBucket(), config.getPrefix() ); @@ -179,7 +179,7 @@ public void killAll() throws IOException public void killOlderThan(long timestamp) throws IOException { log.info( - "Deleting all task logs from aliyun-oss location [bucket: '%s' prefix: '%s'] older than %s.", + "Deleting all task logs from aliyun OSS location [bucket: '%s' prefix: '%s'] older than %s.", config.getBucket(), config.getPrefix(), new Date(timestamp) @@ -194,7 +194,7 @@ public void killOlderThan(long timestamp) throws IOException ); } catch (Exception e) { - log.error("Error occurred while deleting task log files from oss. Error: %s", e.getMessage()); + log.error("Error occurred while deleting task log files from aliyun OSS. Error: %s", e.getMessage()); throw new IOException(e); } } diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java index f19d9eea340e..0b3f708477de 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinder.java @@ -34,8 +34,6 @@ public class OssTimestampVersionedDataFinder extends OssDataSegmentPuller implements SearchableVersionedDataFinder { - private static final int MAX_LISTING_KEYS = 1000; - @Inject public OssTimestampVersionedDataFinder(OSS client) { @@ -46,11 +44,11 @@ public OssTimestampVersionedDataFinder(OSS client) * Gets the key with the most recently modified timestamp. * `pattern` is evaluated against the entire key AFTER the path given in `uri`. * The substring `pattern` is matched against will have a leading `/` removed. - * For example `aliyun-oss://some_bucket/some_prefix/some_key` with a URI of `aliyun-oss://some_bucket/some_prefix` will match against `some_key`. - * `aliyun-oss://some_bucket/some_prefixsome_key` with a URI of `aliyun-oss://some_bucket/some_prefix` will match against `some_key` - * `aliyun-oss://some_bucket/some_prefix//some_key` with a URI of `aliyun-oss://some_bucket/some_prefix` will match against `/some_key` + * For example `oss://some_bucket/some_prefix/some_key` with a URI of `oss://some_bucket/some_prefix` will match against `some_key`. + * `oss://some_bucket/some_prefixsome_key` with a URI of `oss://some_bucket/some_prefix` will match against `some_key` + * `oss://some_bucket/some_prefix//some_key` with a URI of `oss://some_bucket/some_prefix` will match against `/some_key` * - * @param uri The URI of in the form of `aliyun-oss://some_bucket/some_key` + * @param uri The URI of in the form of `oss://some_bucket/some_key` * @param pattern The pattern matcher to determine if a *key* is of interest, or `null` to match everything. * @return A URI to the most recently modified object which matched the pattern. */ @@ -64,7 +62,7 @@ public URI getLatestVersion(final URI uri, final @Nullable Pattern pattern) final Iterator objectSummaryIterator = OssUtils.objectSummaryIterator( client, Collections.singletonList(uri), - MAX_LISTING_KEYS + OssUtils.MAX_LISTING_LENGTH ); while (objectSummaryIterator.hasNext()) { final OSSObjectSummary objectSummary = objectSummaryIterator.next(); diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java index f665c66857cf..1a707c785570 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssUtils.java @@ -21,8 +21,6 @@ import com.aliyun.oss.OSS; import com.aliyun.oss.OSSException; -import com.aliyun.oss.model.AccessControlList; -import com.aliyun.oss.model.CannedAccessControlList; import com.aliyun.oss.model.DeleteObjectsRequest; import com.aliyun.oss.model.ListObjectsRequest; import com.aliyun.oss.model.OSSObjectSummary; @@ -50,6 +48,7 @@ public class OssUtils private static final String SCHEME = OssStorageDruidModule.SCHEME; private static final Joiner JOINER = Joiner.on("/").skipNulls(); private static final Logger log = new Logger(OssUtils.class); + public static final int MAX_LISTING_LENGTH = 1000; //limited by Aliyun OSS SDK static boolean isServiceExceptionRecoverable(OSSException ex) @@ -78,7 +77,7 @@ public boolean apply(Throwable e) }; /** - * Retries aliyun-oss operations that fail due to io-related exceptions. Service-level exceptions (access denied, file not + * Retries aliyun OSS operations that fail due to io-related exceptions. Service-level exceptions (access denied, file not * found, etc) are not retried. */ static T retry(Task f) throws Exception @@ -106,12 +105,12 @@ static boolean isObjectInBucketIgnoringPermission( } /** - * Create an iterator over a set of aliyun-oss objects specified by a set of prefixes. + * Create an iterator over a set of aliyun OSS objects specified by a set of prefixes. *

* For each provided prefix URI, the iterator will walk through all objects that are in the same bucket as the * provided URI and whose keys start with that URI's path, except for directory placeholders (which will be * ignored). The iterator is computed incrementally by calling {@link OSS#listObjects} for - * each prefix in batches of {@param maxListLength}. The first call is made at the same time the iterator is + * each prefix in batches of {@param maxListingLength}. The first call is made at the same time the iterator is * constructed. */ public static Iterator objectSummaryIterator( @@ -120,14 +119,14 @@ public static Iterator objectSummaryIterator( final int maxListingLength ) { - return new ObjectSummaryIterator(client, prefixes, maxListingLength); + return new OssObjectSummaryIterator(client, prefixes, maxListingLength); } /** * Create an {@link URI} from the given {@link OSSObjectSummary}. The result URI is composed as below. * * 

-   * {@code aliyun-oss://{BUCKET_NAME}/{OBJECT_KEY}}
+   * {@code oss://{BUCKET_NAME}/{OBJECT_KEY}}
    *
*/ public static URI summaryToUri(OSSObjectSummary object) @@ -148,13 +147,6 @@ static String constructSegmentPath(String baseKey, String storageDir) ) + "/index.zip"; } - static CannedAccessControlList grantFullControlToBucketOwner(OSS client, String bucket) - { - final AccessControlList acl = client.getBucketAcl(bucket); - return acl.getCannedACL(); - //acl.grantAllPermissions(new Grant(new Grantee(acl.getOwner().getId()), Permission.FullControl)); - } - public static String extractKey(URI uri) { return StringUtils.maybeRemoveLeadingSlash(uri.getPath()); @@ -169,12 +161,12 @@ public static URI checkURI(URI uri) } /** - * Gets a single {@link OSSObjectSummary} from aliyun-oss. Since this method might return a wrong object if there are multiple + * Gets a single {@link OSSObjectSummary} from aliyun OSS. Since this method might return a wrong object if there are multiple * objects that match the given key, this method should be used only when it's guaranteed that the given key is unique * in the given bucket. * - * @param client aliyun-oss client - * @param bucket aliyun-oss bucket + * @param client aliyun OSS client + * @param bucket aliyun OSS bucket * @param key unique key for the object to be retrieved */ public static OSSObjectSummary getSingleObjectSummary(OSS client, String bucket, String key) @@ -200,11 +192,11 @@ public static OSSObjectSummary getSingleObjectSummary(OSS client, String bucket, } /** - * Delete the files from aliyun-oss in a specified bucket, matching a specified prefix and filter + * Delete the files from aliyun OSS in a specified bucket, matching a specified prefix and filter * - * @param client aliyun-oss client - * @param config specifies the configuration to use when finding matching files in aliyun-oss to delete - * @param bucket aliyun-oss bucket + * @param client aliyun OSS client + * @param config specifies the configuration to use when finding matching files in aliyun OSS to delete + * @param bucket aliyun OSS bucket * @param prefix the file prefix * @param filter function which returns true if the prefix file found should be deleted and false otherwise. * @throws Exception @@ -219,7 +211,7 @@ public static void deleteObjectsInPath( throws Exception { final List keysToDelete = new ArrayList<>(config.getMaxListingLength()); - final ObjectSummaryIterator iterator = new ObjectSummaryIterator( + final OssObjectSummaryIterator iterator = new OssObjectSummaryIterator( client, ImmutableList.of(new CloudObjectLocation(bucket, prefix).toUri("http")), config.getMaxListingLength() @@ -258,11 +250,11 @@ private static void deleteBucketKeys( } /** - * Uploads a file to aliyun-oss if possible. First trying to set ACL to give the bucket owner full control of the file before uploading. + * Uploads a file to aliyun OSS if possible. First trying to set ACL to give the bucket owner full control of the file before uploading. * * @param client aliyun OSS client * @param key The key under which to store the new object. - * @param file The path of the file to upload to aliyun-oss. + * @param file The path of the file to upload to aliyun OSS. */ static void uploadFileIfPossible( OSS client, diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java index deff2a45b2b2..638348379a27 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentKillerTest.java @@ -46,12 +46,12 @@ public class OssDataSegmentKillerTest extends EasyMockSupport private static final String KEY_2 = "key2"; private static final String TEST_BUCKET = "test_bucket"; private static final String TEST_PREFIX = "test_prefix"; - private static final URI PREFIX_URI = URI.create(StringUtils.format("aliyun-oss://%s/%s", TEST_BUCKET, TEST_PREFIX)); + private static final URI PREFIX_URI = URI.create(StringUtils.format(OssStorageDruidModule.SCHEME + "://%s/%s", TEST_BUCKET, TEST_PREFIX)); private static final long TIME_0 = 0L; private static final long TIME_1 = 1L; private static final int MAX_KEYS = 1; - private static final Exception RECOVERABLE_EXCEPTION = new ClientException(new IOException()); - private static final Exception NON_RECOVERABLE_EXCEPTION = new ClientException(new NullPointerException()); + private static final Exception RECOVERABLE_EXCEPTION = new ClientException(new IOException("mocked by test case")); + private static final Exception NON_RECOVERABLE_EXCEPTION = new ClientException(new NullPointerException("mocked by test case")); @Mock private OSS client; diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java index 726d551b970d..a90a7504b914 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java @@ -75,7 +75,7 @@ public void testSimpleGetVersion() throws IOException EasyMock.replay(s3Client); - String version = puller.getVersion(URI.create(StringUtils.format("aliyun-oss://%s/%s", bucket, objectSummary.getKey()))); + String version = puller.getVersion(URI.create(StringUtils.format(OssStorageDruidModule.SCHEME + "://%s/%s", bucket, objectSummary.getKey()))); EasyMock.verify(s3Client); diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java index a42d3e0dd163..d558a08068cd 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherConfigTest.java @@ -20,17 +20,11 @@ package org.apache.druid.storage.aliyun; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Iterators; import org.apache.druid.jackson.DefaultObjectMapper; import org.junit.Assert; import org.junit.Test; -import javax.validation.ConstraintViolation; -import javax.validation.Validation; -import javax.validation.Validator; import java.io.IOException; -import java.util.Locale; -import java.util.Set; public class OssDataSegmentPusherConfigTest { @@ -39,8 +33,7 @@ public class OssDataSegmentPusherConfigTest @Test public void testSerialization() throws IOException { - String jsonConfig = "{\"bucket\":\"bucket1\",\"prefix\":\"dataSource1\"," - + "\"maxListingLength\":2000}"; + String jsonConfig = "{\"bucket\":\"bucket1\",\"prefix\":\"dataSource1\"}"; OssStorageConfig config = JSON_MAPPER.readValue(jsonConfig, OssStorageConfig.class); Assert.assertEquals(jsonConfig, JSON_MAPPER.writeValueAsString(config)); @@ -50,31 +43,9 @@ public void testSerialization() throws IOException public void testSerializationWithDefaults() throws IOException { String jsonConfig = "{\"bucket\":\"bucket1\",\"prefix\":\"dataSource1\"}"; - String expectedJsonConfig = "{\"bucket\":\"bucket1\",\"prefix\":\"dataSource1\"," - + "\"maxListingLength\":1000}"; + String expectedJsonConfig = "{\"bucket\":\"bucket1\",\"prefix\":\"dataSource1\"}"; OssStorageConfig config = JSON_MAPPER.readValue(jsonConfig, OssStorageConfig.class); Assert.assertEquals(expectedJsonConfig, JSON_MAPPER.writeValueAsString(config)); } - - @Test - public void testSerializationValidatingMaxListingLength() throws IOException - { - Locale old = Locale.getDefault(); - Locale.setDefault(Locale.ENGLISH); - try { - String jsonConfig = "{\"bucket\":\"bucket1\",\"prefix\":\"dataSource1\"," - + "\"maxListingLength\":-1}"; - Validator validator = Validation.buildDefaultValidatorFactory().getValidator(); - - OssStorageConfig config = JSON_MAPPER.readValue(jsonConfig, OssStorageConfig.class); - Set> violations = validator.validate(config); - Assert.assertEquals(1, violations.size()); - ConstraintViolation violation = Iterators.getOnlyElement(violations.iterator()); - Assert.assertEquals("must be greater than or equal to 1", violation.getMessage()); - } - finally { - Locale.setDefault(old); - } - } } diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java index 1c8c1159a0ff..b3d91c7af548 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPusherTest.java @@ -118,7 +118,7 @@ private void testPushInternal(boolean useUniquePath, String matcher) throws Exce segment.getLoadSpec().get("key").toString(), Pattern.compile(matcher).matcher(segment.getLoadSpec().get("key").toString()).matches() ); - Assert.assertEquals("aliyun-oss_zip", segment.getLoadSpec().get("type")); + Assert.assertEquals("oss_zip", segment.getLoadSpec().get("type")); EasyMock.verify(client); } diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssObjectSummaryIteratorTest.java similarity index 67% rename from extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java rename to extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssObjectSummaryIteratorTest.java index fba1d494ba5c..d124b6bf6f9c 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/ObjectSummaryIteratorTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssObjectSummaryIteratorTest.java @@ -34,7 +34,7 @@ import java.util.List; import java.util.stream.Collectors; -public class ObjectSummaryIteratorTest +public class OssObjectSummaryIteratorTest { private static final ImmutableList TEST_OBJECTS = ImmutableList.of( @@ -54,8 +54,8 @@ public class ObjectSummaryIteratorTest public void testSingleObject() { test( - ImmutableList.of("aliyun-oss://b/foo/baz"), - ImmutableList.of("aliyun-oss://b/foo/baz"), + ImmutableList.of(OssStorageDruidModule.SCHEME + "://b/foo/baz"), + ImmutableList.of(OssStorageDruidModule.SCHEME + "://b/foo/baz"), 5 ); } @@ -65,13 +65,13 @@ public void testMultiObjectOneKeyAtATime() { test( ImmutableList.of( - "aliyun-oss://b/foo/bar1", - "aliyun-oss://b/foo/bar2", - "aliyun-oss://b/foo/bar3", - "aliyun-oss://b/foo/bar4", - "aliyun-oss://b/foo/baz" + OssStorageDruidModule.SCHEME + "://b/foo/bar1", + OssStorageDruidModule.SCHEME + "://b/foo/bar2", + OssStorageDruidModule.SCHEME + "://b/foo/bar3", + OssStorageDruidModule.SCHEME + "://b/foo/bar4", + OssStorageDruidModule.SCHEME + "://b/foo/baz" ), - ImmutableList.of("aliyun-oss://b/foo/"), + ImmutableList.of(OssStorageDruidModule.SCHEME + "://b/foo/"), 1 ); } @@ -81,13 +81,13 @@ public void testMultiObjectTwoKeysAtATime() { test( ImmutableList.of( - "aliyun-oss://b/foo/bar1", - "aliyun-oss://b/foo/bar2", - "aliyun-oss://b/foo/bar3", - "aliyun-oss://b/foo/bar4", - "aliyun-oss://b/foo/baz" + OssStorageDruidModule.SCHEME + "://b/foo/bar1", + OssStorageDruidModule.SCHEME + "://b/foo/bar2", + OssStorageDruidModule.SCHEME + "://b/foo/bar3", + OssStorageDruidModule.SCHEME + "://b/foo/bar4", + OssStorageDruidModule.SCHEME + "://b/foo/baz" ), - ImmutableList.of("aliyun-oss://b/foo/"), + ImmutableList.of(OssStorageDruidModule.SCHEME + "://b/foo/"), 2 ); } @@ -97,13 +97,13 @@ public void testMultiObjectTenKeysAtATime() { test( ImmutableList.of( - "aliyun-oss://b/foo/bar1", - "aliyun-oss://b/foo/bar2", - "aliyun-oss://b/foo/bar3", - "aliyun-oss://b/foo/bar4", - "aliyun-oss://b/foo/baz" + OssStorageDruidModule.SCHEME + "://b/foo/bar1", + OssStorageDruidModule.SCHEME + "://b/foo/bar2", + OssStorageDruidModule.SCHEME + "://b/foo/bar3", + OssStorageDruidModule.SCHEME + "://b/foo/bar4", + OssStorageDruidModule.SCHEME + "://b/foo/baz" ), - ImmutableList.of("aliyun-oss://b/foo/"), + ImmutableList.of(OssStorageDruidModule.SCHEME + "://b/foo/"), 10 ); } @@ -112,8 +112,8 @@ public void testMultiObjectTenKeysAtATime() public void testPrefixInMiddleOfKey() { test( - ImmutableList.of("aliyun-oss://b/foo/bar1", "aliyun-oss://b/foo/bar2", "aliyun-oss://b/foo/bar3", "aliyun-oss://b/foo/bar4"), - ImmutableList.of("aliyun-oss://b/foo/bar"), + ImmutableList.of(OssStorageDruidModule.SCHEME + "://b/foo/bar1", OssStorageDruidModule.SCHEME + "://b/foo/bar2", OssStorageDruidModule.SCHEME + "://b/foo/bar3", OssStorageDruidModule.SCHEME + "://b/foo/bar4"), + ImmutableList.of(OssStorageDruidModule.SCHEME + "://b/foo/bar"), 10 ); } @@ -123,14 +123,14 @@ public void testNoPath() { test( ImmutableList.of( - "aliyun-oss://b/foo", - "aliyun-oss://b/foo/bar1", - "aliyun-oss://b/foo/bar2", - "aliyun-oss://b/foo/bar3", - "aliyun-oss://b/foo/bar4", - "aliyun-oss://b/foo/baz" + OssStorageDruidModule.SCHEME + "://b/foo", + OssStorageDruidModule.SCHEME + "://b/foo/bar1", + OssStorageDruidModule.SCHEME + "://b/foo/bar2", + OssStorageDruidModule.SCHEME + "://b/foo/bar3", + OssStorageDruidModule.SCHEME + "://b/foo/bar4", + OssStorageDruidModule.SCHEME + "://b/foo/baz" ), - ImmutableList.of("aliyun-oss://b"), + ImmutableList.of(OssStorageDruidModule.SCHEME + "://b"), 10 ); } @@ -140,14 +140,14 @@ public void testSlashPath() { test( ImmutableList.of( - "aliyun-oss://b/foo", - "aliyun-oss://b/foo/bar1", - "aliyun-oss://b/foo/bar2", - "aliyun-oss://b/foo/bar3", - "aliyun-oss://b/foo/bar4", - "aliyun-oss://b/foo/baz" + OssStorageDruidModule.SCHEME + "://b/foo", + OssStorageDruidModule.SCHEME + "://b/foo/bar1", + OssStorageDruidModule.SCHEME + "://b/foo/bar2", + OssStorageDruidModule.SCHEME + "://b/foo/bar3", + OssStorageDruidModule.SCHEME + "://b/foo/bar4", + OssStorageDruidModule.SCHEME + "://b/foo/baz" ), - ImmutableList.of("aliyun-oss://b/"), + ImmutableList.of(OssStorageDruidModule.SCHEME + "://b/"), 10 ); } @@ -157,7 +157,7 @@ public void testDifferentBucket() { test( ImmutableList.of(), - ImmutableList.of("aliyun-oss://bx/foo/"), + ImmutableList.of(OssStorageDruidModule.SCHEME + "://bx/foo/"), 10 ); } @@ -167,13 +167,13 @@ public void testWithMultiplePrefixesReturningAllNonEmptyObjectsStartingWithOneOf { test( ImmutableList.of( - "aliyun-oss://b/foo/bar1", - "aliyun-oss://b/foo/bar2", - "aliyun-oss://b/foo/bar3", - "aliyun-oss://b/foo/bar4", - "aliyun-oss://b/foo/baz" + OssStorageDruidModule.SCHEME + "://b/foo/bar1", + OssStorageDruidModule.SCHEME + "://b/foo/bar2", + OssStorageDruidModule.SCHEME + "://b/foo/bar3", + OssStorageDruidModule.SCHEME + "://b/foo/bar4", + OssStorageDruidModule.SCHEME + "://b/foo/baz" ), - ImmutableList.of("aliyun-oss://b/foo/bar", "aliyun-oss://b/foo/baz"), + ImmutableList.of(OssStorageDruidModule.SCHEME + "://b/foo/bar", OssStorageDruidModule.SCHEME + "://b/foo/baz"), 10 ); } @@ -214,8 +214,8 @@ private static void test( } /** - * Makes a mock S3 client that handles enough of "listObjectsV2" to test the functionality of the - * {@link ObjectSummaryIterator} class. + * Makes a mock OSS client that handles enough of "listObjects" to test the functionality of the + * {@link OssObjectSummaryIterator} class. */ private static OSS makeMockClient( final List objects diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java index 3cebb8fcb394..cde32ed683d9 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java @@ -69,11 +69,11 @@ public void testSimpleLatestVersion() EasyMock.replay(client); - URI latest = finder.getLatestVersion(URI.create(StringUtils.format("aliyun-oss://%s/%s", bucket, keyPrefix)), pattern); + URI latest = finder.getLatestVersion(URI.create(StringUtils.format("%s://%s/%s", OssStorageDruidModule.SCHEME, bucket, keyPrefix)), pattern); EasyMock.verify(client); - URI expected = URI.create(StringUtils.format("aliyun-oss://%s/%s", bucket, object1.getKey())); + URI expected = URI.create(StringUtils.format("%s://%s/%s", OssStorageDruidModule.SCHEME, bucket, object1.getKey())); Assert.assertEquals(expected, latest); } @@ -98,7 +98,7 @@ public void testMissing() EasyMock.replay(oss); - URI latest = finder.getLatestVersion(URI.create(StringUtils.format("aliyun-oss://%s/%s", bucket, keyPrefix)), pattern); + URI latest = finder.getLatestVersion(URI.create(StringUtils.format("%s://%s/%s", OssStorageDruidModule.SCHEME, bucket, keyPrefix)), pattern); EasyMock.verify(oss); @@ -133,11 +133,11 @@ public void testFindSelf() EasyMock.replay(s3Client); - URI latest = finder.getLatestVersion(URI.create(StringUtils.format("aliyun-oss://%s/%s", bucket, keyPrefix)), pattern); + URI latest = finder.getLatestVersion(URI.create(StringUtils.format("%s://%s/%s", OssStorageDruidModule.SCHEME, bucket, keyPrefix)), pattern); EasyMock.verify(s3Client); - URI expected = URI.create(StringUtils.format("aliyun-oss://%s/%s", bucket, object0.getKey())); + URI expected = URI.create(StringUtils.format("%s://%s/%s", OssStorageDruidModule.SCHEME, bucket, object0.getKey())); Assert.assertEquals(expected, latest); } @@ -167,11 +167,11 @@ public void testFindExact() EasyMock.replay(s3Client); - URI latest = finder.getLatestVersion(URI.create(StringUtils.format("aliyun-oss://%s/%s", bucket, object0.getKey())), null); + URI latest = finder.getLatestVersion(URI.create(StringUtils.format("%s://%s/%s", OssStorageDruidModule.SCHEME, bucket, object0.getKey())), null); EasyMock.verify(s3Client); - URI expected = URI.create(StringUtils.format("aliyun-oss://%s/%s", bucket, object0.getKey())); + URI expected = URI.create(StringUtils.format("%s://%s/%s", OssStorageDruidModule.SCHEME, bucket, object0.getKey())); Assert.assertEquals(expected, latest); } diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractOssInputSourceParallelIndexTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractOssInputSourceParallelIndexTest.java index 9e11f04b6c63..72b0d355b2aa 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractOssInputSourceParallelIndexTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractOssInputSourceParallelIndexTest.java @@ -48,14 +48,14 @@ public static Object[][] resources() return new Object[][]{ {new Pair<>(INPUT_SOURCE_URIS_KEY, ImmutableList.of( - "aliyun-oss://%%BUCKET%%/%%PATH%%" + WIKIPEDIA_DATA_1, - "aliyun-oss://%%BUCKET%%/%%PATH%%" + WIKIPEDIA_DATA_2, - "aliyun-oss://%%BUCKET%%/%%PATH%%" + WIKIPEDIA_DATA_3 + "oss://%%BUCKET%%/%%PATH%%" + WIKIPEDIA_DATA_1, + "oss://%%BUCKET%%/%%PATH%%" + WIKIPEDIA_DATA_2, + "oss://%%BUCKET%%/%%PATH%%" + WIKIPEDIA_DATA_3 ) )}, {new Pair<>(INPUT_SOURCE_PREFIXES_KEY, ImmutableList.of( - "aliyun-oss://%%BUCKET%%/%%PATH%%" + "oss://%%BUCKET%%/%%PATH%%" ) )}, {new Pair<>(INPUT_SOURCE_OBJECTS_KEY, @@ -100,7 +100,7 @@ void doTest(Pair inputSource) throws Exception spec = StringUtils.replace( spec, "%%INPUT_SOURCE_TYPE%%", - "aliyun-oss" + "oss" ); spec = StringUtils.replace( spec, From 5ee9ddcf11fec2bc387b21334481409335b6da07 Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 26 Jun 2020 19:00:02 +0800 Subject: [PATCH 17/24] add license info Signed-off-by: frank chen --- LICENSE | 1 + .../apache/druid/storage/aliyun/OssObjectSummaryIterator.java | 3 +++ 2 files changed, 4 insertions(+) diff --git a/LICENSE b/LICENSE index 6171bc011057..aa65ab9e0a70 100644 --- a/LICENSE +++ b/LICENSE @@ -274,6 +274,7 @@ SOURCE/JAVA-CORE This product contains s3 directory place holder check code adapted from JetS3t (https://bitbucket.org/jmurty/jets3t/wiki/Home). * extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java + * extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssObjectSummaryIterator.java This product contains lpad and rpad methods adapted from Apache Flink. * core/src/main/java/org/apache/druid/java/util/common/StringUtils.java diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssObjectSummaryIterator.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssObjectSummaryIterator.java index fb284a5c3c75..8bba8961eeeb 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssObjectSummaryIterator.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssObjectSummaryIterator.java @@ -145,6 +145,9 @@ private void advanceObjectSummary() /** * Checks if a given object is a directory placeholder and should be ignored. + * + * Based on {@link org.apache.druid.storage.s3.ObjectSummaryIterator} which is adapted from org.jets3t.service.model.StorageObject.isDirectoryPlaceholder(). + * */ private static boolean isDirectory(final OSSObjectSummary objectSummary) { From 7f255d09e78b707e4650307c757c1dfc294ba421 Mon Sep 17 00:00:00 2001 From: frank chen Date: Sun, 28 Jun 2020 11:11:10 +0800 Subject: [PATCH 18/24] fix doc Signed-off-by: frank chen --- .../extensions-contrib/aliyun-oss-extensions.md | 14 +++++++------- website/.spelling | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/development/extensions-contrib/aliyun-oss-extensions.md b/docs/development/extensions-contrib/aliyun-oss-extensions.md index 5930bb7da4c1..36b7cd647b84 100644 --- a/docs/development/extensions-contrib/aliyun-oss-extensions.md +++ b/docs/development/extensions-contrib/aliyun-oss-extensions.md @@ -33,7 +33,7 @@ To use aliyun OSS as deep storage, first config as below |Property|Description|Possible Values|Default| |--------|---------------|-----------|-------| -|`druid.oss.accessKey`|the 'AccessKey ID' of your account which can be used to access the bucket||Must be set.| +|`druid.oss.accessKey`|the 'AccessKey ID' of your account which can be used to access the bucket| |Must be set.| |`druid.oss.secretKey`|the 'AccessKey Secret' of your account which can be used to access the bucket| |Must be set. | |`druid.oss.endpoint`|the endpoint url of your OSS storage| |Must be set.| @@ -41,14 +41,14 @@ if you want to use OSS as deep storage, use the configurations below |Property|Description|Possible Values|Default| |--------|---------------|-----------|-------| -|`druid.storage.type`| | oss|Must be set.| -|`druid.storage.oss.bucket`||storage bucket name.|Must be set.| -|`druid.storage.oss.prefix`|a prefix string prepended to the file names for the segments published to Aliyun OSS deep storage| druid/segments | | +|`druid.storage.type`| Global deep storage provider. Must be set to `oss` to make use of this extension. | oss |Must be set.| +|`druid.storage.oss.bucket`|storage bucket name.| | Must be set.| +|`druid.storage.oss.prefix`|a prefix string prepended to the file names for the segments published to aliyun OSS deep storage| druid/segments | | To save index logs to OSS, apply the configurations below: |Property|Description|Possible Values|Default| |--------|---------------|-----------|-------| -|`druid.indexer.logs.type`| | oss|Must be set.| -|`druid.indexer.logs.oss.bucket`|the bucket used to keep logs||Must be set.| -|`druid.indexer.logs.oss.prefix`|a prefix string prepended to the log files| || +|`druid.indexer.logs.type`| Global deep storage provider. Must be set to `oss` to make use of this extension. | oss |Must be set.| +|`druid.indexer.logs.oss.bucket`|the bucket used to keep logs| |Must be set.| +|`druid.indexer.logs.oss.prefix`|a prefix string prepended to the log files.| | | diff --git a/website/.spelling b/website/.spelling index 56f34a31afc3..31e596ad6267 100644 --- a/website/.spelling +++ b/website/.spelling @@ -573,6 +573,7 @@ aliyun OSS AccessKey aliyun-oss +oss url - ../docs/development/extensions-core/approximate-histograms.md approxHistogram From b09263fbfaebe441b40c7287d097436d7b14fa57 Mon Sep 17 00:00:00 2001 From: frank chen Date: Sun, 28 Jun 2020 15:41:45 +0800 Subject: [PATCH 19/24] exclude execution of IT testcases of OSS extension from CI Signed-off-by: frank chen --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6a508919c720..723d5b972ed7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -373,7 +373,7 @@ jobs: name: "(Compile=openjdk8, Run=openjdk8) other integration test" jdk: openjdk8 services: *integration_test_services - env: TESTNG_GROUPS='-DexcludedGroups=batch-index,perfect-rollup-parallel-batch-index,kafka-index,query,realtime-index,security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage' JVM_RUNTIME='-Djvm.runtime=8' + env: TESTNG_GROUPS='-DexcludedGroups=batch-index,perfect-rollup-parallel-batch-index,kafka-index,query,realtime-index,security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage' JVM_RUNTIME='-Djvm.runtime=8' script: *run_integration_test after_failure: *integration_test_diags # END - Integration tests for Compile with Java 8 and Run with Java 8 @@ -407,7 +407,7 @@ jobs: - <<: *integration_tests name: "(Compile=openjdk8, Run=openjdk11) other integration test" jdk: openjdk8 - env: TESTNG_GROUPS='-DexcludedGroups=batch-index,perfect-rollup-parallel-batch-index,kafka-index,query,realtime-index,security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage' JVM_RUNTIME='-Djvm.runtime=11' + env: TESTNG_GROUPS='-DexcludedGroups=batch-index,perfect-rollup-parallel-batch-index,kafka-index,query,realtime-index,security,s3-deep-storage,gcs-deep-storage,azure-deep-storage,hdfs-deep-storage,s3-ingestion,kinesis-index,kinesis-data-format,kafka-transactional-index,kafka-index-slow,kafka-transactional-index-slow,kafka-data-format,hadoop-s3-to-s3-deep-storage,hadoop-s3-to-hdfs-deep-storage,hadoop-azure-to-azure-deep-storage,hadoop-azure-to-hdfs-deep-storage,hadoop-gcs-to-gcs-deep-storage,hadoop-gcs-to-hdfs-deep-storage,aliyun-oss-deep-storage' JVM_RUNTIME='-Djvm.runtime=11' # END - Integration tests for Compile with Java 8 and Run with Java 11 - name: "security vulnerabilities" From b158c0dae5df10fce32bc0a9b7a3b38b1a77c4a4 Mon Sep 17 00:00:00 2001 From: frank chen Date: Wed, 1 Jul 2020 14:30:46 +0800 Subject: [PATCH 20/24] put the extensions under contrib group and add to distribution --- distribution/pom.xml | 2 ++ .../extensions-contrib/aliyun-oss-extensions.md | 6 +++--- docs/development/extensions.md | 1 + extensions-contrib/aliyun-oss-extensions/pom.xml | 16 +++++++++++----- .../environment-configs/override-examples/oss | 14 +++++++------- 5 files changed, 24 insertions(+), 15 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index 2b5c3d5d760b..98eec645f76f 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -425,6 +425,8 @@ org.apache.druid.extensions.contrib:druid-tdigestsketch -c org.apache.druid.extensions.contrib:gce-extensions + -c + org.apache.druid.extensions.contrib:aliyun-oss-extensions diff --git a/docs/development/extensions-contrib/aliyun-oss-extensions.md b/docs/development/extensions-contrib/aliyun-oss-extensions.md index 36b7cd647b84..64fcd87039d1 100644 --- a/docs/development/extensions-contrib/aliyun-oss-extensions.md +++ b/docs/development/extensions-contrib/aliyun-oss-extensions.md @@ -23,7 +23,7 @@ title: "Aliyun OSS" --> -To use this Apache Druid extension, make sure to [include](../../development/extensions.md#loading-extensions) `druid-aliyun-oss-extensions` extension. +To use this Apache Druid extension, make sure to [include](../../development/extensions.md#loading-extensions) `aliyun-oss-extensions` extension. ## Deep Storage @@ -33,8 +33,8 @@ To use aliyun OSS as deep storage, first config as below |Property|Description|Possible Values|Default| |--------|---------------|-----------|-------| -|`druid.oss.accessKey`|the 'AccessKey ID' of your account which can be used to access the bucket| |Must be set.| -|`druid.oss.secretKey`|the 'AccessKey Secret' of your account which can be used to access the bucket| |Must be set. | +|`druid.oss.accessKey`|the `AccessKey ID` of your account which can be used to access the bucket| |Must be set.| +|`druid.oss.secretKey`|the `AccessKey Secret` of your account which can be used to access the bucket| |Must be set. | |`druid.oss.endpoint`|the endpoint url of your OSS storage| |Must be set.| if you want to use OSS as deep storage, use the configurations below diff --git a/docs/development/extensions.md b/docs/development/extensions.md index 7eac4ed85f49..1bc474f33210 100644 --- a/docs/development/extensions.md +++ b/docs/development/extensions.md @@ -73,6 +73,7 @@ All of these community extensions can be downloaded using [pull-deps](../operati |Name|Description|Docs| |----|-----------|----| +|aliyun-oss-extensions|Aliyun OSS deep storage |[link](../development/extensions-contrib/aliyun-oss-extensions.md)| |ambari-metrics-emitter|Ambari Metrics Emitter |[link](../development/extensions-contrib/ambari-metrics-emitter.md)| |druid-cassandra-storage|Apache Cassandra deep storage.|[link](../development/extensions-contrib/cassandra.md)| |druid-cloudfiles-extensions|Rackspace Cloudfiles deep storage and firehose.|[link](../development/extensions-contrib/cloudfiles.md)| diff --git a/extensions-contrib/aliyun-oss-extensions/pom.xml b/extensions-contrib/aliyun-oss-extensions/pom.xml index a03d7a1aae39..27122822d523 100644 --- a/extensions-contrib/aliyun-oss-extensions/pom.xml +++ b/extensions-contrib/aliyun-oss-extensions/pom.xml @@ -21,11 +21,10 @@ 4.0.0 - - org.apache.druid.extensions - druid-aliyun-oss-extensions - druid-aliyun-oss-extensions - druid-aliyun-oss-extensions + org.apache.druid.extensions.contrib + aliyun-oss-extensions + aliyun-oss-extensions + aliyun-oss-extensions org.apache.druid @@ -110,6 +109,13 @@ + + org.apache.druid + druid-core + ${project.parent.version} + test-jar + test + junit junit diff --git a/integration-tests/docker/environment-configs/override-examples/oss b/integration-tests/docker/environment-configs/override-examples/oss index 6ec534d3578d..dc2cf89085d2 100644 --- a/integration-tests/docker/environment-configs/override-examples/oss +++ b/integration-tests/docker/environment-configs/override-examples/oss @@ -21,10 +21,10 @@ # Example of override config file to provide. # Please replace with your cloud configs/credentials # -druid_storage_type=aliyun-oss -druid_storage_aliyun_oss_bucket= -druid_storage_aliyun_oss_prefix= -druid_aliyun_oss_accessKey= -druid_aliyun_oss_secretKey= -druid_aliyun_oss_endpoint= -druid_extensions_loadList=["druid-aliyun-oss-extensions"] \ No newline at end of file +druid_storage_type=oss +druid_storage_oss_bucket= +druid_storage_oss_prefix= +druid_oss_accessKey= +druid_oss_secretKey= +druid_oss_endpoint= +druid_extensions_loadList=["aliyun-oss-extensions"] \ No newline at end of file From 57ffebbc2abefcef46afc6553007563b13ae0deb Mon Sep 17 00:00:00 2001 From: frank chen Date: Wed, 1 Jul 2020 14:32:13 +0800 Subject: [PATCH 21/24] fix names in test cases --- .../data/input/aliyun/OssClientConfig.java | 8 ++++ .../storage/aliyun/OssStorageDruidModule.java | 2 +- .../aliyun/OssDataSegmentMoverTest.java | 4 +- .../aliyun/OssDataSegmentPullerTest.java | 42 +++++++++---------- .../druid/storage/aliyun/OssTaskLogsTest.java | 2 +- .../OssTimestampVersionedDataFinderTest.java | 10 ++--- .../org/apache/druid/tests/TestNGGroup.java | 4 +- 7 files changed, 40 insertions(+), 32 deletions(-) diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java index 0ad4539a0e66..9b91dbc98421 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/data/input/aliyun/OssClientConfig.java @@ -22,6 +22,7 @@ import com.aliyun.oss.OSS; import com.aliyun.oss.OSSClientBuilder; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import org.apache.druid.metadata.PasswordProvider; @@ -76,6 +77,13 @@ public PasswordProvider getSecretKey() return secretKey; } + @JsonIgnore + public boolean isCredentialsConfigured() + { + return accessKey != null && + secretKey != null; + } + @Override public String toString() { diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java index 9ff7da4b381d..d682bbac8232 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java +++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssStorageDruidModule.java @@ -49,7 +49,7 @@ public List getJacksonModules() @Override public String getModuleName() { - return "DruidOss-" + System.identityHashCode(this); + return "DruidAliyunOss-" + System.identityHashCode(this); } @Override diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java index a1ff64950406..2e32f701b621 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java @@ -117,8 +117,8 @@ public void testMoveNoop() throws Exception @Test(expected = SegmentLoadingException.class) public void testMoveException() throws Exception { - MockClient mockS3Client = new MockClient(); - OssDataSegmentMover mover = new OssDataSegmentMover(mockS3Client, new OssStorageConfig()); + MockClient mockClient = new MockClient(); + OssDataSegmentMover mover = new OssDataSegmentMover(mockClient, new OssStorageConfig()); mover.move( SOURCE_SEGMENT, diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java index a90a7504b914..46584cac5e18 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentPullerTest.java @@ -58,7 +58,7 @@ public void testSimpleGetVersion() throws IOException { String bucket = "bucket"; String keyPrefix = "prefix/dir/0"; - OSS s3Client = EasyMock.createStrictMock(OSS.class); + OSS ossClient = EasyMock.createStrictMock(OSS.class); final OSSObjectSummary objectSummary = new OSSObjectSummary(); objectSummary.setBucketName(bucket); @@ -68,16 +68,16 @@ public void testSimpleGetVersion() throws IOException final ObjectListing result = new ObjectListing(); result.getObjectSummaries().add(objectSummary); - EasyMock.expect(s3Client.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) + EasyMock.expect(ossClient.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) .andReturn(result) .once(); - OssDataSegmentPuller puller = new OssDataSegmentPuller(s3Client); + OssDataSegmentPuller puller = new OssDataSegmentPuller(ossClient); - EasyMock.replay(s3Client); + EasyMock.replay(ossClient); String version = puller.getVersion(URI.create(StringUtils.format(OssStorageDruidModule.SCHEME + "://%s/%s", bucket, objectSummary.getKey()))); - EasyMock.verify(s3Client); + EasyMock.verify(ossClient); Assert.assertEquals(StringUtils.format("%d", new Date(0).getTime()), version); } @@ -87,7 +87,7 @@ public void testGZUncompress() throws IOException, SegmentLoadingException { final String bucket = "bucket"; final String keyPrefix = "prefix/dir/0"; - final OSS s3Client = EasyMock.createStrictMock(OSS.class); + final OSS ossClient = EasyMock.createStrictMock(OSS.class); final byte[] value = bucket.getBytes(StandardCharsets.UTF_8); final File tmpFile = temporaryFolder.newFile("gzTest.gz"); @@ -112,25 +112,25 @@ public void testGZUncompress() throws IOException, SegmentLoadingException final File tmpDir = temporaryFolder.newFolder("gzTestDir"); - EasyMock.expect(s3Client.doesObjectExist(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) + EasyMock.expect(ossClient.doesObjectExist(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) .andReturn(true) .once(); - EasyMock.expect(s3Client.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) + EasyMock.expect(ossClient.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) .andReturn(listObjectsResult) .once(); - EasyMock.expect(s3Client.getObject(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) + EasyMock.expect(ossClient.getObject(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) .andReturn(object0) .once(); - OssDataSegmentPuller puller = new OssDataSegmentPuller(s3Client); + OssDataSegmentPuller puller = new OssDataSegmentPuller(ossClient); - EasyMock.replay(s3Client); + EasyMock.replay(ossClient); FileUtils.FileCopyResult result = puller.getSegmentFiles( new CloudObjectLocation( bucket, object0.getKey() ), tmpDir ); - EasyMock.verify(s3Client); + EasyMock.verify(ossClient); Assert.assertEquals(value.length, result.size()); File expected = new File(tmpDir, "renames-0"); @@ -143,7 +143,7 @@ public void testGZUncompressRetries() throws IOException, SegmentLoadingExceptio { final String bucket = "bucket"; final String keyPrefix = "prefix/dir/0"; - final OSS s3Client = EasyMock.createStrictMock(OSS.class); + final OSS ossClient = EasyMock.createStrictMock(OSS.class); final byte[] value = bucket.getBytes(StandardCharsets.UTF_8); final File tmpFile = temporaryFolder.newFile("gzTest.gz"); @@ -170,31 +170,31 @@ public void testGZUncompressRetries() throws IOException, SegmentLoadingExceptio File tmpDir = temporaryFolder.newFolder("gzTestDir"); OSSException exception = new OSSException("OssDataSegmentPullerTest", "NoSuchKey", null, null, null, null, null); - EasyMock.expect(s3Client.doesObjectExist(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) + EasyMock.expect(ossClient.doesObjectExist(EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey()))) .andReturn(true) .once(); - EasyMock.expect(s3Client.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) + EasyMock.expect(ossClient.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) .andReturn(listObjectsResult) .once(); - EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))) + EasyMock.expect(ossClient.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))) .andThrow(exception) .once(); - EasyMock.expect(s3Client.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) + EasyMock.expect(ossClient.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) .andReturn(listObjectsResult) .once(); - EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))) + EasyMock.expect(ossClient.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey()))) .andReturn(object0) .once(); - OssDataSegmentPuller puller = new OssDataSegmentPuller(s3Client); + OssDataSegmentPuller puller = new OssDataSegmentPuller(ossClient); - EasyMock.replay(s3Client); + EasyMock.replay(ossClient); FileUtils.FileCopyResult result = puller.getSegmentFiles( new CloudObjectLocation( bucket, object0.getKey() ), tmpDir ); - EasyMock.verify(s3Client); + EasyMock.verify(ossClient); Assert.assertEquals(value.length, result.size()); File expected = new File(tmpDir, "renames-0"); diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java index dc0debe539a0..1264a0fe9d7a 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTaskLogsTest.java @@ -57,7 +57,7 @@ public class OssTaskLogsTest extends EasyMockSupport private static final String KEY_2 = "key2"; private static final String TEST_BUCKET = "test_bucket"; private static final String TEST_PREFIX = "test_prefix"; - private static final URI PREFIX_URI = URI.create(StringUtils.format("s3://%s/%s", TEST_BUCKET, TEST_PREFIX)); + private static final URI PREFIX_URI = URI.create(StringUtils.format("oss://%s/%s", TEST_BUCKET, TEST_PREFIX)); private static final long TIME_0 = 0L; private static final long TIME_1 = 1L; private static final long TIME_NOW = 2L; diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java index cde32ed683d9..4024fe2fb51d 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java @@ -110,7 +110,7 @@ public void testFindSelf() { String bucket = "bucket"; String keyPrefix = "prefix/dir/0"; - OSS s3Client = EasyMock.createStrictMock(OSS.class); + OSS ossClient = EasyMock.createStrictMock(OSS.class); OSSObjectSummary object0 = new OSSObjectSummary(); @@ -123,19 +123,19 @@ public void testFindSelf() result.getObjectSummaries().add(object0); result.setTruncated(false); - EasyMock.expect(s3Client.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) + EasyMock.expect(ossClient.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) .andReturn(result) .once(); - OssTimestampVersionedDataFinder finder = new OssTimestampVersionedDataFinder(s3Client); + OssTimestampVersionedDataFinder finder = new OssTimestampVersionedDataFinder(ossClient); Pattern pattern = Pattern.compile("renames-[0-9]*\\.gz"); - EasyMock.replay(s3Client); + EasyMock.replay(ossClient); URI latest = finder.getLatestVersion(URI.create(StringUtils.format("%s://%s/%s", OssStorageDruidModule.SCHEME, bucket, keyPrefix)), pattern); - EasyMock.verify(s3Client); + EasyMock.verify(ossClient); URI expected = URI.create(StringUtils.format("%s://%s/%s", OssStorageDruidModule.SCHEME, bucket, object0.getKey())); diff --git a/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java b/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java index 32a65d98ddfd..bc8e613730e0 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/TestNGGroup.java @@ -77,9 +77,9 @@ public class TestNGGroup public static final String AZURE_DEEP_STORAGE = "azure-deep-storage"; /** - * This group is not part of CI. To run this group, azure configs/credentials for your azure must be provided in a file. + * This group is not part of CI. To run this group, azure configs/credentials for your oss must be provided in a file. * The path of the file must then be pass to mvn with -Doverride.config.path= - * See integration-tests/docker/environment-configs/override-examples/azures for env vars to provide. + * See integration-tests/docker/environment-configs/override-examples/oss for env vars to provide. */ public static final String ALIYUN_OSS_DEEP_STORAGE = "aliyun-oss-deep-storage"; From 8c434ce5729751425b5d7ba2e5906e6c38468187 Mon Sep 17 00:00:00 2001 From: frank chen Date: Wed, 1 Jul 2020 14:32:46 +0800 Subject: [PATCH 22/24] add unit test to cover OssInputSource --- .../data/input/aliyun/OssInputSourceTest.java | 660 ++++++++++++++++++ 1 file changed, 660 insertions(+) create mode 100644 extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/data/input/aliyun/OssInputSourceTest.java diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/data/input/aliyun/OssInputSourceTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/data/input/aliyun/OssInputSourceTest.java new file mode 100644 index 000000000000..2bd9d5816acc --- /dev/null +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/data/input/aliyun/OssInputSourceTest.java @@ -0,0 +1,660 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.aliyun; + +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClient; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.GetObjectRequest; +import com.aliyun.oss.model.ListObjectsRequest; +import com.aliyun.oss.model.OSSObject; +import com.aliyun.oss.model.OSSObjectSummary; +import com.aliyun.oss.model.ObjectListing; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.module.guice.ObjectMapperModule; +import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Provides; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.InputSourceReader; +import org.apache.druid.data.input.InputSplit; +import org.apache.druid.data.input.MaxSizeSplitHintSpec; +import org.apache.druid.data.input.impl.CloudObjectLocation; +import org.apache.druid.data.input.impl.CsvInputFormat; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.JsonInputFormat; +import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.initialization.DruidModule; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.parsers.CloseableIterator; +import org.apache.druid.java.util.common.parsers.JSONPathSpec; +import org.apache.druid.metadata.DefaultPasswordProvider; +import org.apache.druid.storage.aliyun.OssInputDataConfig; +import org.apache.druid.storage.aliyun.OssUtils; +import org.apache.druid.testing.InitializedNullHandlingTest; +import org.apache.druid.utils.CompressionUtils; +import org.easymock.EasyMock; +import org.easymock.IArgumentMatcher; +import org.hamcrest.CoreMatchers; +import org.joda.time.DateTime; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.internal.matchers.ThrowableMessageMatcher; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URI; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class OssInputSourceTest extends InitializedNullHandlingTest +{ + private static final ObjectMapper MAPPER = createObjectMapper(); + private static final OSS OSSCLIENT = EasyMock.createMock(OSSClient.class); + private static final OssInputDataConfig INPUT_DATA_CONFIG; + private static final int MAX_LISTING_LENGTH = 10; + + private static final List EXPECTED_URIS = Arrays.asList( + URI.create("oss://foo/bar/file.csv"), + URI.create("oss://bar/foo/file2.csv") + ); + + private static final List EXPECTED_COMPRESSED_URIS = Arrays.asList( + URI.create("oss://foo/bar/file.csv.gz"), + URI.create("oss://bar/foo/file2.csv.gz") + ); + + private static final List> EXPECTED_COORDS = + EXPECTED_URIS.stream() + .map(uri -> Collections.singletonList(new CloudObjectLocation(uri))) + .collect(Collectors.toList()); + + private static final List PREFIXES = Arrays.asList( + URI.create("oss://foo/bar"), + URI.create("oss://bar/foo") + ); + + private static final OssClientConfig CLOUD_CONFIG_PROPERTIES = new OssClientConfig( + "test.oss-cn.aliyun.com", + new DefaultPasswordProvider("myKey"), + new DefaultPasswordProvider("mySecret")); + + private static final List EXPECTED_LOCATION = + ImmutableList.of(new CloudObjectLocation("foo", "bar/file.csv")); + + private static final DateTime NOW = DateTimes.nowUtc(); + private static final byte[] CONTENT = + StringUtils.toUtf8(StringUtils.format("%d,hello,world", NOW.getMillis())); + + static { + INPUT_DATA_CONFIG = new OssInputDataConfig(); + INPUT_DATA_CONFIG.setMaxListingLength(MAX_LISTING_LENGTH); + } + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void testSerdeWithUris() throws Exception + { + final OssInputSource withUris = new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + EXPECTED_URIS, + null, + null, + null + ); + final OssInputSource serdeWithUris = MAPPER.readValue(MAPPER.writeValueAsString(withUris), OssInputSource.class); + Assert.assertEquals(withUris, serdeWithUris); + } + + @Test + public void testSerdeWithPrefixes() throws Exception + { + final OssInputSource withPrefixes = new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + null, + PREFIXES, + null, + null + ); + final OssInputSource serdeWithPrefixes = + MAPPER.readValue(MAPPER.writeValueAsString(withPrefixes), OssInputSource.class); + Assert.assertEquals(withPrefixes, serdeWithPrefixes); + } + + @Test + public void testSerdeWithObjects() throws Exception + { + final OssInputSource withPrefixes = new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + null, + null, + EXPECTED_LOCATION, + null + ); + final OssInputSource serdeWithPrefixes = + MAPPER.readValue(MAPPER.writeValueAsString(withPrefixes), OssInputSource.class); + Assert.assertEquals(withPrefixes, serdeWithPrefixes); + } + + @Test + public void testInputSourceUseDefaultPasswordWhenCloudConfigPropertiesWithoutCrediential() + { + OssClientConfig mockConfigPropertiesWithoutKeyAndSecret = EasyMock.createMock(OssClientConfig.class); + EasyMock.reset(mockConfigPropertiesWithoutKeyAndSecret); + EasyMock.expect(mockConfigPropertiesWithoutKeyAndSecret.isCredentialsConfigured()) + .andStubReturn(false); + EasyMock.expect(mockConfigPropertiesWithoutKeyAndSecret.buildClient()) + .andReturn(OSSCLIENT); + EasyMock.replay(mockConfigPropertiesWithoutKeyAndSecret); + final OssInputSource withPrefixes = new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + null, + null, + EXPECTED_LOCATION, + mockConfigPropertiesWithoutKeyAndSecret + ); + Assert.assertNotNull(withPrefixes); + + withPrefixes.createEntity(new CloudObjectLocation("bucket", "path")); + EasyMock.verify(mockConfigPropertiesWithoutKeyAndSecret); + } + + @Test + public void testSerdeOssClientLazyInitializedWithCrediential() throws Exception + { + OssClientConfig clientConfig = EasyMock.createMock(OssClientConfig.class); + EasyMock.replay(clientConfig); + final OssInputSource withPrefixes = new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + null, + null, + EXPECTED_LOCATION, + CLOUD_CONFIG_PROPERTIES + ); + final OssInputSource serdeWithPrefixes = + MAPPER.readValue(MAPPER.writeValueAsString(withPrefixes), OssInputSource.class); + Assert.assertEquals(withPrefixes, serdeWithPrefixes); + EasyMock.verify(clientConfig); + } + + @Test + public void testSerdeOssClientLazyInitializedWithoutCrediential() throws Exception + { + OssClientConfig clientConfig = EasyMock.createMock(OssClientConfig.class); + EasyMock.replay(clientConfig); + final OssInputSource withPrefixes = new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + null, + null, + EXPECTED_LOCATION, + null + ); + final OssInputSource serdeWithPrefixes = + MAPPER.readValue(MAPPER.writeValueAsString(withPrefixes), OssInputSource.class); + Assert.assertEquals(withPrefixes, serdeWithPrefixes); + EasyMock.verify(clientConfig); + } + + @Test + public void testSerdeWithExtraEmptyLists() throws Exception + { + final OssInputSource withPrefixes = new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + ImmutableList.of(), + ImmutableList.of(), + EXPECTED_LOCATION, + null + ); + final OssInputSource serdeWithPrefixes = + MAPPER.readValue(MAPPER.writeValueAsString(withPrefixes), OssInputSource.class); + Assert.assertEquals(withPrefixes, serdeWithPrefixes); + } + + @Test + public void testSerdeWithInvalidArgs() + { + expectedException.expect(IllegalArgumentException.class); + // constructor will explode + new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + EXPECTED_URIS, + PREFIXES, + EXPECTED_LOCATION, + null + ); + } + + @Test + public void testSerdeWithOtherInvalidArgs() + { + expectedException.expect(IllegalArgumentException.class); + // constructor will explode + new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + EXPECTED_URIS, + PREFIXES, + ImmutableList.of(), + null + ); + } + + @Test + public void testSerdeWithOtherOtherInvalidArgs() + { + expectedException.expect(IllegalArgumentException.class); + // constructor will explode + new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + ImmutableList.of(), + PREFIXES, + EXPECTED_LOCATION, + null + ); + } + + @Test + public void testWithUrisSplit() + { + OssInputSource inputSource = new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + EXPECTED_URIS, + null, + null, + null + ); + + Stream>> splits = inputSource.createSplits( + new JsonInputFormat(JSONPathSpec.DEFAULT, null, null), + null + ); + + Assert.assertEquals(EXPECTED_COORDS, splits.map(InputSplit::get).collect(Collectors.toList())); + } + + @Test + public void testWithPrefixesSplit() + { + EasyMock.reset(OSSCLIENT); + expectListObjects(PREFIXES.get(0), ImmutableList.of(EXPECTED_URIS.get(0)), CONTENT); + expectListObjects(PREFIXES.get(1), ImmutableList.of(EXPECTED_URIS.get(1)), CONTENT); + EasyMock.replay(OSSCLIENT); + + OssInputSource inputSource = new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + null, + PREFIXES, + null, + null + ); + + Stream>> splits = inputSource.createSplits( + new JsonInputFormat(JSONPathSpec.DEFAULT, null, null), + new MaxSizeSplitHintSpec(1L) // set maxSplitSize to 1 so that each inputSplit has only one object + ); + + Assert.assertEquals(EXPECTED_COORDS, splits.map(InputSplit::get).collect(Collectors.toList())); + EasyMock.verify(OSSCLIENT); + } + + @Test + public void testCreateSplitsWithSplitHintSpecRespectingHint() + { + EasyMock.reset(OSSCLIENT); + expectListObjects(PREFIXES.get(0), ImmutableList.of(EXPECTED_URIS.get(0)), CONTENT); + expectListObjects(PREFIXES.get(1), ImmutableList.of(EXPECTED_URIS.get(1)), CONTENT); + EasyMock.replay(OSSCLIENT); + + OssInputSource inputSource = new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + null, + PREFIXES, + null, + null + ); + + Stream>> splits = inputSource.createSplits( + new JsonInputFormat(JSONPathSpec.DEFAULT, null, null), + new MaxSizeSplitHintSpec(CONTENT.length * 3L) + ); + + Assert.assertEquals( + ImmutableList.of(EXPECTED_URIS.stream().map(CloudObjectLocation::new).collect(Collectors.toList())), + splits.map(InputSplit::get).collect(Collectors.toList()) + ); + EasyMock.verify(OSSCLIENT); + } + + @Test + public void testCreateSplitsWithEmptyObjectsIteratingOnlyNonEmptyObjects() + { + EasyMock.reset(OSSCLIENT); + expectListObjects(PREFIXES.get(0), ImmutableList.of(EXPECTED_URIS.get(0)), CONTENT); + expectListObjects(PREFIXES.get(1), ImmutableList.of(EXPECTED_URIS.get(1)), new byte[0]); + EasyMock.replay(OSSCLIENT); + + OssInputSource inputSource = new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + null, + PREFIXES, + null, + null + ); + + Stream>> splits = inputSource.createSplits( + new JsonInputFormat(JSONPathSpec.DEFAULT, null, null), + null + ); + Assert.assertEquals( + ImmutableList.of(ImmutableList.of(new CloudObjectLocation(EXPECTED_URIS.get(0)))), + splits.map(InputSplit::get).collect(Collectors.toList()) + ); + EasyMock.verify(OSSCLIENT); + } + + @Test + public void testAccessDeniedWhileListingPrefix() + { + EasyMock.reset(OSSCLIENT); + expectListObjects(PREFIXES.get(0), ImmutableList.of(EXPECTED_URIS.get(0)), CONTENT); + expectListObjectsAndThrowAccessDenied(EXPECTED_URIS.get(1)); + EasyMock.replay(OSSCLIENT); + + OssInputSource inputSource = new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + null, + ImmutableList.of(PREFIXES.get(0), EXPECTED_URIS.get(1)), + null, + null + ); + + expectedException.expectMessage("Failed to get object summaries from aliyun OSS bucket[bar], prefix[foo/file2.csv]"); + expectedException.expectCause( + ThrowableMessageMatcher.hasMessage(CoreMatchers.containsString("can't list that bucket")) + ); + + inputSource.createSplits( + new JsonInputFormat(JSONPathSpec.DEFAULT, null, null), + null + ).collect(Collectors.toList()); + } + + @Test + public void testReader() throws IOException + { + EasyMock.reset(OSSCLIENT); + expectListObjects(PREFIXES.get(0), ImmutableList.of(EXPECTED_URIS.get(0)), CONTENT); + expectListObjects(EXPECTED_URIS.get(1), ImmutableList.of(EXPECTED_URIS.get(1)), CONTENT); + expectGetObject(EXPECTED_URIS.get(0)); + expectGetObject(EXPECTED_URIS.get(1)); + EasyMock.replay(OSSCLIENT); + + OssInputSource inputSource = new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + null, + ImmutableList.of(PREFIXES.get(0), EXPECTED_URIS.get(1)), + null, + null + ); + + InputRowSchema someSchema = new InputRowSchema( + new TimestampSpec("time", "auto", null), + new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("dim1", "dim2"))), + ImmutableList.of("count") + ); + + InputSourceReader reader = inputSource.reader( + someSchema, + new CsvInputFormat(ImmutableList.of("time", "dim1", "dim2"), "|", false, null, 0), + temporaryFolder.newFolder() + ); + + CloseableIterator iterator = reader.read(); + + while (iterator.hasNext()) { + InputRow nextRow = iterator.next(); + Assert.assertEquals(NOW, nextRow.getTimestamp()); + Assert.assertEquals("hello", nextRow.getDimension("dim1").get(0)); + Assert.assertEquals("world", nextRow.getDimension("dim2").get(0)); + } + + EasyMock.verify(OSSCLIENT); + } + + @Test + public void testCompressedReader() throws IOException + { + EasyMock.reset(OSSCLIENT); + expectListObjects(PREFIXES.get(0), ImmutableList.of(EXPECTED_COMPRESSED_URIS.get(0)), CONTENT); + expectListObjects(EXPECTED_COMPRESSED_URIS.get(1), ImmutableList.of(EXPECTED_COMPRESSED_URIS.get(1)), CONTENT); + expectGetObjectCompressed(EXPECTED_COMPRESSED_URIS.get(0)); + expectGetObjectCompressed(EXPECTED_COMPRESSED_URIS.get(1)); + EasyMock.replay(OSSCLIENT); + + OssInputSource inputSource = new OssInputSource( + OSSCLIENT, + INPUT_DATA_CONFIG, + null, + ImmutableList.of(PREFIXES.get(0), EXPECTED_COMPRESSED_URIS.get(1)), + null, + null + ); + + InputRowSchema someSchema = new InputRowSchema( + new TimestampSpec("time", "auto", null), + new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("dim1", "dim2"))), + ImmutableList.of("count") + ); + + InputSourceReader reader = inputSource.reader( + someSchema, + new CsvInputFormat(ImmutableList.of("time", "dim1", "dim2"), "|", false, null, 0), + temporaryFolder.newFolder() + ); + + CloseableIterator iterator = reader.read(); + + while (iterator.hasNext()) { + InputRow nextRow = iterator.next(); + Assert.assertEquals(NOW, nextRow.getTimestamp()); + Assert.assertEquals("hello", nextRow.getDimension("dim1").get(0)); + Assert.assertEquals("world", nextRow.getDimension("dim2").get(0)); + } + + EasyMock.verify(OSSCLIENT); + } + + private static void expectListObjects(URI prefix, List uris, byte[] content) + { + final ObjectListing result = new ObjectListing(); + result.setBucketName(prefix.getAuthority()); + result.setMaxKeys(uris.size()); + for (URI uri : uris) { + final String bucket = uri.getAuthority(); + final String key = OssUtils.extractKey(uri); + final OSSObjectSummary objectSummary = new OSSObjectSummary(); + objectSummary.setBucketName(bucket); + objectSummary.setKey(key); + objectSummary.setSize(content.length); + result.getObjectSummaries().add(objectSummary); + } + + EasyMock.expect( + OSSCLIENT.listObjects(matchListObjectsRequest(prefix)) + ).andReturn(result).once(); + } + + private static void expectListObjectsAndThrowAccessDenied(final URI prefix) + { + OSSException boom = new OSSException("oh dang, you can't list that bucket friend"); + boom.setRawResponseError("403"); + EasyMock.expect( + OSSCLIENT.listObjects(matchListObjectsRequest(prefix)) + ).andThrow(boom).once(); + } + + private static void expectGetObject(URI uri) + { + final String bucket = uri.getAuthority(); + final String key = OssUtils.extractKey(uri); + + OSSObject someObject = new OSSObject(); + someObject.setBucketName(bucket); + someObject.setKey(key); + someObject.setObjectContent(new ByteArrayInputStream(CONTENT)); + EasyMock.expect(OSSCLIENT.getObject(EasyMock.anyObject(GetObjectRequest.class))).andReturn(someObject).once(); + } + + private static void expectGetObjectCompressed(URI uri) throws IOException + { + final String bucket = uri.getAuthority(); + final String key = OssUtils.extractKey(uri); + + OSSObject someObject = new OSSObject(); + someObject.setBucketName(bucket); + someObject.setKey(key); + ByteArrayOutputStream gzipped = new ByteArrayOutputStream(); + CompressionUtils.gzip(new ByteArrayInputStream(CONTENT), gzipped); + someObject.setObjectContent(new ByteArrayInputStream(gzipped.toByteArray())); + EasyMock.expect(OSSCLIENT.getObject(EasyMock.anyObject(GetObjectRequest.class))).andReturn(someObject).once(); + } + + private static ListObjectsRequest matchListObjectsRequest(final URI prefixUri) + { + // Use an IArgumentMatcher to verify that the request has the correct bucket and prefix. + EasyMock.reportMatcher( + new IArgumentMatcher() + { + @Override + public boolean matches(Object argument) + { + if (!(argument instanceof ListObjectsRequest)) { + return false; + } + + final ListObjectsRequest request = (ListObjectsRequest) argument; + return prefixUri.getAuthority().equals(request.getBucketName()) + && OssUtils.extractKey(prefixUri).equals(request.getPrefix()); + } + + @Override + public void appendTo(StringBuffer buffer) + { + buffer.append(""); + } + } + ); + + return null; + } + + public static ObjectMapper createObjectMapper() + { + DruidModule baseModule = new TestOssModule(); + final Injector injector = Guice.createInjector( + new ObjectMapperModule(), + baseModule + ); + final ObjectMapper baseMapper = injector.getInstance(ObjectMapper.class); + + baseModule.getJacksonModules().forEach(baseMapper::registerModule); + return baseMapper; + } + + public static class TestOssModule implements DruidModule + { + @Override + public List getJacksonModules() + { + // Deserializer is need for OSS even though it is injected. + // See https://github.com/FasterXML/jackson-databind/issues/962. + return ImmutableList.of( + new SimpleModule() + .addDeserializer(OSS.class, new ItemDeserializer()) + ); + } + + @Override + public void configure(Binder binder) + { + } + + @Provides + public OSS getOssClient() + { + return OSSCLIENT; + } + } + + public static class ItemDeserializer extends StdDeserializer + { + ItemDeserializer() + { + this(null); + } + + ItemDeserializer(Class vc) + { + super(vc); + } + + @Override + public T deserialize(JsonParser jp, DeserializationContext ctxt) + { + throw new UnsupportedOperationException(); + } + } +} From cd44f1d8b3d072cd7647ea91903a7a34c7c898ed Mon Sep 17 00:00:00 2001 From: frank chen Date: Wed, 1 Jul 2020 14:41:23 +0800 Subject: [PATCH 23/24] fix names in test cases --- .../storage/aliyun/OssDataSegmentMoverTest.java | 12 ++++++------ .../aliyun/OssTimestampVersionedDataFinderTest.java | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java index 2e32f701b621..66c6f25006f1 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssDataSegmentMoverTest.java @@ -91,10 +91,10 @@ public void testMove() throws Exception @Test public void testMoveNoop() throws Exception { - MockClient mockS3Client = new MockClient(); - OssDataSegmentMover mover = new OssDataSegmentMover(mockS3Client, new OssStorageConfig()); + MockClient mockOssClient = new MockClient(); + OssDataSegmentMover mover = new OssDataSegmentMover(mockOssClient, new OssStorageConfig()); - mockS3Client.putObject( + mockOssClient.putObject( "archive", "targetBaseKey/test/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/1/0/index.zip" ); @@ -111,7 +111,7 @@ public void testMoveNoop() throws Exception MapUtils.getString(targetLoadSpec, "key") ); Assert.assertEquals("archive", MapUtils.getString(targetLoadSpec, "bucket")); - Assert.assertFalse(mockS3Client.didMove()); + Assert.assertFalse(mockOssClient.didMove()); } @Test(expected = SegmentLoadingException.class) @@ -129,8 +129,8 @@ public void testMoveException() throws Exception @Test public void testIgnoresGoneButAlreadyMoved() throws Exception { - MockClient mockS3Client = new MockClient(); - OssDataSegmentMover mover = new OssDataSegmentMover(mockS3Client, new OssStorageConfig()); + MockClient mockOssClient = new MockClient(); + OssDataSegmentMover mover = new OssDataSegmentMover(mockOssClient, new OssStorageConfig()); mover.move(new DataSegment( "test", Intervals.of("2013-01-01/2013-01-02"), diff --git a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java index 4024fe2fb51d..8443d2f4abc6 100644 --- a/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java +++ b/extensions-contrib/aliyun-oss-extensions/src/test/java/org/apache/druid/storage/aliyun/OssTimestampVersionedDataFinderTest.java @@ -147,7 +147,7 @@ public void testFindExact() { String bucket = "bucket"; String keyPrefix = "prefix/dir/0"; - OSS s3Client = EasyMock.createStrictMock(OSS.class); + OSS ossClient = EasyMock.createStrictMock(OSS.class); OSSObjectSummary object0 = new OSSObjectSummary(); @@ -160,16 +160,16 @@ public void testFindExact() result.getObjectSummaries().add(object0); result.setTruncated(false); - EasyMock.expect(s3Client.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) + EasyMock.expect(ossClient.listObjects(EasyMock.anyObject(ListObjectsRequest.class))) .andReturn(result) .once(); - OssTimestampVersionedDataFinder finder = new OssTimestampVersionedDataFinder(s3Client); + OssTimestampVersionedDataFinder finder = new OssTimestampVersionedDataFinder(ossClient); - EasyMock.replay(s3Client); + EasyMock.replay(ossClient); URI latest = finder.getLatestVersion(URI.create(StringUtils.format("%s://%s/%s", OssStorageDruidModule.SCHEME, bucket, object0.getKey())), null); - EasyMock.verify(s3Client); + EasyMock.verify(ossClient); URI expected = URI.create(StringUtils.format("%s://%s/%s", OssStorageDruidModule.SCHEME, bucket, object0.getKey())); From 870a124d6f8372c0d06638ac6fb9a54494e52d5d Mon Sep 17 00:00:00 2001 From: frank chen Date: Thu, 2 Jul 2020 09:54:50 +0800 Subject: [PATCH 24/24] fix dependency problem reported by CI Signed-off-by: frank chen --- extensions-contrib/aliyun-oss-extensions/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/extensions-contrib/aliyun-oss-extensions/pom.xml b/extensions-contrib/aliyun-oss-extensions/pom.xml index 27122822d523..0597337a43c2 100644 --- a/extensions-contrib/aliyun-oss-extensions/pom.xml +++ b/extensions-contrib/aliyun-oss-extensions/pom.xml @@ -143,6 +143,12 @@ equalsverifier test + + org.hamcrest + hamcrest-core + 1.3 + test +