From e45c6b1b704c3943d491159ad16270d6cedefeea Mon Sep 17 00:00:00 2001 From: Pavel Popov Date: Sun, 19 Nov 2017 00:32:26 +0300 Subject: [PATCH 1/4] Add support for ceph --- modules/ceph/pom.xml | 30 ++++ .../containers/CephContainer.java | 133 ++++++++++++++++++ .../containers/CephContainerTest.java | 98 +++++++++++++ pom.xml | 1 + 4 files changed, 262 insertions(+) create mode 100644 modules/ceph/pom.xml create mode 100644 modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java create mode 100644 modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java diff --git a/modules/ceph/pom.xml b/modules/ceph/pom.xml new file mode 100644 index 00000000000..d933b9eabd7 --- /dev/null +++ b/modules/ceph/pom.xml @@ -0,0 +1,30 @@ + + + 4.0.0 + + + org.testcontainers + testcontainers-parent + 0-SNAPSHOT + ../../pom.xml + + + ceph + TestContainers :: ceph + + + + ${project.groupId} + testcontainers + ${project.version} + + + com.amazonaws + aws-java-sdk-s3 + 1.11.231 + + + + \ No newline at end of file diff --git a/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java b/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java new file mode 100644 index 00000000000..c892913f12d --- /dev/null +++ b/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java @@ -0,0 +1,133 @@ +package org.testcontainers.containers; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import org.testcontainers.containers.wait.LogMessageWaitStrategy; + +import java.time.Duration; + +public class CephContainer> extends GenericContainer { + + private static final String IMAGE = "ceph/demo"; + + private String awsAccessKey = "ceph"; + + private String awsSecretKey = "ceph"; + + private String bucketName = "CEPH"; + + private String rgwName = "localhost"; + + private String demoUid = "ceph"; + + private NetworkAutoDetectMode networkAutoDetectMode = NetworkAutoDetectMode.IPV4_ONLY; + + private Duration startupTimeout = Duration.ofSeconds(20); + + public CephContainer() { + super(IMAGE + ":latest"); + } + + public CephContainer(String dockerImageName) { + super(dockerImageName); + } + + @Override + protected void configure() { + withEnv("RGW_NAME", rgwName); + withEnv("NETWORK_AUTO_DETECT", networkAutoDetectMode.value); + withEnv("CEPH_DEMO_UID", demoUid); + withEnv("CEPH_DEMO_ACCESS_KEY", awsAccessKey); + withEnv("CEPH_DEMO_SECRET_KEY", awsSecretKey); + withEnv("CEPH_DEMO_BUCKET", bucketName); + withExposedPorts(6789, 6800, 6801, 6802, 6803, 6804, 6805, 80, 5000); + waitingFor( + new LogMessageWaitStrategy() + .withRegEx(".*\\/entrypoint.sh: SUCCESS\n") + .withStartupTimeout(startupTimeout) + ); + } + + public AWSCredentialsProvider getAWSCredentialsProvider() { + return new AWSStaticCredentialsProvider( + new BasicAWSCredentials(awsAccessKey, awsSecretKey) + ); + } + + public AwsClientBuilder.EndpointConfiguration getAWSEndpointConfiguration() { + return new AwsClientBuilder.EndpointConfiguration( + getContainerIpAddress() + ":" + getMappedPort(80), + "us-east-1" + ); + } + + public SELF withAwsAccessKey(String awsAccessKey) { + this.awsAccessKey = awsAccessKey; + return self(); + } + + public SELF withAwsSecretKey(String awsSecretKey) { + this.awsSecretKey = awsSecretKey; + return self(); + } + + public String getBucketName() { + return bucketName; + } + + public SELF withBucketName(String bucketName) { + //because s3cmd transforming bucket name to uppercase + this.bucketName = bucketName.toUpperCase(); + return self(); + } + + public String getRgwName() { + return rgwName; + } + + public SELF withRgwName(String rgwName) { + this.rgwName = rgwName; + return self(); + } + + public String getDemoUid() { + return demoUid; + } + + public SELF withDemoUid(String demoUid) { + this.demoUid = demoUid; + return self(); + } + + public NetworkAutoDetectMode getNetworkAutoDetectMode() { + return networkAutoDetectMode; + } + + public SELF withNetworkAutoDetectMode(NetworkAutoDetectMode networkAutoDetectMode) { + this.networkAutoDetectMode = networkAutoDetectMode; + return self(); + } + + public Duration getStartupTimeout() { + return startupTimeout; + } + + public SELF withStartupTimeout(Duration startupTimeout) { + this.startupTimeout = startupTimeout; + return self(); + } + + public enum NetworkAutoDetectMode { + IPV6_OR_IPV4("1"), + IPV4_ONLY("4"), + IPV6_ONLY("6"); + + private String value; + + NetworkAutoDetectMode(String value) { + this.value = value; + } + } +} diff --git a/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java b/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java new file mode 100644 index 00000000000..d69d519c8ff --- /dev/null +++ b/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java @@ -0,0 +1,98 @@ +package org.testcontainers.containers; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.Bucket; +import com.amazonaws.services.s3.model.S3Object; +import org.apache.commons.io.IOUtils; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.Date; +import java.util.List; + +import static org.junit.Assert.*; + +public class CephContainerTest { + + @Rule + public CephContainer cephContainer = new CephContainer() + .withAwsAccessKey("test") + .withAwsSecretKey("test") + .withBucketName("test") + .withStartupTimeout(Duration.ofMinutes(5)); + + private AmazonS3 amazonS3; + + @Before + public void setUp() { + amazonS3 = AmazonS3ClientBuilder.standard() + .withCredentials(cephContainer.getAWSCredentialsProvider()) + .withEndpointConfiguration(cephContainer.getAWSEndpointConfiguration()) + .withPathStyleAccessEnabled(true) + .withClientConfiguration( + new ClientConfiguration() + .withProtocol(Protocol.HTTP) + .withSignerOverride("S3SignerType") + ) + .build(); + } + + @Test + public void testS3Bucket() { + String bucketName = "test2"; + + //check current bucket + assertTrue(amazonS3.doesBucketExistV2(cephContainer.getBucketName())); + + //create another bucket + amazonS3.createBucket(bucketName); + assertTrue(amazonS3.doesBucketExistV2(bucketName)); + List buckets = amazonS3.listBuckets(); + assertEquals(2, buckets.size()); + assertTrue(buckets.stream().anyMatch( + bucket -> bucket.getName().equals(bucketName) + )); + + //remove bucket + amazonS3.deleteBucket(bucketName); + assertFalse(amazonS3.doesBucketExistV2(bucketName)); + buckets = amazonS3.listBuckets(); + assertEquals(1, buckets.size()); + assertFalse(buckets.stream().anyMatch( + bucket -> bucket.getName().equals(bucketName) + )); + } + + @Test + public void testS3Object() throws IOException { + String objectId = "test"; + String testData = "This is test data"; + + //put object + amazonS3.putObject(cephContainer.getBucketName(), objectId, testData); + assertEquals(1, amazonS3.listObjects(cephContainer.getBucketName()).getObjectSummaries().size()); + S3Object object = amazonS3.getObject(cephContainer.getBucketName(), objectId); + assertEquals(testData, IOUtils.toString(object.getObjectContent(), StandardCharsets.UTF_8)); + + //generate presigned url and download file + URL url = amazonS3.generatePresignedUrl( + cephContainer.getBucketName(), + objectId, + Date.from(Instant.now().plusSeconds(Duration.ofMinutes(5).toMillis()))); + + try (InputStream inputStream = url.openStream()) { + assertEquals(testData, IOUtils.toString(inputStream, StandardCharsets.UTF_8)); + } + } + +} diff --git a/pom.xml b/pom.xml index bf158718386..7aa61de3745 100644 --- a/pom.xml +++ b/pom.xml @@ -166,6 +166,7 @@ modules/selenium modules/nginx modules/jdbc-test + modules/ceph From ef38b4249b2d031ec90db09d6186a8b6567ce20e Mon Sep 17 00:00:00 2001 From: Pavel Popov Date: Sun, 19 Nov 2017 15:58:13 +0300 Subject: [PATCH 2/4] Review fixes --- modules/ceph/pom.xml | 1 + .../containers/CephContainer.java | 60 +++++++------------ .../containers/CephContainerTest.java | 3 +- 3 files changed, 23 insertions(+), 41 deletions(-) diff --git a/modules/ceph/pom.xml b/modules/ceph/pom.xml index d933b9eabd7..63c6ac43f1d 100644 --- a/modules/ceph/pom.xml +++ b/modules/ceph/pom.xml @@ -24,6 +24,7 @@ com.amazonaws aws-java-sdk-s3 1.11.231 + provided diff --git a/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java b/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java index c892913f12d..565f3ec0390 100644 --- a/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java +++ b/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java @@ -4,13 +4,18 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; -import org.testcontainers.containers.wait.LogMessageWaitStrategy; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.testcontainers.containers.wait.HttpWaitStrategy; import java.time.Duration; +@Getter public class CephContainer> extends GenericContainer { - private static final String IMAGE = "ceph/demo"; + public static final String IMAGE = "ceph/demo"; + public static final int S3_PORT = 80; + public static final int REST_API_PORT = 5000; private String awsAccessKey = "ceph"; @@ -24,10 +29,8 @@ public class CephContainer> extends GenericCont private NetworkAutoDetectMode networkAutoDetectMode = NetworkAutoDetectMode.IPV4_ONLY; - private Duration startupTimeout = Duration.ofSeconds(20); - public CephContainer() { - super(IMAGE + ":latest"); + super(IMAGE + ":tag-stable-3.0-jewel-ubuntu-16.04"); } public CephContainer(String dockerImageName) { @@ -42,14 +45,20 @@ protected void configure() { withEnv("CEPH_DEMO_ACCESS_KEY", awsAccessKey); withEnv("CEPH_DEMO_SECRET_KEY", awsSecretKey); withEnv("CEPH_DEMO_BUCKET", bucketName); - withExposedPorts(6789, 6800, 6801, 6802, 6803, 6804, 6805, 80, 5000); + withExposedPorts(REST_API_PORT, S3_PORT); waitingFor( - new LogMessageWaitStrategy() - .withRegEx(".*\\/entrypoint.sh: SUCCESS\n") - .withStartupTimeout(startupTimeout) + new HttpWaitStrategy() + .forPath("/api/v0.1/health") + .forStatusCode(200) + .withStartupTimeout(Duration.ofMinutes(5)) ); } + @Override + protected Integer getLivenessCheckPort() { + return getMappedPort(REST_API_PORT); + } + public AWSCredentialsProvider getAWSCredentialsProvider() { return new AWSStaticCredentialsProvider( new BasicAWSCredentials(awsAccessKey, awsSecretKey) @@ -58,7 +67,7 @@ public AWSCredentialsProvider getAWSCredentialsProvider() { public AwsClientBuilder.EndpointConfiguration getAWSEndpointConfiguration() { return new AwsClientBuilder.EndpointConfiguration( - getContainerIpAddress() + ":" + getMappedPort(80), + getContainerIpAddress() + ":" + getMappedPort(S3_PORT), "us-east-1" ); } @@ -73,61 +82,34 @@ public SELF withAwsSecretKey(String awsSecretKey) { return self(); } - public String getBucketName() { - return bucketName; - } - public SELF withBucketName(String bucketName) { //because s3cmd transforming bucket name to uppercase this.bucketName = bucketName.toUpperCase(); return self(); } - public String getRgwName() { - return rgwName; - } - public SELF withRgwName(String rgwName) { this.rgwName = rgwName; return self(); } - public String getDemoUid() { - return demoUid; - } - public SELF withDemoUid(String demoUid) { this.demoUid = demoUid; return self(); } - public NetworkAutoDetectMode getNetworkAutoDetectMode() { - return networkAutoDetectMode; - } - public SELF withNetworkAutoDetectMode(NetworkAutoDetectMode networkAutoDetectMode) { this.networkAutoDetectMode = networkAutoDetectMode; return self(); } - public Duration getStartupTimeout() { - return startupTimeout; - } - - public SELF withStartupTimeout(Duration startupTimeout) { - this.startupTimeout = startupTimeout; - return self(); - } - + @AllArgsConstructor public enum NetworkAutoDetectMode { IPV6_OR_IPV4("1"), IPV4_ONLY("4"), IPV6_ONLY("6"); - private String value; + private final String value; - NetworkAutoDetectMode(String value) { - this.value = value; - } } } diff --git a/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java b/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java index d69d519c8ff..71c1239c355 100644 --- a/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java +++ b/modules/ceph/src/test/java/org/testcontainers/containers/CephContainerTest.java @@ -28,8 +28,7 @@ public class CephContainerTest { public CephContainer cephContainer = new CephContainer() .withAwsAccessKey("test") .withAwsSecretKey("test") - .withBucketName("test") - .withStartupTimeout(Duration.ofMinutes(5)); + .withBucketName("test"); private AmazonS3 amazonS3; From b5ab8cf7d2e731e4292f7ecf148afb5df9b46779 Mon Sep 17 00:00:00 2001 From: Pavel Popov Date: Mon, 26 Feb 2018 15:03:47 +0300 Subject: [PATCH 3/4] Fix deps --- modules/ceph/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/ceph/build.gradle b/modules/ceph/build.gradle index 721b2e57f78..9e5db0f67c0 100644 --- a/modules/ceph/build.gradle +++ b/modules/ceph/build.gradle @@ -1,5 +1,7 @@ description = "Testcontainers :: JDBC :: Ceph" dependencies { + compile project(':testcontainers') + provided 'com.amazonaws:aws-java-sdk-s3:1.11.231' } From 736437e4641c85385fbdf658391518cf9c4af810 Mon Sep 17 00:00:00 2001 From: Pavel Popov Date: Sun, 10 Feb 2019 04:23:05 +0300 Subject: [PATCH 4/4] Another review fixes; Switch to new ceph image --- modules/ceph/build.gradle | 4 +-- .../containers/CephContainer.java | 33 ++++++++++--------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/modules/ceph/build.gradle b/modules/ceph/build.gradle index 9e5db0f67c0..75a6a98c444 100644 --- a/modules/ceph/build.gradle +++ b/modules/ceph/build.gradle @@ -1,7 +1,7 @@ -description = "Testcontainers :: JDBC :: Ceph" +description = "Testcontainers :: Ceph" dependencies { compile project(':testcontainers') - provided 'com.amazonaws:aws-java-sdk-s3:1.11.231' + provided 'com.amazonaws:aws-java-sdk-s3:1.11.495' } diff --git a/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java b/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java index 565f3ec0390..547919d9e58 100644 --- a/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java +++ b/modules/ceph/src/main/java/org/testcontainers/containers/CephContainer.java @@ -6,15 +6,18 @@ import com.amazonaws.client.builder.AwsClientBuilder; import lombok.AllArgsConstructor; import lombok.Getter; -import org.testcontainers.containers.wait.HttpWaitStrategy; +import lombok.extern.slf4j.Slf4j; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; import java.time.Duration; +@Slf4j @Getter -public class CephContainer> extends GenericContainer { +public class CephContainer extends GenericContainer { - public static final String IMAGE = "ceph/demo"; - public static final int S3_PORT = 80; + public static final String IMAGE = "ceph/daemon"; + public static final int S3_PORT = 8080; public static final int REST_API_PORT = 5000; private String awsAccessKey = "ceph"; @@ -30,7 +33,7 @@ public class CephContainer> extends GenericCont private NetworkAutoDetectMode networkAutoDetectMode = NetworkAutoDetectMode.IPV4_ONLY; public CephContainer() { - super(IMAGE + ":tag-stable-3.0-jewel-ubuntu-16.04"); + super(IMAGE + ":v3.0.5-stable-3.0-luminous-centos-7"); } public CephContainer(String dockerImageName) { @@ -41,6 +44,7 @@ public CephContainer(String dockerImageName) { protected void configure() { withEnv("RGW_NAME", rgwName); withEnv("NETWORK_AUTO_DETECT", networkAutoDetectMode.value); + withEnv("CEPH_DAEMON", "demo"); withEnv("CEPH_DEMO_UID", demoUid); withEnv("CEPH_DEMO_ACCESS_KEY", awsAccessKey); withEnv("CEPH_DEMO_SECRET_KEY", awsSecretKey); @@ -49,14 +53,11 @@ protected void configure() { waitingFor( new HttpWaitStrategy() .forPath("/api/v0.1/health") + .forPort(REST_API_PORT) .forStatusCode(200) .withStartupTimeout(Duration.ofMinutes(5)) ); - } - - @Override - protected Integer getLivenessCheckPort() { - return getMappedPort(REST_API_PORT); + withLogConsumer(new Slf4jLogConsumer(log)); } public AWSCredentialsProvider getAWSCredentialsProvider() { @@ -72,33 +73,33 @@ public AwsClientBuilder.EndpointConfiguration getAWSEndpointConfiguration() { ); } - public SELF withAwsAccessKey(String awsAccessKey) { + public CephContainer withAwsAccessKey(String awsAccessKey) { this.awsAccessKey = awsAccessKey; return self(); } - public SELF withAwsSecretKey(String awsSecretKey) { + public CephContainer withAwsSecretKey(String awsSecretKey) { this.awsSecretKey = awsSecretKey; return self(); } - public SELF withBucketName(String bucketName) { + public CephContainer withBucketName(String bucketName) { //because s3cmd transforming bucket name to uppercase this.bucketName = bucketName.toUpperCase(); return self(); } - public SELF withRgwName(String rgwName) { + public CephContainer withRgwName(String rgwName) { this.rgwName = rgwName; return self(); } - public SELF withDemoUid(String demoUid) { + public CephContainer withDemoUid(String demoUid) { this.demoUid = demoUid; return self(); } - public SELF withNetworkAutoDetectMode(NetworkAutoDetectMode networkAutoDetectMode) { + public CephContainer withNetworkAutoDetectMode(NetworkAutoDetectMode networkAutoDetectMode) { this.networkAutoDetectMode = networkAutoDetectMode; return self(); }