Skip to content

WIP JAMES-4167 Triage rustf #2930

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Arsnael wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

WIP JAMES-4167 Triage rustf #2930

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 20 additions & 35 deletions server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioDocker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@

package org.apache.james.blob.objectstorage.aws;

import static java.util.Collections.singletonMap;

import java.time.Duration;
import java.util.UUID;

Expand All @@ -34,13 +36,12 @@

public class S3MinioDocker {

public static final DockerImageName DOCKER_IMAGE_NAME = DockerImageName.parse("minio/minio")
.withTag("RELEASE.2025-06-13T11-33-47Z");
public static final DockerImageName DOCKER_IMAGE_NAME = DockerImageName.parse("rustfs/rustfs")
.withTag("1.0.0-alpha.81");

public static final int MINIO_PORT = 9000;
public static final int MINIO_WEB_ADMIN_PORT = 9090;
public static final String MINIO_ROOT_USER = "minio";
public static final String MINIO_ROOT_PASSWORD = "minio123";
public static final int S3_PORT = 9000;
public static final String S3_ACCESS_KEY = "testaccesskey";
public static final String S3_SECRET_KEY = "testsecretkey";

private final GenericContainer<?> container;

Expand All @@ -55,25 +56,21 @@ public S3MinioDocker(Network network) {

private GenericContainer<?> getContainer() {
return new GenericContainer<>(DOCKER_IMAGE_NAME)
.withExposedPorts(MINIO_PORT, MINIO_WEB_ADMIN_PORT)
.withEnv("MINIO_ROOT_USER", MINIO_ROOT_USER)
.withEnv("MINIO_ROOT_PASSWORD", MINIO_ROOT_PASSWORD)
.withCommand("server", "--certs-dir", "/opt/minio/certs", "/data", "--console-address", ":" + MINIO_WEB_ADMIN_PORT)
.withClasspathResourceMapping("/minio/private.key",
"/opt/minio/certs/private.key",
BindMode.READ_ONLY)
.withClasspathResourceMapping("/minio/public.crt",
"/opt/minio/certs/public.crt",
BindMode.READ_ONLY)
.waitingFor(Wait.forLogMessage(".*MinIO Object Storage Server.*", 1)
.withExposedPorts(S3_PORT)
.withEnv("RUSTFS_ACCESS_KEY", S3_ACCESS_KEY)
.withEnv("RUSTFS_SECRET_KEY", S3_SECRET_KEY)
.withEnv("RUSTFS_VOLUMES", "/data/rustfs{0..3}")
.withTmpFs(singletonMap("/data", "rw,mode=1777"))
//.waitingFor(Wait.forLogMessage(".*started successfully.*", 1)
.waitingFor(Wait.forLogMessage(".*Console WebUI.*", 2)
.withStartupTimeout(Duration.ofMinutes(2)))
.withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("james-minio-s3-test-" + UUID.randomUUID()));
// .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("james-rustfs-s3-test-" + UUID.randomUUID()));
.withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("james-rustfs-s3-test"));
}

public void start() {
if (!container.isRunning()) {
container.start();
setupMC();
}
}

Expand All @@ -85,25 +82,13 @@ public AwsS3AuthConfiguration getAwsS3AuthConfiguration() {
Preconditions.checkArgument(container.isRunning(), "Container is not running");
return AwsS3AuthConfiguration.builder()
.endpoint(Throwing.supplier(() -> new URIBuilder()
.setScheme("https")
.setScheme("http")
.setHost(container.getHost())
.setPort(container.getMappedPort(MINIO_PORT))
.setPort(container.getMappedPort(S3_PORT))
.build()).get())
.accessKeyId(MINIO_ROOT_USER)
.secretKey(MINIO_ROOT_PASSWORD)
.accessKeyId(S3_ACCESS_KEY)
.secretKey(S3_SECRET_KEY)
.trustAll(true)
.build();
}

private void setupMC() {
Preconditions.checkArgument(container.isRunning(), "Container is not running");
Throwing.runnable(() -> container.execInContainer("mc", "alias", "set", "--insecure", "james", "https://localhost:9000", MINIO_ROOT_USER, MINIO_ROOT_PASSWORD)).run();
}

public void flushAll() {
// Remove all objects
Throwing.runnable(() -> container.execInContainer("mc", "--insecure", "rm", "--recursive", "--force", "--dangerous", "james/")).run();
// Remove all buckets
Throwing.runnable(() -> container.execInContainer("mc", "--insecure", "rb", "--force", "--dangerous", "james/")).run();
}
}
5 changes: 0 additions & 5 deletions .../blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioExtension.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,6 @@ public void afterAll(ExtensionContext extensionContext) {
s3MinioDocker.stop();
}

@Override
public void afterEach(ExtensionContext extensionContext) {
s3MinioDocker.flushAll();
}

@Override
public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
return parameterContext.getParameter().getType() == S3MinioDocker.class;
Expand Down
47 changes: 4 additions & 43 deletions server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,28 +24,23 @@
import static org.apache.james.blob.api.BlobStoreDAOFixture.TEST_BUCKET_NAME;
import static org.apache.james.blob.objectstorage.aws.S3BlobStoreConfiguration.UPLOAD_RETRY_EXCEPTION_PREDICATE;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

import java.util.Optional;
import java.util.concurrent.ExecutionException;

import org.apache.james.blob.api.BlobId;
import org.apache.james.blob.api.BlobStoreDAO;
import org.apache.james.blob.api.BlobStoreDAOContract;
import org.apache.james.blob.api.TestBlobId;
import org.apache.james.metrics.api.NoopGaugeRegistry;
import org.apache.james.metrics.tests.RecordingMetricFactory;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;

import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.util.retry.Retry;
import software.amazon.awssdk.services.s3.model.S3Exception;

public class S3MinioTest implements BlobStoreDAOContract {

Expand Down Expand Up @@ -75,56 +70,22 @@ static void tearDownClass() {
s3ClientFactory.close();
}

@BeforeEach
void beforeEach() throws Exception {
// Why? https://github.com/apache/james-project/pull/1981#issuecomment-2380396460
createBucket(TEST_BUCKET_NAME.asString());
}

private void createBucket(String bucketName) throws Exception {
s3ClientFactory.get().createBucket(builder -> builder.bucket(bucketName))
.get();
}

private void deleteBucket(String bucketName) {
try {
s3ClientFactory.get().deleteBucket(builder -> builder.bucket(bucketName))
.get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException("Error while deleting bucket", e);
}
@AfterEach
void tearDown() {
testee.deleteAllBuckets().block();
}

@Override
public BlobStoreDAO testee() {
return testee;
}

@Test
void saveWillThrowWhenBlobIdHasSlashCharacters() {
BlobId invalidBlobId = new TestBlobId("test-blob//id");
assertThatThrownBy(() -> Mono.from(testee.save(TEST_BUCKET_NAME, invalidBlobId, SHORT_BYTEARRAY)).block())
.isInstanceOf(S3Exception.class)
.hasMessageContaining("Object name contains unsupported characters");
}

@Test
void saveShouldWorkWhenValidBlobId() {
Mono.from(testee.save(TEST_BUCKET_NAME, TEST_BLOB_ID, SHORT_BYTEARRAY)).block();
assertThat(Mono.from(testee.readBytes(TEST_BUCKET_NAME, TEST_BLOB_ID)).block()).isEqualTo(SHORT_BYTEARRAY);
}

@Test
@Override
public void listBucketsShouldReturnEmptyWhenNone() {
deleteBucket(TEST_BUCKET_NAME.asString());

BlobStoreDAO store = testee();

assertThat(Flux.from(store.listBuckets()).collectList().block())
.isEmpty();
}

@Test
@Override
@Disabled("S3minio return `Connection: close` in header response, https://github.com/apache/james-project/pull/1981#issuecomment-2380396460")
Expand Down