From 1603472d869b97c1791fda27a5d9f5905ea45341 Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Mon, 27 Jun 2022 00:29:43 -0700 Subject: [PATCH 1/4] HDDS-6942. Fix default group permissions for S3 API Ozone vol/bucket/objects created via S3 should not allow read access for users in same group --- .../org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index b3f4d914e33d..da7c0e4aea0e 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.function.Function; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.audit.AuditAction; import org.apache.hadoop.ozone.audit.AuditEventStatus; import org.apache.hadoop.ozone.audit.AuditLogger; @@ -63,6 +64,8 @@ public abstract class EndpointBase implements Auditor { private S3Auth s3Auth; @Context private ContainerRequestContext context; + @Inject + private OzoneConfiguration ozoneConfiguration; private static final Logger LOG = LoggerFactory.getLogger(EndpointBase.class); @@ -98,6 +101,7 @@ protected OzoneBucket getBucket(OzoneVolume volume, String bucketName) @PostConstruct public void initialization() { LOG.debug("S3 access id: {}", s3Auth.getAccessID()); + ozoneConfiguration.setIfUnset("ozone.om.group.rights", "NONE"); getClient().getObjectStore() .getClientProxy() .setThreadLocalS3Auth(s3Auth); From 82751def298b1de6e58e2aea56026e0f268abbf9 Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Mon, 27 Jun 2022 19:13:57 -0700 Subject: [PATCH 2/4] Move S3 configuration setup to OzoneConfigurationHolder --- .../apache/hadoop/ozone/s3/OzoneConfigurationHolder.java | 7 +++++++ .../org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java | 4 ---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java index 4aeab1f3c4a6..05933c50a176 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java @@ -39,5 +39,12 @@ public OzoneConfiguration configuration() { public static void setConfiguration( OzoneConfiguration conf) { OzoneConfigurationHolder.configuration = conf; + applyOzoneS3Configs(); + } + + private static void applyOzoneS3Configs() { + OzoneConfigurationHolder + .configuration + .setIfUnset("ozone.om.group.rights", "NONE"); } } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index da7c0e4aea0e..b3f4d914e33d 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.function.Function; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.audit.AuditAction; import org.apache.hadoop.ozone.audit.AuditEventStatus; import org.apache.hadoop.ozone.audit.AuditLogger; @@ -64,8 +63,6 @@ public abstract class EndpointBase implements Auditor { private S3Auth s3Auth; @Context private ContainerRequestContext context; - @Inject - private OzoneConfiguration ozoneConfiguration; private static final Logger LOG = LoggerFactory.getLogger(EndpointBase.class); @@ -101,7 +98,6 @@ protected OzoneBucket getBucket(OzoneVolume volume, String bucketName) @PostConstruct public void initialization() { LOG.debug("S3 access id: {}", s3Auth.getAccessID()); - ozoneConfiguration.setIfUnset("ozone.om.group.rights", "NONE"); getClient().getObjectStore() .getClientProxy() .setThreadLocalS3Auth(s3Auth); From ba640bd0925ff23863b7336bcdaea6191e262235 Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Tue, 28 Jun 2022 18:38:07 -0700 Subject: [PATCH 3/4] Add smoke test --- hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot b/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot index 9b9077fab5fc..2429303a82e5 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot @@ -40,3 +40,7 @@ Create bucket with invalid bucket name ${randStr} = Generate Ozone String ${result} = Execute AWSS3APICli and checkrc create-bucket --bucket invalid_bucket_${randStr} 255 Should contain ${result} InvalidBucketName +Create new bucket and check no group ACL + ${bucket} = Create bucket + ${acl} = Execute ozone sh bucket getacl s3v/${bucket} + Should not contain ${acl} \"type\" : \"GROUP\" \ No newline at end of file From 1a2e7dc67034d9e08881215d783a27dc3aaac951 Mon Sep 17 00:00:00 2001 From: Ritesh H Shukla Date: Thu, 30 Jun 2022 03:43:33 -0700 Subject: [PATCH 4/4] Try config fix against CI --- .../dist/src/main/smoketest/s3/bucketcreate.robot | 8 +++++++- .../dist/src/main/smoketest/s3/commonawslib.robot | 3 ++- .../org/apache/hadoop/ozone/s3/OzoneClientProducer.java | 1 + .../apache/hadoop/ozone/s3/OzoneConfigurationHolder.java | 7 ------- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot b/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot index 2429303a82e5..aaf0da4b77cd 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/bucketcreate.robot @@ -43,4 +43,10 @@ Create bucket with invalid bucket name Create new bucket and check no group ACL ${bucket} = Create bucket ${acl} = Execute ozone sh bucket getacl s3v/${bucket} - Should not contain ${acl} \"type\" : \"GROUP\" \ No newline at end of file + ${group} = Get Regexp Matches ${acl} "GROUP" + IF '${group}' is not '[]' + ${json} = Evaluate json.loads('''${acl}''') json + # make sure this check is for group acl + Should contain ${json}[1][type] GROUP + Should contain ${json}[1][aclList] NONE + END \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot index 6f3bbef175d1..0b03f3c75e30 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot @@ -65,7 +65,8 @@ Setup v4 headers Setup secure v4 headers ${result} = Execute ozone s3 getsecret ${OM_HA_PARAM} ${accessKey} = Get Regexp Matches ${result} (?<=awsAccessKey=).* - ${accessKey} = Get Variable Value ${accessKey} sdsdasaasdasd + # Use a valid user that are created in the Docket image Ex: testuser if it is not a secure cluster + ${accessKey} = Get Variable Value ${accessKey} testuser ${secret} = Get Regexp Matches ${result} (?<=awsSecret=).* ${accessKey} = Set Variable ${accessKey[0]} ${secret} = Set Variable ${secret[0]} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java index ca8d32b48d4f..4df235cca323 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java @@ -66,6 +66,7 @@ public class OzoneClientProducer { @Produces public synchronized OzoneClient createClient() throws WebApplicationException, IOException { + ozoneConfiguration.set("ozone.om.group.rights", "NONE"); client = getClient(ozoneConfiguration); return client; } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java index 05933c50a176..4aeab1f3c4a6 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java @@ -39,12 +39,5 @@ public OzoneConfiguration configuration() { public static void setConfiguration( OzoneConfiguration conf) { OzoneConfigurationHolder.configuration = conf; - applyOzoneS3Configs(); - } - - private static void applyOzoneS3Configs() { - OzoneConfigurationHolder - .configuration - .setIfUnset("ozone.om.group.rights", "NONE"); } }