From 5bee934ed7b82bc6c9baea45be744d4f8ca5e91b Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Wed, 18 Aug 2021 15:29:59 +0100 Subject: [PATCH 1/3] Add first version of instructions for the IDR object store - explain bucket creation, policy and CORS configuration setting --- docs/object-store.md | 95 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 docs/object-store.md diff --git a/docs/object-store.md b/docs/object-store.md new file mode 100644 index 00000000..af2f1155 --- /dev/null +++ b/docs/object-store.md @@ -0,0 +1,95 @@ +# IDR object store + +IDR makes use of object storage to export images and plates from published +studies into a rich cloud-aware representation following the +[OME-NGFF specification](https://ngff.openmicroscopy.org/latest/). + +## Set-up + +The Embassy Cloud Version 4 includes an S3 compatible Object Storage backend. To use the object store, it is required to have a local installation of the AWS CLI as well as AWS access and secret keys. + +Full instructions are given in https://docs.embassy.ebi.ac.uk/userguide/Embassyv4.html#s3-object-store. + + +## Public bucket creation + +The following examples use `idr0000` as the study name. + +First, create a bucket named after a study e.g. `idr0000` + + aws --endpoint-url https://uk1s3.embassy.ebi.ac.uk s3 mb s3://idr0000 + +To list all available buckets: + + aws --endpoint-url https://uk1s3.embassy.ebi.ac.uk s3 ls + + +### Policy + +There are two ways to make keys in a bucket publicly readable, either via object ACL or via bucket policy. S3 ACLs is a [legacy access control mechanism ](https://aws.amazon.com/blogs/security/iam-policies-and-bucket-policies-and-acls-oh-my-controlling-access-to-s3-resources/) and can to be set +either when copying objects e.g. via `aws s3 cp` by passing the appropriate +ACL e.g. `--acl public-read`. + +To let the world list the keys of a bucket and read all keys, create a +`policy.json` that grants the actions to anyone: + + { + "Version":"2012-10-17", + "Statement":[ + { + "Sid":"PublicRead", + "Effect":"Allow", + "Principal": "*", + "Action":[ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:ListBucket"], + "Resource":[ + "arn:aws:s3:::idr0066/*", + "arn:aws:s3:::idr0066"] + } + ] + } + +Set the bucket policy using `s3api put-bucket-policy`: + + aws --endpoint-url https://uk1s3.embassy.ebi.ac.uk s3api put-bucket-policy --bucket idr0000 --policy file://policy.json + +Optionally, inspect the bucket policy using `s3api get-bucket-policy`:: + + aws --endpoint-url https://uk1s3.embassy.ebi.ac.uk s3api get-bucket-policy --bucket idr0000 --output text + + +### CORS + +For browser access like vizarr, the CORS headers needs to be configured to +allow GET and HEAD data access. Create a `cors.json` file: + + { + "CORSRules": [ + { + "AllowedOrigins": ["*"], + "AllowedHeaders": ["Authorization"], + "AllowedMethods": ["GET", "HEAD"], + "MaxAgeSeconds": 3000 + } + ] + } + +Set the bucket CORS configuration using `s3api put-bucket-cors`: + + aws --endpoint-url https://uk1s3.embassy.ebi.ac.uk s3api put-bucket-cors --bucket idr0066 --cors-configuration file://cors.json + +Optionally, inspect the bucketCORS configuration using `s3api put-bucket-cors`: + + aws --endpoint-url https://uk1s3.embassy.ebi.ac.uk s3api get-bucket-cors --bucket idr0066 + +## Copy and list objects + +To copy objects to the a bucket use `s3 cp` with the `--recursive` flag: + + aws --endpoint-url https://uk1s3.embassy.ebi.ac.uk s3 cp --recursive /data/1.zarr s3://idr0000/1.zarr/ + +Objects can be inspected using `s3 ls`: + + aws --endpoint-url https://uk1s3.embassy.ebi.ac.uk s3 ls s3://idr0000/ From 14ced5e1627f208f6d32d5a5414a401eddc973f2 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Thu, 19 Aug 2021 14:12:59 +0100 Subject: [PATCH 2/3] Unify bucket name in instructions --- docs/object-store.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/object-store.md b/docs/object-store.md index af2f1155..b8670f4a 100644 --- a/docs/object-store.md +++ b/docs/object-store.md @@ -45,8 +45,8 @@ To let the world list the keys of a bucket and read all keys, create a "s3:GetObjectVersion", "s3:ListBucket"], "Resource":[ - "arn:aws:s3:::idr0066/*", - "arn:aws:s3:::idr0066"] + "arn:aws:s3:::idr0000/*", + "arn:aws:s3:::idr0000"] } ] } @@ -78,11 +78,11 @@ allow GET and HEAD data access. Create a `cors.json` file: Set the bucket CORS configuration using `s3api put-bucket-cors`: - aws --endpoint-url https://uk1s3.embassy.ebi.ac.uk s3api put-bucket-cors --bucket idr0066 --cors-configuration file://cors.json + aws --endpoint-url https://uk1s3.embassy.ebi.ac.uk s3api put-bucket-cors --bucket idr0000 --cors-configuration file://cors.json Optionally, inspect the bucketCORS configuration using `s3api put-bucket-cors`: - aws --endpoint-url https://uk1s3.embassy.ebi.ac.uk s3api get-bucket-cors --bucket idr0066 + aws --endpoint-url https://uk1s3.embassy.ebi.ac.uk s3api get-bucket-cors --bucket idr0000 ## Copy and list objects From d6425b301e831cd52ea416f0e000aa3f9dd6b404 Mon Sep 17 00:00:00 2001 From: Sebastien Besson Date: Thu, 19 Aug 2021 14:13:03 +0100 Subject: [PATCH 3/3] Add minimal script to create a public bucket --- scripts/s3-create-bucket.sh | 53 +++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100755 scripts/s3-create-bucket.sh diff --git a/scripts/s3-create-bucket.sh b/scripts/s3-create-bucket.sh new file mode 100755 index 00000000..e5a4efa3 --- /dev/null +++ b/scripts/s3-create-bucket.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Create public bucket + +set -u + +if [ $# -ne 1 ]; then + echo "USAGE: $(basename "$0") bucket_name [endpoint_url]" + exit 1 +fi + +BUCKET_NAME=$1 +ENDPOINT_URL=${2:-https://uk1s3.embassy.ebi.ac.uk} +POLICY=$(mktemp) +cat > $POLICY << EOL +{ + "Version":"2012-10-17", + "Statement":[ + { + "Sid":"PublicRead", + "Effect":"Allow", + "Principal": "*", + "Action":[ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:ListBucket"], + "Resource":[ + "arn:aws:s3:::$BUCKET_NAME/*", + "arn:aws:s3:::$BUCKET_NAME"] + } + ] +} +EOL +cat $POLICY + +CORS=$(mktemp) +cat > $CORS << EOL +{ + "CORSRules": [ + { + "AllowedOrigins": ["*"], + "AllowedHeaders": ["Authorization"], + "AllowedMethods": ["GET", "HEAD"], + "MaxAgeSeconds": 3000 + } + ] +} +EOL +cat $CORS + +aws --endpoint-url $ENDPOINT_URL s3 mb s3://$BUCKET_NAME + +aws --endpoint-url $ENDPOINT_URL s3api put-bucket-policy --bucket $BUCKET_NAME --policy file://$POLICY +aws --endpoint-url $ENDPOINT_URL s3api put-bucket-cors --bucket $BUCKET_NAME --cors-configuration file://$CORS