From 10f74a5309f50fc921bc864ad5c9e71c36bac77e Mon Sep 17 00:00:00 2001 From: Ross Delinger Date: Fri, 8 Mar 2019 19:00:10 -0800 Subject: [PATCH 1/8] Add support for serving from a local path and with no ldap --- gallery/__init__.py | 154 ++++++++++++++------------- gallery/file_store.py | 118 ++++++++++++++++++++ gallery/ldap.py | 85 +++++++++------ gallery/models.py | 3 +- gallery/s3.py | 26 ----- gallery/templates/view_dir.html | 4 +- gallery/templates/view_file.html | 2 +- gallery/templates/view_filtered.html | 4 +- gallery/util.py | 15 ++- mypy.ini | 3 + requirements.txt | 2 +- 11 files changed, 264 insertions(+), 152 deletions(-) create mode 100644 gallery/file_store.py delete mode 100644 gallery/s3.py diff --git a/gallery/__init__.py b/gallery/__init__.py index 02a1311..b07bbe6 100644 --- a/gallery/__init__.py +++ b/gallery/__init__.py @@ -33,7 +33,8 @@ ClientMetadata, ) from gallery._version import __version__, BUILD_REFERENCE, COMMIT_HASH -from gallery.s3 import S3 +from gallery.file_store import (S3Storage, LocalStorage, FileStorage) +from gallery.ldap import LDAPWrapper import flask_migrate import requests from werkzeug import secure_filename @@ -54,6 +55,7 @@ app_config.from_pyfile(os.path.join(os.getcwd(), "config.py")) else: app_config.from_pyfile(os.path.join(os.getcwd(), "config.env.py")) +app.config.update(app_config) db: SQLAlchemy = SQLAlchemy(app) migrate = flask_migrate.Migrate(app, db) @@ -63,21 +65,35 @@ auth = OIDCAuthentication({ 'default': ProviderConfiguration( - issuer=app.config['OIDC_ISSUER'], + issuer=app_config['OIDC_ISSUER'], client_metadata=ClientMetadata( - client_id=app.config['OIDC_CLIENT_ID'], - client_secret=app.config['OIDC_CLIENT_SECRET'] + client_id=app_config['OIDC_CLIENT_ID'], + client_secret=app_config['OIDC_CLIENT_SECRET'] ) ) }, app) -ldap = CSHLDAP(app.config['LDAP_BIND_DN'], - app.config['LDAP_BIND_PW']) +if "LDAP_BIND_DN" in app.config: + ldap = LDAPWrapper(CSHLDAP( + app.config['LDAP_BIND_DN'], + app.config['LDAP_BIND_PW'], + )) +else: + ldap = LDAPWrapper( + None, + app.config.get("EBOARD_UIDS", "").split(","), + app.config.get("RTP_UIDS", "").split(","), + ) -s3 = S3('s3.csh.rit.edu', - access_key=app.config['S3_ACCESS_ID'], - secret_key=app.config['S3_SECRET_KEY'], - secure=True) +app.add_template_global(ldap, name="ldap") + +storage_interface: FileStorage +if "LOCAL_STORAGE_PATH" in app.config: + storage_interface = LocalStorage(app) +elif "S3_URI" in app.config: + storage_interface = S3Storage(app) +else: + raise Exception("Please configure a storage provider") # pylint: disable=C0413 from gallery.models import Directory @@ -97,43 +113,6 @@ from gallery.file_modules import supported_mimetypes from gallery.file_modules import FileModule -import gallery.ldap as gallery_ldap -from gallery.ldap import ldap_convert_uuid_to_displayname -from gallery.ldap import ldap_get_members - -for func in inspect.getmembers(gallery_ldap): - if func[0].startswith("ldap_"): - unwrapped = inspect.unwrap(func[1]) - if inspect.isfunction(unwrapped): - app.add_template_global(inspect.unwrap(unwrapped), name=func[0]) - -# Ensure that we have a root directory -# XXX there's definitely a better way to do this, I don't have access to the -# docs right now since I'm on a plane, but I'd wager the SQLAlchemy has a way to -# get the number of rows in a table without retrieving them (especially since -# Postgres definitely support this) -if len([d for d in Directory.query.all()]) == 0: - root_dir = Directory(None, "Gallery!", - "A Multimedia Gallery Written in Python with Flask!", - "root", DEFAULT_THUMBNAIL_NAME, "{\"g\":[]}") - db.session.add(root_dir) - db.session.flush() - db.session.commit() - - # Upload the default thumbnail photo to S3 if it's not already up there - # XXX it's probably a good idea to move this outside of the root directory - # creation check. That way if a deployment is given incorrect S3 credentials - # when the root directory is created we can still recover from the case - # where there is not default thumbnail - default_thumbnail_path = "thumbnails/" + DEFAULT_THUMBNAIL_NAME + ".jpg" - file_stat = os.stat(default_thumbnail_path) - - with open(default_thumbnail_path, "rb") as f_hnd: - s3.put_object(app.config['S3_BUCKET_ID'], - "files/" + DEFAULT_THUMBNAIL_NAME, - f_hnd, - file_stat.st_size) - @app.route("/") @auth.oidc_auth('default') @@ -203,20 +182,20 @@ def upload_file(auth_dict: Optional[Dict[str, Any]] = None): # Upload File file_stat = os.stat(filepath) with open(filepath, "rb") as f_hnd: - s3.put_object(app.config['S3_BUCKET_ID'], - "files/" + file_model.s3_id, - f_hnd, - file_stat.st_size) + storage_interface.put( + "files/{}".format(file_model.s3_id), + f_hnd + ) os.remove(filepath) # Upload Thumbnail filepath = os.path.join(dir_path, file_model.thumbnail_uuid) file_stat = os.stat(filepath) with open(filepath, "rb") as f_hnd: - s3.put_object(app.config['S3_BUCKET_ID'], - "thumbnails/" + file_model.s3_id, - f_hnd, - file_stat.st_size) + storage_interface.put( + "thumbnails/" + file_model.s3_id, + f_hnd, + ) os.remove(filepath) os.rmdir(dir_path) if file_model is None: @@ -335,6 +314,36 @@ def refresh_thumbnails(): refresh_thumbnail() +@app.cli.command() +def init_root(): + click.echo("Initializing root directory") + # Ensure that we have a root directory + # XXX there's definitely a better way to do this, I don't have access to the + # docs right now since I'm on a plane, but I'd wager the SQLAlchemy has a way to + # get the number of rows in a table without retrieving them (especially since + # Postgres definitely support this) + if len([d for d in Directory.query.all()]) == 0: + root_dir = Directory(None, "Gallery!", + "A Multimedia Gallery Written in Python with Flask!", + "root", DEFAULT_THUMBNAIL_NAME, "{\"g\":[]}") + db.session.add(root_dir) + db.session.flush() + db.session.commit() + + # Upload the default thumbnail photo to S3 if it's not already up there + # XXX it's probably a good idea to move this outside of the root directory + # creation check. That way if a deployment is given incorrect S3 credentials + # when the root directory is created we can still recover from the case + # where there is not default thumbnail + default_thumbnail_path = "thumbnails/" + DEFAULT_THUMBNAIL_NAME + ".jpg" + + with open(default_thumbnail_path, "rb") as f_hnd: + storage_interface.put( + "files/{}".format(DEFAULT_THUMBNAIL_NAME), + f_hnd, + ) + + def add_directory(parent_id: str, name: str, description: str, owner: str): dir_siblings = Directory.query.filter(Directory.parent == parent_id).all() for sibling in dir_siblings: @@ -433,10 +442,13 @@ def delete_file(file_id: int, auth_dict: Optional[Dict[str, Any]] = None): ).filter( File.s3_id == file_model.s3_id ).first() is None: - s3.remove_object(app.config['S3_BUCKET_ID'], - "files/" + file_model.s3_id) - s3.remove_object(app.config['S3_BUCKET_ID'], - "thumbnails/" + file_model.s3_id) + + storage_interface.remove( + "files/" + file_model.s3_id, + ) + storage_interface.remove( + "thumbnails/" + file_model.s3_id, + ) current_tags = Tag.query.filter(Tag.file_id == file_id).all() for tag in current_tags: @@ -646,16 +658,13 @@ def tag_file(file_id: int): @app.route("/api/file/get/") @auth.oidc_auth('default') def display_file(file_id: int): - file_id = int(file_id) file_model = File.query.filter(File.id == file_id).first() if file_model is None: return "file not found", 404 - presigned_url = s3.presigned_get_object(app.config['S3_BUCKET_ID'], - "files/" + file_model.s3_id, - expires=timedelta(minutes=5)) - return redirect(presigned_url) + link = storage_interface.get_link("files/{}".format(file_model.s3_id)) + return redirect(link) @app.route("/api/thumbnail/get/") @@ -663,10 +672,8 @@ def display_file(file_id: int): def display_thumbnail(file_id: int): file_model = File.query.filter(File.id == file_id).first() - presigned_url = s3.presigned_get_object(app.config['S3_BUCKET_ID'], - "thumbnails/" + file_model.s3_id, - expires=timedelta(minutes=5)) - return redirect(presigned_url) + link = storage_interface.get_link("thumbnails/{}".format(file_model.s3_id)) + return redirect(link) @app.route("/api/thumbnail/get/dir/") @@ -679,11 +686,8 @@ def display_dir_thumbnail(dir_id: int): if len(thumbnail_uuid.split('.')) > 1: thumbnail_uuid = thumbnail_uuid.split('.')[0] - presigned_url = s3.presigned_get_object(app.config['S3_BUCKET_ID'], - "thumbnails/" + - thumbnail_uuid, - expires=timedelta(minutes=5)) - return redirect(presigned_url) + link = storage_interface.get_link("thumbnails/{}".format(thumbnail_uuid)) + return redirect(link) @app.route("/api/file/next/") @@ -746,7 +750,7 @@ def get_dir_children(dir_id: int) -> Any: children.sort(key=lambda x: x['name']) return children - root = Directory.query.filter(Directory.parent is None).first() + root = Directory.query.filter(Directory.parent == None).first() tree = {} @@ -904,7 +908,7 @@ def view_filtered(auth_dict: Optional[Dict[str, Any]] = None): @app.route("/api/memberlist") @auth.oidc_auth('default') def get_member_list(): - return jsonify(ldap_get_members()) + return jsonify(ldap.get_members()) @app.errorhandler(404) diff --git a/gallery/file_store.py b/gallery/file_store.py new file mode 100644 index 0000000..5d85f86 --- /dev/null +++ b/gallery/file_store.py @@ -0,0 +1,118 @@ +import os +import shutil +from datetime import timedelta +from typing import IO + +import boto3 +import flask +from flask import abort, send_from_directory, url_for +from itsdangerous import ( + URLSafeTimedSerializer +) + + +class FileStorage(object): + """ + FileStorage represents the interface for interacting with some kind of + backing storage system. This system is used to store the actual image and + thumbnail data persistantly. + """ + + def put(self, key: str, handle: IO[bytes]): + """ + put is used to stream data from a local file descriptor into the backing + storage implementation. + """ + pass + + def remove(self, key: str): + """ + delete will remove a given file from the backing storage implementation + """ + pass + + def get_link(self, key: str) -> str: + """ + get_link is used to return a publically facing link to a file in the + backing storage system. + """ + pass + + +class LocalStorage(FileStorage): + """ + LocalStorage uses a local filesystem path as the basis for storing and + serving photos and thumbnails. We generate temp links using the + itsdangerous library. + """ + def __init__(self, app: flask.Flask): + self._serializer = URLSafeTimedSerializer( + secret_key=app.config["SECRET_KEY"], + ) + self._base_dir = app.config["LOCAL_STORAGE_PATH"] + self._link_expiration = timedelta(days=7) + app.route("/public/")(self._temp_link_handler) + + def _temp_link_handler(self, token: str): + try: + payload = self._serializer.loads(token, max_age=int(self._link_expiration.total_seconds())) + except Exception: + # NOTE(rossdylan): We are being broad here because in any case that + # this fails we just want to abort. This is because the token has + # expired, been tampered with, or is just invalid. Maybe we can + # log each case differently later... + abort(404) + + if "key" not in payload: + abort(404) + + # NOTE(rossdylan): We are relying on flask's protections to avoid + # traversals or any other weirdness here. /should/ be fine since we + # sign and verify the file path anyway + return send_from_directory(self._base_dir, payload["key"]) + + def put(self, key: str, handle: IO[bytes]): + local_path = os.path.join(self._base_dir, key) + os.makedirs(os.path.dirname(local_path), exist_ok=True) + with open(local_path, 'wb+') as f: + shutil.copyfileobj(handle, f) + + def remove(self, key: str): + local_path = os.path.join(self._base_dir, key) + os.remove(local_path) + + def get_link(self, key: str): + return url_for("_temp_link_handler", token=self._serializer.dumps({"key": key})) + + +class S3Storage(FileStorage): + """ + S3Storage is the main storage implementation that uses an s3-like system + to store thumbnails and photos. Links are generated using the presigned + url function of S3. + """ + def __init__(self, app: flask.Flask): + self._client = boto3.client( + 's3', + aws_access_key_id=app.config['S3_ACCESS_ID'], + aws_secret_access_key=app.config['S3_ACCESS_ID'], + endpoint_url=app.config['S3_URI'], + ) + self._bucket = app.config['S3_BUCKET'] + self._link_expiration = timedelta(minutes=5) + + def put(self, key: str, handle: IO[bytes]): + self._client.put_object(Bucket=self._bucket, Key=key, Body=handle) + + def remove(self, key: str): + self._client.remove_object(Bucket=self._bucket, Key=key) + + def get_link(self, key: str) -> str: + return self._client.generate_presigned_url( + "get_object", + Params={ + "Bucket": self._bucket, + "Key": key, + }, + ExpiresIn=self._link_expiration.total_seconds(), + ) diff --git a/gallery/ldap.py b/gallery/ldap.py index 4524bc0..df8e229 100644 --- a/gallery/ldap.py +++ b/gallery/ldap.py @@ -1,36 +1,53 @@ from typing import Dict, List -from gallery import ldap import ldap as pyldap # type: ignore - - -def ldap_convert_uuid_to_displayname(uuid: str) -> str: - if uuid == "root": - return uuid - return ldap.get_member(uuid).displayName - - -def ldap_is_eboard(uid: str) -> bool: - eboard_group = ldap.get_group('eboard') - return eboard_group.check_member(ldap.get_member(uid, uid=True)) - - -def ldap_is_rtp(uid: str) -> bool: - rtp_group = ldap.get_group('rtp') - return rtp_group.check_member(ldap.get_member(uid, uid=True)) - - -def ldap_get_members() -> List[Dict[str, str]]: - con = ldap.get_con() - - res = con.search_s( - "dc=csh,dc=rit,dc=edu", - pyldap.SCOPE_SUBTREE, - "(memberof=cn=member,cn=groups,cn=accounts,dc=csh,dc=rit,dc=edu)", - ["ipaUniqueID", "displayName"]) - - members = filter(lambda m: 'displayName' in m[1], res) - - return [{ - "name": m[1]['displayName'][0].decode('utf-8'), - "uuid": m[1]['ipaUniqueID'][0].decode('utf-8') - } for m in members] +from typing import Optional, List, Dict +from csh_ldap import CSHLDAP + + +class LDAPWrapper(object): + def __init__(self, ldap: Optional[CSHLDAP], eboard: Optional[List[str]] = None, rtp: Optional[List[str]] = None): + self._ldap = ldap + self._eboard: List[str] = [] + self._rtp: List[str] = [] + + if eboard: + self._eboard = eboard + if rtp: + self._rtp = rtp + + def convert_uuid_to_displayname(self, uuid: str) -> str: + if uuid == "root": + return uuid + if self._ldap is None: + return "unknown" + return self._ldap.get_member(uuid).displayName + + def is_eboard(self, uid: str) -> bool: + if self._ldap is None: + return uid in self._eboard + eboard_group = self._ldap.get_group('eboard') + return eboard_group.check_member(self._ldap.get_member(uid, uid=True)) + + def is_rtp(self, uid: str) -> bool: + if self._ldap is None: + return uid in self._rtp + rtp_group = self._ldap.get_group('rtp') + return rtp_group.check_member(self._ldap.get_member(uid, uid=True)) + + def get_members(self) -> List[Dict[str, str]]: + if self._ldap is None: + return [] + con = self._ldap.get_con() + + res = con.search_s( + "dc=csh,dc=rit,dc=edu", + pyldap.SCOPE_SUBTREE, + "(memberof=cn=member,cn=groups,cn=accounts,dc=csh,dc=rit,dc=edu)", + ["ipaUniqueID", "displayName"]) + + members = filter(lambda m: 'displayName' in m[1], res) + + return [{ + "name": m[1]['displayName'][0].decode('utf-8'), + "uuid": m[1]['ipaUniqueID'][0].decode('utf-8') + } for m in members] diff --git a/gallery/models.py b/gallery/models.py index 89c5eac..612f9ca 100644 --- a/gallery/models.py +++ b/gallery/models.py @@ -12,8 +12,7 @@ from typing import TYPE_CHECKING -if TYPE_CHECKING: - import flask_sqlalchemy # type: ignore +import flask_sqlalchemy # type: ignore strfformat = "%Y-%m-%d" diff --git a/gallery/s3.py b/gallery/s3.py deleted file mode 100644 index c77670a..0000000 --- a/gallery/s3.py +++ /dev/null @@ -1,26 +0,0 @@ -import boto -import boto.s3.connection -from boto.s3.key import Key -from datetime import timedelta - -class S3(): - con = None - - def __init__(self, host, access_key=None, secret_key=None, secure=True): - self.con = boto.connect_s3(aws_access_key_id=access_key, - aws_secret_access_key=secret_key, - host=host, - calling_format=boto.s3.connection.OrdinaryCallingFormat()) - def put_object(self, bucket_name, key, file_handle, file_size): - bucket = self.con.get_bucket(bucket_name) - _key = bucket.new_key(key) - _key.set_contents_from_file(file_handle) - - def remove_object(self, bucket_name, key): - bucket = self.con.get_bucket(bucket_name) - bucket.delete_key(key) - - def presigned_get_object(self, bucket_name, key, expires=timedelta(days=7)): - bucket = self.con.get_bucket(bucket_name) - _key = bucket.get_key(key) - return _key.generate_url(expires.total_seconds(), query_auth=True) diff --git a/gallery/templates/view_dir.html b/gallery/templates/view_dir.html index 38c35da..9adb790 100644 --- a/gallery/templates/view_dir.html +++ b/gallery/templates/view_dir.html @@ -40,14 +40,14 @@

{{ child.get_name() }}

 -

Owner: {{ ldap_convert_uuid_to_displayname(child.author) }}

+

Owner: {{ ldap.convert_uuid_to_displayname(child.author) }}

Date Created: {{ child.date() }}

{% elif child_type == "File" %}

{{ child.get_name() }}

 -

Owner: {{ ldap_convert_uuid_to_displayname(child.author) }}

+

Owner: {{ ldap.convert_uuid_to_displayname(child.author) }}

Date Uploaded: {{ child.date() }}

{% endif %} diff --git a/gallery/templates/view_file.html b/gallery/templates/view_file.html index 6aa968a..7c5dc85 100644 --- a/gallery/templates/view_file.html +++ b/gallery/templates/view_file.html @@ -61,7 +61,7 @@ Tagged:
    {% for tag in tags %} -
  • {{ ldap_convert_uuid_to_displayname(tag) }}{% if not loop.last %}, {% endif %}
    • +
  • {{ ldap.convert_uuid_to_displayname(tag) }}{% if not loop.last %}, {% endif %}
    • {% endfor %}
diff --git a/gallery/templates/view_filtered.html b/gallery/templates/view_filtered.html index 6d84c70..9ae2d3a 100644 --- a/gallery/templates/view_filtered.html +++ b/gallery/templates/view_filtered.html @@ -13,7 +13,7 @@

Files of {% for uuid in uuids %} - {{ ldap_convert_uuid_to_displayname(uuid) }}{% if not loop.last %}, {% endif %} + {{ ldap.convert_uuid_to_displayname(uuid) }}{% if not loop.last %}, {% endif %} {% endfor %}

@@ -27,7 +27,7 @@

Files of

{{ file.get_name() }}

-

Owner: {{ ldap_convert_uuid_to_displayname(file.author) }}

+

Owner: {{ ldap.convert_uuid_to_displayname(file.author) }}

Date Uploaded: {{ file.date() }}

diff --git a/gallery/util.py b/gallery/util.py index 96e353b..45dabf5 100644 --- a/gallery/util.py +++ b/gallery/util.py @@ -5,11 +5,8 @@ import hashlib import os -from gallery.ldap import ldap_is_eboard -from gallery.ldap import ldap_is_rtp -from gallery.ldap import ldap_convert_uuid_to_displayname - from gallery.models import Directory, File, Tag +from gallery import ldap DEFAULT_THUMBNAIL_NAME = 'reedphoto' @@ -68,11 +65,11 @@ def get_files_tagged(uuids: List[str]) -> List[File]: def gallery_auth(func: Callable[..., Any]) -> Callable[..., Any]: @wraps(func) def wrapped_function(*args: Any, **kwargs: Any) -> Any: - uuid = str(session['userinfo'].get('sub', '')) - uid = str(session['userinfo'].get('preferred_username', '')) - name = ldap_convert_uuid_to_displayname(uuid) - is_eboard = ldap_is_eboard(uid) - is_rtp = ldap_is_rtp(uid) + uuid = str(session.get('userinfo', {}).get('sub', '')) + uid = str(session.get('userinfo', {}).get('preferred_username', '')) + name = ldap.convert_uuid_to_displayname(uuid) + is_eboard = ldap.is_eboard(uid) + is_rtp = ldap.is_rtp(uid) # NOTE(rossdylan): This is probably a more precise type than we need, # if different data is needed just expand the value type to Any diff --git a/mypy.ini b/mypy.ini index 52dd080..31b3aee 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,9 @@ [mypy] plugins = sqlmypy +[mypy-boto3] +ignore_missing_imports = True + [mypy-magic] ignore_missing_imports = True diff --git a/requirements.txt b/requirements.txt index 2ce3662..41b9495 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ piexif==1.1.2 wand==0.5.0 gunicorn==19.9.0 moviepy==0.2.3.5 -boto==2.49.0 +boto3 From 38f7374be7a28f7e0feaba40ea00ab302b36d3ab Mon Sep 17 00:00:00 2001 From: Ross Delinger Date: Fri, 8 Mar 2019 19:16:05 -0800 Subject: [PATCH 2/8] add s3 uri config --- config.env.py | 1 + 1 file changed, 1 insertion(+) diff --git a/config.env.py b/config.env.py index df30bc2..b89b145 100644 --- a/config.env.py +++ b/config.env.py @@ -23,6 +23,7 @@ 'sqlite:///{}'.format(os.path.join(os.getcwd(), 'data.db'))) SQLALCHEMY_TRACK_MODIFICATIONS = False +S3_URI = os.environ.get('GALLERY_S3_URI', '') S3_ACCESS_ID = os.environ.get('GALLERY_S3_ACCESS_ID','') S3_SECRET_KEY = os.environ.get('GALLERY_S3_SECRET_KEY','') S3_BUCKET_ID = os.environ.get('GALLERY_S3_BUCKET_ID','') From c48f3392d8117b29bfe28b3dff23882934401589 Mon Sep 17 00:00:00 2001 From: Ross Delinger Date: Fri, 8 Mar 2019 19:21:44 -0800 Subject: [PATCH 3/8] more s3 fixes --- gallery/file_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallery/file_store.py b/gallery/file_store.py index 5d85f86..f2bfe18 100644 --- a/gallery/file_store.py +++ b/gallery/file_store.py @@ -98,7 +98,7 @@ def __init__(self, app: flask.Flask): aws_secret_access_key=app.config['S3_ACCESS_ID'], endpoint_url=app.config['S3_URI'], ) - self._bucket = app.config['S3_BUCKET'] + self._bucket = app.config['S3_BUCKET_ID'] self._link_expiration = timedelta(minutes=5) def put(self, key: str, handle: IO[bytes]): From 019aa710f70335c50c4ac7dedcb41c31571cbe2f Mon Sep 17 00:00:00 2001 From: Ross Delinger Date: Fri, 8 Mar 2019 19:31:32 -0800 Subject: [PATCH 4/8] force imageio to version 2.4.0 to avoid deprecation issues --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 41b9495..5483e37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,5 @@ piexif==1.1.2 wand==0.5.0 gunicorn==19.9.0 moviepy==0.2.3.5 +imagio==2.4.0 boto3 From 332f72b92ae36bfdbeeafe865d4f7c2c24a421ff Mon Sep 17 00:00:00 2001 From: Ross Delinger Date: Fri, 8 Mar 2019 19:32:34 -0800 Subject: [PATCH 5/8] fix typo --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5483e37..de06ac5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,5 +10,5 @@ piexif==1.1.2 wand==0.5.0 gunicorn==19.9.0 moviepy==0.2.3.5 -imagio==2.4.0 +imageio==2.4.0 boto3 From ff1a145b4afcfdcadbdcfaeabf54c8357b2fa7a4 Mon Sep 17 00:00:00 2001 From: Ross Delinger Date: Fri, 8 Mar 2019 19:59:46 -0800 Subject: [PATCH 6/8] set the correct signature version --- gallery/file_store.py | 2 ++ mypy.ini | 3 +++ 2 files changed, 5 insertions(+) diff --git a/gallery/file_store.py b/gallery/file_store.py index f2bfe18..cf93133 100644 --- a/gallery/file_store.py +++ b/gallery/file_store.py @@ -4,6 +4,7 @@ from typing import IO import boto3 +from botocore.client import Config import flask from flask import abort, send_from_directory, url_for from itsdangerous import ( @@ -97,6 +98,7 @@ def __init__(self, app: flask.Flask): aws_access_key_id=app.config['S3_ACCESS_ID'], aws_secret_access_key=app.config['S3_ACCESS_ID'], endpoint_url=app.config['S3_URI'], + config=Config(signature_version='s3v4'), ) self._bucket = app.config['S3_BUCKET_ID'] self._link_expiration = timedelta(minutes=5) diff --git a/mypy.ini b/mypy.ini index 31b3aee..8a791a7 100644 --- a/mypy.ini +++ b/mypy.ini @@ -4,6 +4,9 @@ plugins = sqlmypy [mypy-boto3] ignore_missing_imports = True +[mypy-botocore.*] +ignore_missing_imports = True + [mypy-magic] ignore_missing_imports = True From 03170bb9a6074f8cae6658327cd3b023e8419924 Mon Sep 17 00:00:00 2001 From: Ross Delinger Date: Fri, 8 Mar 2019 20:50:34 -0800 Subject: [PATCH 7/8] use proper secret id in s3 file store --- gallery/file_store.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gallery/file_store.py b/gallery/file_store.py index cf93133..4bacb44 100644 --- a/gallery/file_store.py +++ b/gallery/file_store.py @@ -96,7 +96,7 @@ def __init__(self, app: flask.Flask): self._client = boto3.client( 's3', aws_access_key_id=app.config['S3_ACCESS_ID'], - aws_secret_access_key=app.config['S3_ACCESS_ID'], + aws_secret_access_key=app.config['S3_SECRET_KEY'], endpoint_url=app.config['S3_URI'], config=Config(signature_version='s3v4'), ) @@ -104,10 +104,10 @@ def __init__(self, app: flask.Flask): self._link_expiration = timedelta(minutes=5) def put(self, key: str, handle: IO[bytes]): - self._client.put_object(Bucket=self._bucket, Key=key, Body=handle) + self._client.Bucket(self._bucket).upload_fileobj(handle, key) def remove(self, key: str): - self._client.remove_object(Bucket=self._bucket, Key=key) + self._client.Object(self._bucket, key).delete() def get_link(self, key: str) -> str: return self._client.generate_presigned_url( From a218f2fed561334b1f06874dff795005ea15816c Mon Sep 17 00:00:00 2001 From: Ross Delinger Date: Fri, 8 Mar 2019 20:59:15 -0800 Subject: [PATCH 8/8] hopefully final fix of the s3 api commands --- gallery/file_store.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gallery/file_store.py b/gallery/file_store.py index 4bacb44..f050337 100644 --- a/gallery/file_store.py +++ b/gallery/file_store.py @@ -104,10 +104,10 @@ def __init__(self, app: flask.Flask): self._link_expiration = timedelta(minutes=5) def put(self, key: str, handle: IO[bytes]): - self._client.Bucket(self._bucket).upload_fileobj(handle, key) + self._client.upload_fileobj(handle, self._bucket, key) def remove(self, key: str): - self._client.Object(self._bucket, key).delete() + self._client.delete_object(Bucket=self._bucket, Key=key) def get_link(self, key: str) -> str: return self._client.generate_presigned_url(