diff --git a/.circleci/config.yml b/.circleci/config.yml index 0d6f21d..f913d96 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,6 +11,9 @@ jobs: command: | pip install pipenv --upgrade pipenv install --dev + - run: + name: build data + command: pipenv run build - run: name: run tests command: pipenv run test diff --git a/.gitignore b/.gitignore index a32ebae..48b43fe 100755 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,19 @@ -# Elsa output -_build - -### OSX +# OS +._* *.DS_Store -# Icon must end with two \r -Icon +# IDEs +.idea +.vscode -# Thumbnails -._* +# Cache +.cache +.pytest_cache -# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Generated +/pyvecorg/_build +/pyvecorg/data/members_list.yml +/pyvecorg/static/img/avatars -# User-specific stuff: -.idea +# Private +google_service_account.json diff --git a/Pipfile b/Pipfile index 445ec4a..c44b20f 100644 --- a/Pipfile +++ b/Pipfile @@ -4,6 +4,7 @@ verify_ssl = true name = "pypi" [scripts] +build = "python pyvecorg/build.py" test = "pytest" freeze = "python pyvecorg freeze" serve = "python pyvecorg serve" @@ -19,6 +20,8 @@ pyyaml = "~=5.1" requests = "~=2.22.0" python-slugify = "~=3.0.2" pillow = "~=6.1.0" +gspread = "~=3.1.0" +oauth2client = "~=4.1.3" [dev-packages] # Pinning packages with ~= unless their version starts with 0., diff --git a/Pipfile.lock b/Pipfile.lock index 6b11ec5..e541564 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f5d0983d2ac89e2fe55733d1b7ad346fead0d02aeaac89b4f735776d206c25b4" + "sha256": "fca06590b9d6ea98aa80b27c90fab97c9266d57d6cc348bde0c1b231cc503915" }, "pipfile-spec": 6, "requires": { @@ -66,6 +66,21 @@ ], "version": "==0.5.5" }, + "gspread": { + "hashes": [ + "sha256:dd945e3ae5d3d0325ad9982e0d5667f79ca121d0bb6f35274dc84371bbb79dd5", + "sha256:f7ce6c06250f694976c3cd4944e3b607b0810b93383839e5b67c7199ce2f0d3d" + ], + "index": "pypi", + "version": "==3.1.0" + }, + "httplib2": { + "hashes": [ + "sha256:6901c8c0ffcf721f9ce270ad86da37bc2b4d32b8802d4a9cec38274898a64044", + "sha256:cf6f9d5876d796539ec922a2c9b9a7cad9bfd90f04badcdc3bcfa537168052c3" + ], + "version": "==0.13.1" + }, "idna": { "hashes": [ "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", @@ -128,6 +143,14 @@ ], "version": "==1.1.1" }, + "oauth2client": { + "hashes": [ + "sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac", + "sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6" + ], + "index": "pypi", + "version": "==4.1.3" + }, "pillow": { "hashes": [ "sha256:0804f77cb1e9b6dbd37601cee11283bba39a8d44b9ddb053400c58e0c0d7d9de", @@ -160,6 +183,20 @@ "index": "pypi", "version": "==6.1.0" }, + "pyasn1": { + "hashes": [ + "sha256:3bb81821d47b17146049e7574ab4bf1e315eb7aead30efe5d6a9ca422c9710be", + "sha256:b773d5c9196ffbc3a1e13bdf909d446cad80a039aa3340bcad72f395b76ebc86" + ], + "version": "==0.4.6" + }, + "pyasn1-modules": { + "hashes": [ + "sha256:43c17a83c155229839cc5c6b868e8d0c6041dba149789b6d6e28801c64821722", + "sha256:e30199a9d221f1b26c885ff3d87fd08694dbbe18ed0e8e405a2a7126d30ce4c0" + ], + "version": "==0.2.6" + }, "python-slugify": { "hashes": [ "sha256:a9f468227cb11e20e251670d78e1b5f6b0b15dd37bbd5c9814a25a904e44ff66" @@ -194,6 +231,20 @@ "index": "pypi", "version": "==2.22.0" }, + "rsa": { + "hashes": [ + "sha256:14ba45700ff1ec9eeb206a2ce76b32814958a98e372006c8fb76ba820211be66", + "sha256:1a836406405730121ae9823e19c6e806c62bbad73f890574fff50efa4122c487" + ], + "version": "==4.0" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, "text-unidecode": { "hashes": [ "sha256:5a1375bb2ba7968740508ae38d92e1f889a0832913cb1c447d5e2046061a396d", diff --git a/README.md b/README.md index c09af95..fec51e5 100755 --- a/README.md +++ b/README.md @@ -17,26 +17,42 @@ $ pipenv install The site uses [elsa](https://github.com/pyvec/elsa). - Installation: `pipenv install --dev` -- Development server: `pipenv run serve` +- Download data from external sources: `pipenv run build` - Tests: `pipenv run test` +- Development server: `pipenv run serve` ### Data and tests The site is just a single HTML page rendered on top of some static data. -However, the data can get quite complex and most of the texts need to be -translated into two languages. +However, some of the data come from external sources, the data can get quite +complex, and most of the texts need to be translated into two languages. The data is stored in multiple YAML files. When these are read, whenever an object has just `cs` and `en` properties, it is treated as a "translated text" and the property corresponding to a currently selected language becomes the actual value in place of the object. -Also, to keep the complex structure of the YAML files organized and tested, -there are schemas written in [JSON Schema](https://spacetelescope.github.io/understanding-json-schema/) +To keep the complex structure of the YAML files organized and tested, +there are schemas written in [JSON Schema](https://json-schema.org/understanding-json-schema/) ([spec](http://json-schema.org/)). In tests, the YAML files are validated against the schemas. There is also a couple of additional tests to ensure some logical rules which cannot be easily expressed by JSON Schema. +### External sources + +Some data cannot be stored statically in a YAML file. There is a command +`pipenv run build`, which downloads them from external sources and generates +respective static YAML files. This is a separate step, which needs to be done +before developing or deploying the site, otherwise it won't work properly. + +### Members + +Pyvec members are tracked in an internal Google Spreadsheet. The future +intention is to have the list of members public, but we're not there yet (GDPR). +So far only board members are being listed publicly. The `pipenv run build` +command downloads the spreadsheet as CSV and generates the `members_list.yml` +file. It also downloads and caches avatars. + ### Numbers There are stats numbers in [numbers.yml](pyvecorg/data/numbers.yml). They are @@ -50,15 +66,16 @@ as well as it clones and analyzes all relevant repositories. If you need to upda the number, 1. ask him to run the script, -2. bother him to Open Source the script code. +1. bother him to Open Source the script code. -### Twitter avatars +### Google Drive credentials -To figure out correct URLs of Twitter avatars from Twitter usernames, the site -needs to perform an HTTP request for each of them. This is not really an issue, -because on production this is done only once - in the moment of deployment. -However, it can get very annoying during development. Set `DISABLE_TWITTER_AVATARS` -environment variable to truthy value to disable Twitter avatars for development. +1. Follow the steps in the [gspread guide](https://gspread.readthedocs.io/en/latest/oauth2.html). Instead of Google Drive API, enable Google Sheets API. +1. Save the obtained JSON file into the `pyvecorg` package as `google_service_account.json` +1. Make sure it is ignored by Git +1. Run `cat pyvecorg/google_service_account.json | pbcopy` to copy the JSON into your clipboard (macOS) +1. Go to [Travis CI project settings](https://travis-ci.org/pyvec/pyvec.org/settings), section Environment Variables +1. Add `GOOGLE_SERVICE_ACCOUNT` variable and paste the JSON from your clipboard as a value ## Deployment diff --git a/pyvecorg/__init__.py b/pyvecorg/__init__.py index 5417214..f52bb6a 100644 --- a/pyvecorg/__init__.py +++ b/pyvecorg/__init__.py @@ -1,4 +1,3 @@ -import os import itertools from pathlib import Path @@ -27,8 +26,8 @@ def run(self, *args, **kwargs): app.config['JSON_AS_ASCII'] = False -from pyvecorg import views -from pyvecorg import templating +from pyvecorg import views # noqa +from pyvecorg import templating # noqa __all__ = ['app', 'views', 'templating'] diff --git a/pyvecorg/avatars.py b/pyvecorg/avatars.py index b297abb..17fda00 100644 --- a/pyvecorg/avatars.py +++ b/pyvecorg/avatars.py @@ -1,4 +1,3 @@ -import os import io from urllib.parse import quote import hashlib @@ -19,8 +18,6 @@ def get_avatar_url(member): email_hash = hashlib.md5(value.lower().strip().encode()).hexdigest() return f'https://www.gravatar.com/avatar/{email_hash}?size=100&d=404' elif key == 'twitter': - if os.getenv('DISABLE_TWITTER_AVATARS'): - return no_avatar username = quote(value) url = f'https://twitter.com/{username}/profile_image' url = requests.head(url).headers.get('location') diff --git a/pyvecorg/build.py b/pyvecorg/build.py new file mode 100644 index 0000000..e6883be --- /dev/null +++ b/pyvecorg/build.py @@ -0,0 +1,92 @@ +import io +import os +import json +from textwrap import dedent +from pathlib import Path + +import yaml +import requests +from slugify import slugify +import gspread +from oauth2client.service_account import ServiceAccountCredentials + +from pyvecorg.avatars import get_avatar_url, create_thumbnail + + +PACKAGE_DIR = Path(__file__).parent +MEMBERS_LIST_YAML = PACKAGE_DIR / 'data' / 'members_list.yml' +MEMBERS_CSV_URL = 'https://docs.google.com/spreadsheets/d/e/2PACX-1vSWK18MlEy95sAGl1BM6BXWxPgJbIx2UH3tAyJjxES06hHuaXgpsmD5pRz9kkGcFupiZL_U_e7yv4t1/pub?gid=0&single=true&output=csv' # noqa +STATIC_DIR = PACKAGE_DIR / 'static' +AVATARS_DIR = STATIC_DIR / 'img' / 'avatars' + + +def read_spreadsheet(doc_key, sheet_name, google_service_account): + credentials = ServiceAccountCredentials.from_json_keyfile_dict( + google_service_account, + [ + 'https://spreadsheets.google.com/feeds', + 'https://www.googleapis.com/auth/drive', + ] + ) + doc = gspread.authorize(credentials).open_by_key(doc_key) + return doc.worksheet(sheet_name).get_all_values() + + +def parse_members(rows): + head = None + for row in rows: + if frozenset(['name', 'role', 'avatar']) < frozenset(row): + head = row + break + for row in rows: + yield {key: value for key, value in zip(head, row) if value != ''} + + +def generate_yaml(data): + yaml_contents = dedent('''\ + # + # This file has been generated from external sources + # using `pipenv run build`. Do not edit it manually! + # + ''') + return yaml_contents + yaml.dump(data, allow_unicode=True) + + +def create_member_sorting_key(member): + return ( + 0 if member.get('role') == 'chair' else 1, # chair to be first + member.get('nickname', member['name']).split(' ')[-1], # last name + ) + + +if __name__ == '__main__': + # Build data/members_list.yml and avatar images + gsa_path = PACKAGE_DIR / 'google_service_account.json' + gsa_json = os.getenv('GOOGLE_SERVICE_ACCOUNT') or gsa_path.read_text() + gsa = json.loads(gsa_json) + + # Document key appears in the URL if you have the document open + # in your browser + doc_key = '1n8hzBnwZ5ANkUCvwEy8rWsXlqeAAwu-5JBodT5OJx_I' + rows = read_spreadsheet(doc_key, 'list', gsa) + members = sorted([ + member for member in parse_members(rows) + if member.get('role') in ('board', 'chair') + ], key=create_member_sorting_key) + + AVATARS_DIR.mkdir(exist_ok=True) + for member in sorted(members, key=create_member_sorting_key): + avatar_url = get_avatar_url(member) + img_basename = slugify(member['name']) + '.png' + + response = requests.get(avatar_url) + response.raise_for_status() + img_bytes = create_thumbnail(io.BytesIO(response.content), 100).read() + + img_path = (AVATARS_DIR / img_basename) + img_path.write_bytes(img_bytes) + + member['avatar_filename'] = str(img_path.relative_to(STATIC_DIR)) + + data = dict(entries=list(members)) + MEMBERS_LIST_YAML.write_text(generate_yaml(data)) diff --git a/pyvecorg/data/definitions/member_schema.json b/pyvecorg/data/definitions/member_schema.json index 7018d34..1d85411 100644 --- a/pyvecorg/data/definitions/member_schema.json +++ b/pyvecorg/data/definitions/member_schema.json @@ -5,6 +5,9 @@ "name": { "type": "string" }, + "nickname": { + "type": "string" + }, "role": { "type": "string" }, @@ -23,12 +26,20 @@ }, "avatar": { "type": "string" + }, + "avatar_filename": { + "type": "string" + }, + "address": { + "type": "string" + }, + "dob": { + "type": "string" } }, "additionalProperties": false, "required": [ "name", - "role", - "avatar" + "avatar_filename" ] } diff --git a/pyvecorg/data/members.yml b/pyvecorg/data/members.yml index 6c8671b..5342b78 100644 --- a/pyvecorg/data/members.yml +++ b/pyvecorg/data/members.yml @@ -2,75 +2,6 @@ heading: cs: Kdo je Pyvec en: Who is Pyvec -entries: - - name: Martin Bílek - role: chair - github: martinbilek - twitter: ajtea - linkedin: martin-b%C3%ADlek-47699542 - avatar: github - - - name: Jiří Bartoň - role: board - github: whiskybar - twitter: lurkingideas - avatar: github - - - name: Aleš Zoulek - role: board - github: aleszoulek - twitter: aleszoulek - linkedin: zoulek - avatar: github - - - name: Jakub Vysoký - role: board - github: kvbik - twitter: kvbik - linkedin: jakubvysoky - avatar: github - - - name: Vítězslav Pliska - role: board - github: whit - twitter: whiteck - linkedin: vitekpliska - avatar: github - - - name: Robin Gottfried - role: board - github: czervenka - twitter: czervenka - linkedin: robingottfried - avatar: github - - - name: Honza Král - role: board - github: honzakral - twitter: honzakral - linkedin: honzakral - avatar: github - - - name: Martin Burián - role: trustee - twitter: martin_burian - linkedin: martinburi8n - avatar: twitter - - - name: Filip Vařecha - role: trustee - github: xaralis - twitter: xaralis - linkedin: filip-vařecha-51625031 - avatar: github - - - name: Jiří Suchan - role: trustee - github: yedpodtrzitko - twitter: yedpodtrzitko - linkedin: yedpodtrzitko - avatar: twitter - roles: chair: cs: předseda @@ -78,9 +9,6 @@ roles: board: cs: člen rady en: board member - trustee: - cs: člen dozorčí rady - en: board of trustees member note: icon: user-circle-o diff --git a/pyvecorg/data/members_list_schema.json b/pyvecorg/data/members_list_schema.json new file mode 100644 index 0000000..ae4643c --- /dev/null +++ b/pyvecorg/data/members_list_schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "entries": { + "type": "array", + "items": { + "$ref": "definitions/member_schema.json#" + } + } + }, + "additionalProperties": false, + "required": [ + "entries" + ] +} diff --git a/pyvecorg/data/members_schema.json b/pyvecorg/data/members_schema.json index 30957de..7bf7caf 100644 --- a/pyvecorg/data/members_schema.json +++ b/pyvecorg/data/members_schema.json @@ -5,12 +5,6 @@ "heading": { "$ref": "definitions/translated_text_schema.json#" }, - "entries": { - "type": "array", - "items": { - "$ref": "definitions/member_schema.json#" - } - }, "roles": { "type": "object", "patternProperties": { @@ -27,7 +21,6 @@ "additionalProperties": false, "required": [ "heading", - "entries", "roles", "note" ] diff --git a/pyvecorg/templates/index.html b/pyvecorg/templates/index.html index dedbb0c..25333d3 100644 --- a/pyvecorg/templates/index.html +++ b/pyvecorg/templates/index.html @@ -152,46 +152,53 @@
- {% if member.github %} - - GitHub - {% endif %} - {% if member.twitter %} - - Twitter - {% endif %} - {% if member.linkedin %} - - LinkedIn - {% endif %} -
-
- {{ member.name }}
- {{ members.roles[member.role] }}
-
+ {% if member.github %} + + GitHub + {% endif %} + {% if member.twitter %} + + Twitter + {% endif %} + {% if member.linkedin %} + + LinkedIn + {% endif %} +
+
+ {{ member.nickname|default(member.name) }}
+ {{ members.roles[member.role] }}
+
+ Seznam členů se nepodařilo načíst. + +
+ {% endif %}