From b3e2814a6196eff91fa9bfa2d2ffda96d352fe0a Mon Sep 17 00:00:00 2001 From: Honza Javorek Date: Tue, 25 Jun 2019 10:09:52 +0200 Subject: [PATCH 01/10] download the list of members in a build step --- .gitignore | 23 +++---- Pipfile | 1 + README.md | 26 +++++-- pyvecorg/build.py | 41 +++++++++++ pyvecorg/data/definitions/member_schema.json | 4 +- pyvecorg/data/members.yml | 72 -------------------- pyvecorg/data/members_list_schema.json | 16 +++++ pyvecorg/data/members_schema.json | 7 -- tests/test_data.py | 23 ++++--- 9 files changed, 105 insertions(+), 108 deletions(-) create mode 100644 pyvecorg/build.py create mode 100644 pyvecorg/data/members_list_schema.json diff --git a/.gitignore b/.gitignore index a32ebae..bf064ff 100755 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,15 @@ -# Elsa output -_build - -### OSX +# OS +._* *.DS_Store -# Icon must end with two \r -Icon - -# Thumbnails -._* +# IDEs +.idea +.vscode -# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Cache +.cache +.pytest_cache -# User-specific stuff: -.idea +# Generated +/pyvecorg/_build +/pyvecorg/data/members_list.yml diff --git a/Pipfile b/Pipfile index 445ec4a..7b94351 100644 --- a/Pipfile +++ b/Pipfile @@ -4,6 +4,7 @@ verify_ssl = true name = "pypi" [scripts] +build = "python pyvecorg/build.py" test = "pytest" freeze = "python pyvecorg freeze" serve = "python pyvecorg serve" diff --git a/README.md b/README.md index c09af95..efc56b2 100755 --- a/README.md +++ b/README.md @@ -17,26 +17,42 @@ $ pipenv install The site uses [elsa](https://github.com/pyvec/elsa). - Installation: `pipenv install --dev` -- Development server: `pipenv run serve` +- Download data from external sources: `pipenv run build` - Tests: `pipenv run test` +- Development server: `pipenv run serve` ### Data and tests The site is just a single HTML page rendered on top of some static data. -However, the data can get quite complex and most of the texts need to be -translated into two languages. +However, some of the data come from external sources, the data can get quite +complex, and most of the texts need to be translated into two languages. The data is stored in multiple YAML files. When these are read, whenever an object has just `cs` and `en` properties, it is treated as a "translated text" and the property corresponding to a currently selected language becomes the actual value in place of the object. -Also, to keep the complex structure of the YAML files organized and tested, -there are schemas written in [JSON Schema](https://spacetelescope.github.io/understanding-json-schema/) +To keep the complex structure of the YAML files organized and tested, +there are schemas written in [JSON Schema](https://json-schema.org/understanding-json-schema/) ([spec](http://json-schema.org/)). In tests, the YAML files are validated against the schemas. There is also a couple of additional tests to ensure some logical rules which cannot be easily expressed by JSON Schema. +### External sources + +Some data cannot be stored statically in a YAML file. There is a command +`pipenv run build`, which downloads them from external sources and generates +respective static YAML files. This is a separate step, which needs to be done +before developing or deploying the site, otherwise it won't work properly. + +### Members + +Pyvec members are tracked in an internal Google Spreadsheet. The future +intention is to have the list of members public, but we're not there yet (GDPR). +So far only board members are being listed publicly. The `pipenv run build` +command downloads the spreadsheet as CSV and generates the `members_list.yml` +file. + ### Numbers There are stats numbers in [numbers.yml](pyvecorg/data/numbers.yml). They are diff --git a/pyvecorg/build.py b/pyvecorg/build.py new file mode 100644 index 0000000..9849699 --- /dev/null +++ b/pyvecorg/build.py @@ -0,0 +1,41 @@ +import csv +from pathlib import Path + +import yaml +import requests + + +MEMBERS_LIST_YAML = Path(__file__).parent / 'data' / 'members_list.yml' +CSV_URL = 'https://docs.google.com/spreadsheets/d/e/2PACX-1vSWK18MlEy95sAGl1BM6BXWxPgJbIx2UH3tAyJjxES06hHuaXgpsmD5pRz9kkGcFupiZL_U_e7yv4t1/pub?gid=0&single=true&output=csv' # noqa + + +def parse_members_csv(content): + lines = content.splitlines() + rows = csv.reader(lines, delimiter=',') + + head = None + for row in rows: + if frozenset(['name', 'role', 'avatar']) < frozenset(row): + head = row + break + for row in rows: + yield {key: value for key, value in zip(head, row) if value != ''} + + +def generate_yaml(data): + yaml_contents = '''\ +# +# This file has been generated from external sources +# using `pipenv run build`. Do not edit it manually! +# +''' + return yaml_contents + yaml.dump(data, allow_unicode=True) + + +if __name__ == '__main__': + # Build data/members_list.yml + response = requests.get(CSV_URL) + response.raise_for_status() + members = parse_members_csv(response.content.decode('utf-8')) + data = dict(entries=list(members)) + MEMBERS_LIST_YAML.write_text(generate_yaml(data)) diff --git a/pyvecorg/data/definitions/member_schema.json b/pyvecorg/data/definitions/member_schema.json index 7018d34..775205b 100644 --- a/pyvecorg/data/definitions/member_schema.json +++ b/pyvecorg/data/definitions/member_schema.json @@ -27,8 +27,6 @@ }, "additionalProperties": false, "required": [ - "name", - "role", - "avatar" + "name" ] } diff --git a/pyvecorg/data/members.yml b/pyvecorg/data/members.yml index 6c8671b..5342b78 100644 --- a/pyvecorg/data/members.yml +++ b/pyvecorg/data/members.yml @@ -2,75 +2,6 @@ heading: cs: Kdo je Pyvec en: Who is Pyvec -entries: - - name: Martin Bílek - role: chair - github: martinbilek - twitter: ajtea - linkedin: martin-b%C3%ADlek-47699542 - avatar: github - - - name: Jiří Bartoň - role: board - github: whiskybar - twitter: lurkingideas - avatar: github - - - name: Aleš Zoulek - role: board - github: aleszoulek - twitter: aleszoulek - linkedin: zoulek - avatar: github - - - name: Jakub Vysoký - role: board - github: kvbik - twitter: kvbik - linkedin: jakubvysoky - avatar: github - - - name: Vítězslav Pliska - role: board - github: whit - twitter: whiteck - linkedin: vitekpliska - avatar: github - - - name: Robin Gottfried - role: board - github: czervenka - twitter: czervenka - linkedin: robingottfried - avatar: github - - - name: Honza Král - role: board - github: honzakral - twitter: honzakral - linkedin: honzakral - avatar: github - - - name: Martin Burián - role: trustee - twitter: martin_burian - linkedin: martinburi8n - avatar: twitter - - - name: Filip Vařecha - role: trustee - github: xaralis - twitter: xaralis - linkedin: filip-vařecha-51625031 - avatar: github - - - name: Jiří Suchan - role: trustee - github: yedpodtrzitko - twitter: yedpodtrzitko - linkedin: yedpodtrzitko - avatar: twitter - roles: chair: cs: předseda @@ -78,9 +9,6 @@ roles: board: cs: člen rady en: board member - trustee: - cs: člen dozorčí rady - en: board of trustees member note: icon: user-circle-o diff --git a/pyvecorg/data/members_list_schema.json b/pyvecorg/data/members_list_schema.json new file mode 100644 index 0000000..ae4643c --- /dev/null +++ b/pyvecorg/data/members_list_schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "entries": { + "type": "array", + "items": { + "$ref": "definitions/member_schema.json#" + } + } + }, + "additionalProperties": false, + "required": [ + "entries" + ] +} diff --git a/pyvecorg/data/members_schema.json b/pyvecorg/data/members_schema.json index 30957de..7bf7caf 100644 --- a/pyvecorg/data/members_schema.json +++ b/pyvecorg/data/members_schema.json @@ -5,12 +5,6 @@ "heading": { "$ref": "definitions/translated_text_schema.json#" }, - "entries": { - "type": "array", - "items": { - "$ref": "definitions/member_schema.json#" - } - }, "roles": { "type": "object", "patternProperties": { @@ -27,7 +21,6 @@ "additionalProperties": false, "required": [ "heading", - "entries", "roles", "note" ] diff --git a/tests/test_data.py b/tests/test_data.py index 8de47a3..a2216a9 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -63,14 +63,9 @@ def test_data_section_is_valid(section_name, section): @pytest.mark.parametrize('member', [ - member for member in DATA['members']['entries'] + member for member in DATA['members_list']['entries'] ]) def test_data_member_is_valid(member): - assert member['role'] in list(DATA['members']['roles'].keys()) - - assert member.get('avatar') - assert member['avatar'] in ('github', 'twitter', 'email') - if member.get('email'): assert '@' in member['email'] if member.get('linkedin'): @@ -83,7 +78,17 @@ def test_data_member_is_valid(member): @pytest.mark.parametrize('member', [ - member for member in DATA['members']['entries'] + member for member in DATA['members_list']['entries'] + if member.get('role') +]) +def test_data_member_with_role_is_valid(member): + assert member['role'] in list(DATA['members']['roles'].keys()) + assert member.get('avatar') + assert member['avatar'] in ('github', 'twitter', 'email') + + +@pytest.mark.parametrize('member', [ + member for member in DATA['members_list']['entries'] if member.get('avatar') == 'github' ]) def test_data_member_has_valid_github_avatar(member): @@ -93,7 +98,7 @@ def test_data_member_has_valid_github_avatar(member): @pytest.mark.parametrize('member', [ - member for member in DATA['members']['entries'] + member for member in DATA['members_list']['entries'] if member.get('avatar') == 'twitter' ]) def test_data_member_has_valid_twitter_avatar(member): @@ -104,7 +109,7 @@ def test_data_member_has_valid_twitter_avatar(member): @pytest.mark.parametrize('member', [ - member for member in DATA['members']['entries'] + member for member in DATA['members_list']['entries'] if member.get('avatar') == 'email' ]) def test_data_member_has_valid_gravatar(member): From 78b0fc35945c7b6167bb103cdd296aa632e4dd4a Mon Sep 17 00:00:00 2001 From: Honza Javorek Date: Tue, 25 Jun 2019 10:27:00 +0200 Subject: [PATCH 02/10] get avatar URLs during the build step --- README.md | 8 --- pyvecorg/avatars.py | 2 - pyvecorg/build.py | 7 ++- pyvecorg/data/definitions/member_schema.json | 6 +- pyvecorg/views.py | 7 +-- tests/test_data.py | 59 +++++++++++++------- 6 files changed, 54 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index efc56b2..7ed1408 100755 --- a/README.md +++ b/README.md @@ -68,14 +68,6 @@ the number, 1. ask him to run the script, 2. bother him to Open Source the script code. -### Twitter avatars - -To figure out correct URLs of Twitter avatars from Twitter usernames, the site -needs to perform an HTTP request for each of them. This is not really an issue, -because on production this is done only once - in the moment of deployment. -However, it can get very annoying during development. Set `DISABLE_TWITTER_AVATARS` -environment variable to truthy value to disable Twitter avatars for development. - ## Deployment The site uses [elsa](https://github.com/pyvec/elsa). It gets automatically deployed diff --git a/pyvecorg/avatars.py b/pyvecorg/avatars.py index b297abb..3a3ad4d 100644 --- a/pyvecorg/avatars.py +++ b/pyvecorg/avatars.py @@ -19,8 +19,6 @@ def get_avatar_url(member): email_hash = hashlib.md5(value.lower().strip().encode()).hexdigest() return f'https://www.gravatar.com/avatar/{email_hash}?size=100&d=404' elif key == 'twitter': - if os.getenv('DISABLE_TWITTER_AVATARS'): - return no_avatar username = quote(value) url = f'https://twitter.com/{username}/profile_image' url = requests.head(url).headers.get('location') diff --git a/pyvecorg/build.py b/pyvecorg/build.py index 9849699..018dc74 100644 --- a/pyvecorg/build.py +++ b/pyvecorg/build.py @@ -4,6 +4,8 @@ import yaml import requests +from pyvecorg.avatars import get_avatar_url + MEMBERS_LIST_YAML = Path(__file__).parent / 'data' / 'members_list.yml' CSV_URL = 'https://docs.google.com/spreadsheets/d/e/2PACX-1vSWK18MlEy95sAGl1BM6BXWxPgJbIx2UH3tAyJjxES06hHuaXgpsmD5pRz9kkGcFupiZL_U_e7yv4t1/pub?gid=0&single=true&output=csv' # noqa @@ -36,6 +38,9 @@ def generate_yaml(data): # Build data/members_list.yml response = requests.get(CSV_URL) response.raise_for_status() - members = parse_members_csv(response.content.decode('utf-8')) + members = ( + dict(avatar_url=get_avatar_url(member), **member) for member + in parse_members_csv(response.content.decode('utf-8')) + ) data = dict(entries=list(members)) MEMBERS_LIST_YAML.write_text(generate_yaml(data)) diff --git a/pyvecorg/data/definitions/member_schema.json b/pyvecorg/data/definitions/member_schema.json index 775205b..6e7689a 100644 --- a/pyvecorg/data/definitions/member_schema.json +++ b/pyvecorg/data/definitions/member_schema.json @@ -23,10 +23,14 @@ }, "avatar": { "type": "string" + }, + "avatar_url": { + "type": "string" } }, "additionalProperties": false, "required": [ - "name" + "name", + "avatar_url" ] } diff --git a/pyvecorg/views.py b/pyvecorg/views.py index 8827f2a..ef89c72 100755 --- a/pyvecorg/views.py +++ b/pyvecorg/views.py @@ -51,10 +51,9 @@ def avatar(name): except IndexError: abort(404) - url = get_avatar_url(member) - res = requests.get(url) - res.raise_for_status() - file = create_thumbnail(io.BytesIO(res.content), 100) + response = requests.get(member['avatar_url']) + response.raise_for_status() + file = create_thumbnail(io.BytesIO(response.content), 100) return send_file(file, mimetype='image/png', as_attachment=True, attachment_filename=f'{name}.png') diff --git a/tests/test_data.py b/tests/test_data.py index a2216a9..5a57940 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -9,7 +9,6 @@ from pyvecorg.data import (load_data, select_language, DATA_PATH, SUPPORTED_LANGS) -from pyvecorg.avatars import get_avatar_url STATIC_PATH = os.path.join(DATA_PATH, '..', 'static') @@ -64,17 +63,35 @@ def test_data_section_is_valid(section_name, section): @pytest.mark.parametrize('member', [ member for member in DATA['members_list']['entries'] + if member.get('email') ]) -def test_data_member_is_valid(member): - if member.get('email'): - assert '@' in member['email'] - if member.get('linkedin'): - assert not member['linkedin'].startswith('http') - if member.get('github'): - assert not member['github'].startswith('http') - if member.get('twitter'): - assert not member['twitter'].startswith('http') - assert not member['twitter'].startswith('@') +def test_data_member_with_email_is_valid(member): + assert '@' in member['email'] + + +@pytest.mark.parametrize('member', [ + member for member in DATA['members_list']['entries'] + if member.get('linkedin') +]) +def test_data_member_with_linkedin_is_valid(member): + assert not member['linkedin'].startswith('http') + + +@pytest.mark.parametrize('member', [ + member for member in DATA['members_list']['entries'] + if member.get('github') +]) +def test_data_member_with_github_is_valid(member): + assert not member['github'].startswith('http') + + +@pytest.mark.parametrize('member', [ + member for member in DATA['members_list']['entries'] + if member.get('twitter') +]) +def test_data_member_with_twitter_is_valid(member): + assert not member['twitter'].startswith('http') + assert not member['twitter'].startswith('@') @pytest.mark.parametrize('member', [ @@ -83,7 +100,13 @@ def test_data_member_is_valid(member): ]) def test_data_member_with_role_is_valid(member): assert member['role'] in list(DATA['members']['roles'].keys()) - assert member.get('avatar') + + +@pytest.mark.parametrize('member', [ + member for member in DATA['members_list']['entries'] + if member.get('avatar') +]) +def test_data_member_with_avatar_is_valid(member): assert member['avatar'] in ('github', 'twitter', 'email') @@ -93,8 +116,7 @@ def test_data_member_with_role_is_valid(member): ]) def test_data_member_has_valid_github_avatar(member): assert member.get('github') - url = get_avatar_url(member) - is_working_link(url) + is_working_link(member['avatar_url']) @pytest.mark.parametrize('member', [ @@ -103,9 +125,8 @@ def test_data_member_has_valid_github_avatar(member): ]) def test_data_member_has_valid_twitter_avatar(member): assert member.get('twitter') - url = get_avatar_url(member) - is_working_link(url) - assert 'default_profile' not in url + assert 'default_profile' not in member['avatar_url'] + is_working_link(member['avatar_url']) @pytest.mark.parametrize('member', [ @@ -114,8 +135,8 @@ def test_data_member_has_valid_twitter_avatar(member): ]) def test_data_member_has_valid_gravatar(member): assert member.get('email') - url = get_avatar_url(member) - is_working_link(url) + assert '0000' not in member['avatar_url'] + is_working_link(member['avatar_url']) @pytest.mark.parametrize('logo', frozenset( From d0bd255020f2c6be63d0d1283c57ba6203511775 Mon Sep 17 00:00:00 2001 From: Honza Javorek Date: Tue, 25 Jun 2019 10:29:40 +0200 Subject: [PATCH 03/10] fix templates and the avatar view --- pyvecorg/templates/index.html | 2 +- pyvecorg/views.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyvecorg/templates/index.html b/pyvecorg/templates/index.html index dedbb0c..c2bba7a 100644 --- a/pyvecorg/templates/index.html +++ b/pyvecorg/templates/index.html @@ -153,7 +153,7 @@

{{ partners.heading }}

{{ members.heading }}