Download the list of members from Google Spreadsheet, download avatars as files

.circleci/config.yml

@@ -11,6 +11,9 @@ jobs:
           command: |
             pip install pipenv --upgrade
             pipenv install --dev
+      - run:
+          name: build data
+          command: pipenv run build
       - run:
           name: run tests
           command: pipenv run test

.gitignore

@@ -1,16 +1,19 @@
-# Elsa output
-_build
-
-### OSX
+# OS
+._*
 *.DS_Store
 
-# Icon must end with two \r
-Icon
+# IDEs
+.idea
+.vscode
 
-# Thumbnails
-._*
+# Cache
+.cache
+.pytest_cache
 
-# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Generated
+/pyvecorg/_build
+/pyvecorg/data/members_list.yml
+/pyvecorg/static/img/avatars
 
-# User-specific stuff:
-.idea
+# Private
+google_service_account.json

Pipfile

@@ -4,6 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [scripts]
+build = "python pyvecorg/build.py"
 test = "pytest"
 freeze = "python pyvecorg freeze"
 serve = "python pyvecorg serve"
@@ -19,6 +20,8 @@ pyyaml = "~=5.1"
 requests = "~=2.22.0"
 python-slugify = "~=3.0.2"
 pillow = "~=6.1.0"
+gspread = "~=3.1.0"
+oauth2client = "~=4.1.3"
 
 [dev-packages]
 # Pinning packages with ~= unless their version starts with 0.,

README.md

@@ -17,26 +17,42 @@ $ pipenv install
 The site uses [elsa](https://github.com/pyvec/elsa).
 
 - Installation: `pipenv install --dev`
-- Development server: `pipenv run serve`
+- Download data from external sources: `pipenv run build`
 - Tests: `pipenv run test`
+- Development server: `pipenv run serve`
 
 ### Data and tests
 
 The site is just a single HTML page rendered on top of some static data.
-However, the data can get quite complex and most of the texts need to be
-translated into two languages.
+However, some of the data come from external sources, the data can get quite
+complex, and most of the texts need to be translated into two languages.
 
 The data is stored in multiple YAML files. When these are read, whenever
 an object has just `cs` and `en` properties, it is treated as a "translated text"
 and the property corresponding to a currently selected language becomes
 the actual value in place of the object.
 
-Also, to keep the complex structure of the YAML files organized and tested,
-there are schemas written in [JSON Schema](https://spacetelescope.github.io/understanding-json-schema/)
+To keep the complex structure of the YAML files organized and tested,
+there are schemas written in [JSON Schema](https://json-schema.org/understanding-json-schema/)
 ([spec](http://json-schema.org/)). In tests, the YAML files are validated
 against the schemas. There is also a couple of additional tests to ensure some
 logical rules which cannot be easily expressed by JSON Schema.
 
+### External sources
+
+Some data cannot be stored statically in a YAML file. There is a command
+`pipenv run build`, which downloads them from external sources and generates
+respective static YAML files. This is a separate step, which needs to be done
+before developing or deploying the site, otherwise it won't work properly.
+
+### Members
+
+Pyvec members are tracked in an internal Google Spreadsheet. The future
+intention is to have the list of members public, but we're not there yet (GDPR).
+So far only board members are being listed publicly. The `pipenv run build`
+command downloads the spreadsheet as CSV and generates the `members_list.yml`
+file. It also downloads and caches avatars.
+
 ### Numbers
 
 There are stats numbers in [numbers.yml](pyvecorg/data/numbers.yml). They are
@@ -50,15 +66,16 @@ as well as it clones and analyzes all relevant repositories. If you need to upda
 the number,
 
 1. ask him to run the script,
-2. bother him to Open Source the script code.
+1. bother him to Open Source the script code.
 
-### Twitter avatars
+### Google Drive credentials
 
-To figure out correct URLs of Twitter avatars from Twitter usernames, the site
-needs to perform an HTTP request for each of them. This is not really an issue,
-because on production this is done only once - in the moment of deployment.
-However, it can get very annoying during development. Set `DISABLE_TWITTER_AVATARS`
-environment variable to truthy value to disable Twitter avatars for development.
+1.  Follow the steps in the [gspread guide](https://gspread.readthedocs.io/en/latest/oauth2.html). Instead of Google Drive API, enable Google Sheets API.
+1.  Save the obtained JSON file into the `pyvecorg` package as `google_service_account.json`
+1.  Make sure it is ignored by Git
+1.  Run `cat pyvecorg/google_service_account.json | pbcopy` to copy the JSON into your clipboard (macOS)
+1.  Go to [Travis CI project settings](https://travis-ci.org/pyvec/pyvec.org/settings), section Environment Variables
+1.  Add `GOOGLE_SERVICE_ACCOUNT` variable and paste the JSON from your clipboard as a value
 
 ## Deployment
 

pyvecorg/__init__.py

@@ -1,4 +1,3 @@
-import os
 import itertools
 from pathlib import Path
 
@@ -27,8 +26,8 @@ def run(self, *args, **kwargs):
 app.config['JSON_AS_ASCII'] = False
 
 
-from pyvecorg import views
-from pyvecorg import templating
+from pyvecorg import views  # noqa
+from pyvecorg import templating  # noqa
 
 
 __all__ = ['app', 'views', 'templating']

pyvecorg/avatars.py

@@ -1,4 +1,3 @@
-import os
 import io
 from urllib.parse import quote
 import hashlib
@@ -19,8 +18,6 @@ def get_avatar_url(member):
         email_hash = hashlib.md5(value.lower().strip().encode()).hexdigest()
         return f'https://www.gravatar.com/avatar/{email_hash}?size=100&d=404'
     elif key == 'twitter':
-        if os.getenv('DISABLE_TWITTER_AVATARS'):
-            return no_avatar
         username = quote(value)
         url = f'https://twitter.com/{username}/profile_image'
         url = requests.head(url).headers.get('location')

pyvecorg/build.py

@@ -0,0 +1,92 @@
+import io
+import os
+import json
+from textwrap import dedent
+from pathlib import Path
+
+import yaml
+import requests
+from slugify import slugify
+import gspread
+from oauth2client.service_account import ServiceAccountCredentials
+
+from pyvecorg.avatars import get_avatar_url, create_thumbnail
+
+
+PACKAGE_DIR = Path(__file__).parent
+MEMBERS_LIST_YAML = PACKAGE_DIR / 'data' / 'members_list.yml'
+MEMBERS_CSV_URL = 'https://docs.google.com/spreadsheets/d/e/2PACX-1vSWK18MlEy95sAGl1BM6BXWxPgJbIx2UH3tAyJjxES06hHuaXgpsmD5pRz9kkGcFupiZL_U_e7yv4t1/pub?gid=0&single=true&output=csv'  # noqa
+STATIC_DIR = PACKAGE_DIR / 'static'
+AVATARS_DIR = STATIC_DIR / 'img' / 'avatars'
+
+
+def read_spreadsheet(doc_key, sheet_name, google_service_account):
+    credentials = ServiceAccountCredentials.from_json_keyfile_dict(
+        google_service_account,
+        [
+            'https://spreadsheets.google.com/feeds',
+            'https://www.googleapis.com/auth/drive',
+        ]
+    )
+    doc = gspread.authorize(credentials).open_by_key(doc_key)
+    return doc.worksheet(sheet_name).get_all_values()
+
+
+def parse_members(rows):
+    head = None
+    for row in rows:
+        if frozenset(['name', 'role', 'avatar']) < frozenset(row):
+            head = row
+            break
+    for row in rows:
+        yield {key: value for key, value in zip(head, row) if value != ''}
+
+
+def generate_yaml(data):
+    yaml_contents = dedent('''\
+        #
+        #  This file has been generated from external sources
+        #  using `pipenv run build`. Do not edit it manually!
+        #
+    ''')
+    return yaml_contents + yaml.dump(data, allow_unicode=True)
+
+
+def create_member_sorting_key(member):
+    return (
+        0 if member.get('role') == 'chair' else 1,  # chair to be first
+        member.get('nickname', member['name']).split(' ')[-1],  # last name
+    )
+
+
+if __name__ == '__main__':
+    # Build data/members_list.yml and avatar images
+    gsa_path = PACKAGE_DIR / 'google_service_account.json'
+    gsa_json = os.getenv('GOOGLE_SERVICE_ACCOUNT') or gsa_path.read_text()
+    gsa = json.loads(gsa_json)
+
+    # Document key appears in the URL if you have the document open
+    # in your browser
+    doc_key = '1n8hzBnwZ5ANkUCvwEy8rWsXlqeAAwu-5JBodT5OJx_I'
+    rows = read_spreadsheet(doc_key, 'list', gsa)
+    members = sorted([
+        member for member in parse_members(rows)
+        if member.get('role') in ('board', 'chair')
+    ], key=create_member_sorting_key)
+
+    AVATARS_DIR.mkdir(exist_ok=True)
+    for member in sorted(members, key=create_member_sorting_key):
+        avatar_url = get_avatar_url(member)
+        img_basename = slugify(member['name']) + '.png'
+
+        response = requests.get(avatar_url)
+        response.raise_for_status()
+        img_bytes = create_thumbnail(io.BytesIO(response.content), 100).read()
+
+        img_path = (AVATARS_DIR / img_basename)
+        img_path.write_bytes(img_bytes)
+
+        member['avatar_filename'] = str(img_path.relative_to(STATIC_DIR))
+
+    data = dict(entries=list(members))
+    MEMBERS_LIST_YAML.write_text(generate_yaml(data))

pyvecorg/data/definitions/member_schema.json

@@ -5,6 +5,9 @@
     "name": {
       "type": "string"
     },
+    "nickname": {
+      "type": "string"
+    },
     "role": {
       "type": "string"
     },
@@ -23,12 +26,20 @@
     },
     "avatar": {
       "type": "string"
+    },
+    "avatar_filename": {
+      "type": "string"
+    },
+    "address": {
+      "type": "string"
+    },
+    "dob": {
+      "type": "string"
     }
   },
   "additionalProperties": false,
   "required": [
     "name",
-    "role",
-    "avatar"
+    "avatar_filename"
   ]
 }