Adds data pipeline framework and some uk geographical datasources

.devcontainer/devcontainer.json

@@ -41,7 +41,6 @@
     "ms-python.python",
     "mtxr.sqltools",
     "mtxr.sqltools-driver-pg",
-    "matangover.mypy",
     "esbenp.prettier-vscode",
     "orta.vscode-jest"
   ],

.devcontainer/docker-compose.yml

@@ -15,6 +15,7 @@ services:
     environment:
       - DATABASE_URL=postgres://postgres:postgres@db:5432/postgres
       - DEBUG=True
+      - PY_IGNORE_IMPORTMISMATCH=1
 
     # Overrides default command so things don't shut down after the process ends.
     command: sleep infinity

.github/workflows/pull-request.yml

@@ -1,35 +1,35 @@
-name: Deploy to digitalocean
+name: pull-request
 on:
-  push:
+  pull_request:
     branches:
       - main
-
 jobs:
-  # Push image to GitHub Packages.
-  # See also https://docs.docker.com/docker-hub/builds/
-  push:
+  lint-and-test:
     runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: write
-
+    container:
+      image: ghcr.io/commonknowledge/do-app-baseimage-django-node:364385f9d196a2bbe2d5faea025520cc0316501f
+      # Workaround for: https://github.com/actions/checkout/issues/211
+      options: --user 1001
+      volumes:
+        - "/home/runner/docker/.cache:/home/app/.cache"
+      env:
+        DATABASE_URL: postgres://postgres:postgres@db:5432/postgres
+        DEBUG: True
+        PY_IGNORE_IMPORTMISMATCH: 1
+    services:
+      db:
+        image: kartoza/postgis:latest
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DBNAME: postgres
+          POSTGRES_HOSTNAME: postgres
+          POSTGRES_PORT: 5432
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Run tests in devcontainer
-        run: docker-compose -f .devcontainer/docker-compose.yml run -v .:/project -w /project app make ci
-
-        # This ugly bit is necessary if you don't want your cache to grow forever
-        # till it hits GitHub's limit of 5GB.
-        # Temp fix
-        # https://github.com/docker/build-push-action/issues/252
-        # https://github.com/moby/buildkit/issues/1896
-      - name: Move cache
-        if: always()
-        run: |
-          rm -rf /tmp/.buildx-cache
-          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: /home/runner/docker/.cache/poetry
+          key: do-app-baseimage-django-node:364385f9d196a2bbe2d5faea025520cc0316501f-poetry-${{ hashFiles('poetry.lock') }}
+      - run: make install
+      - run: make ci

.prettierignore

@@ -4,3 +4,5 @@ static
 node_modules
 .*_cache
 *.html
+docs/api
+docs/components

.vscode/launch.json

@@ -16,41 +16,29 @@
   ],
   "configurations": [
     {
-      "name": "Test current Django test",
+      "name": "django-runserver",
       "type": "python",
       "request": "launch",
       "program": "${workspaceFolder}/manage.py",
-      "args": ["test", "${file}"],
+      "args": ["runserver"],
       "justMyCode": true,
-      "django": true,
-      "presentation": {
-        "hidden": false,
-        "group": "",
-        "order": 2
-      }
+      "django": true
     },
     {
-      "name": "Run all Django tests",
+      "name": "django-run-cron-tasks",
       "type": "python",
       "request": "launch",
       "program": "${workspaceFolder}/manage.py",
-      "args": ["test", "test/*"],
+      "args": ["run_cron_tasks", "--once"],
       "justMyCode": true,
-      "django": true,
-      "presentation": {
-        "hidden": false,
-        "group": "",
-        "order": 3
-      }
+      "django": true
     },
     {
-      "name": "django-runserver",
+      "name": "gendocs",
       "type": "python",
       "request": "launch",
-      "program": "${workspaceFolder}/manage.py",
-      "args": ["runserver"],
-      "justMyCode": true,
-      "django": true
+      "program": "bin/gendocs.py",
+      "justMyCode": true
     },
     {
       "name": "vite-dev",
@@ -72,6 +60,9 @@
       "program": "${workspaceFolder}/node_modules/.bin/jest",
       "windows": {
         "program": "${workspaceFolder}/node_modules/jest/bin/jest"
+      },
+      "presentation": {
+        "hidden": true
       }
     }
   ]

.vscode/settings.json

@@ -1,11 +1,13 @@
 {
   "python.pythonPath": "/home/vscode/.cache/pypoetry/virtualenvs/groundwork-SsXfCHgY-py3.9/bin/python",
   "python.linting.pylintEnabled": false,
-  "python.linting.mypyEnabled": true,
+  "python.linting.mypyEnabled": false,
   "python.linting.banditEnabled": true,
-  "python.languageServer": "Jedi",
+  "python.testing.autoTestDiscoverOnSaveEnabled": true,
+  "python.testing.pytestEnabled": true,
   "editor.formatOnSave": true,
   "html.format.templating": true,
+  "jest.autoRun": "off",
   "[django-html]": {
     "editor.formatOnSave": false
   }

Makefile

@@ -19,7 +19,6 @@ poetry-remove:
 .PHONY: install
 install:
 	poetry install -n
-	poetry run mypy --install-types --non-interactive ./
 	yarn
 
 .PHONY: pre-commit-install
@@ -82,7 +81,7 @@ deploy-docs: api-docs
 
 .PHONY: test
 test:
-	poetry run python manage.py test test/*
+	poetry run pytest -vs -m "not integration_test"
 	yarn test
 
 .PHONY: check-codestyle
@@ -93,18 +92,19 @@ check-codestyle:
 	yarn tsc --noemit
 	yarn prettier --check .
 
-.PHONY: mypy
-mypy:
-	poetry run mypy .
-
 .PHONY: check-safety
 check-safety:
 	poetry check
 	poetry run safety check --full-report
 	poetry run bandit -ll --recursive pyck tests
 
 .PHONY: lint
-lint: test check-codestyle mypy check-safety
+lint: check-codestyle check-safety test
+
+.PHONY: ci
+ci: lint
+	poetry run pytest
+	yarn test
 
 
 #* Assets
@@ -123,7 +123,7 @@ pycache-remove:
 
 .PHONY: build-remove
 build-remove:
-	rm -rf build/
+	rm -rf build/ dist/ docs/api/ docs/components/ temp/
 
 .PHONY: clean-all
-clean-all: pycache-remove build-remove docker-remove
+clean-all: pycache-remove build-remove

README.md

@@ -1,8 +1,6 @@
 # Pycommonknowledge
 
-![Logo](./logo.png)
-
-An integrated and opinionated collection of Django applications and javascript components addressing needs for people building software for organisers and campaigners.
+An integrated Django and Javascript framework for people who build tools for organisers.
 
 For more information, check out [the documentation site](https://groundwork.commonknowledge.coop/).
 

bin/gendocs.py

@@ -40,9 +40,10 @@ def recursive_htmls(mod):
         yield from recursive_htmls(submod)
 
 
-for mod in modules:
-    for module_name, html in recursive_htmls(mod):
-        docs_path = OUTPUT_DIR / f"{module_name}.md"
+if __name__ == "__main__":
+    for mod in modules:
+        for module_name, html in recursive_htmls(mod):
+            docs_path = OUTPUT_DIR / f"{module_name}.md"
 
-        with open(str(docs_path), "w", encoding="utf8") as f:
-            f.write(html)
+            with open(str(docs_path), "w", encoding="utf8") as f:
+                f.write(html)

docs/contributing.md

@@ -10,7 +10,7 @@ We don't want to have an excessively prescriptive sense of what is 'in' and 'out
 2. Remove any dependencies from application code outside the `commonknowledge` package. Start to think about how it can be tested in isolation (if it isn't already).
 3. Open a feature request against this repository. Describe the new feature, include links to your implementation other repositories. If the application is publicly accessible, include links to it in the live app.
 4. Discuss and refine the API with other contributors.
-5. When the feature request is accepted, fork this repository (or create a feature branch if you have write access) and commit the feature implementation. Ensure that you have good test coverage of both python and javascript components and all public API methods are documented.
+5. When the feature request is accepted, fork this repository (or create a feature branch if you have write access) and commit the feature implementation. Ensure that you have good test coverage of both Python and Javascript components and all public API methods are documented.
 
 ## Bugs & backward-compatible API changes