Add tests to check upstream data

.github/workflows/main.yml

@@ -38,7 +38,7 @@ jobs:
           pip install -r requirements.txt
 
       - name: Run tests
-        run: python -m pytest
+        run: python -m pytest -v -m "not webtest"
         env:
           # The hostname, username used to communicate with the PostgreSQL service container
           POSTGRES_HOST: localhost

.github/workflows/upstream_test.yml

@@ -0,0 +1,51 @@
+on:
+  workflow_dispatch: # allow manual execution
+  push:
+  schedule:
+    # run on every 9 o'clock
+    - cron: '0 9 * * *'
+
+jobs:
+  unit_tests:
+    runs-on: ubuntu-latest
+
+    services:
+      # Label used to access the service container
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_PASSWORD: vulnerablecode
+          POSTGRES_DB: vulnerablecode
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install dependencies
+        run: |
+          sudo apt install  python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev 
+          python -m pip install --upgrade pip 
+          pip install -r requirements.txt
+
+      - name: Run tests
+        run: pytest -v -m webtest
+        env:
+          # The hostname, username used to communicate with the PostgreSQL service container
+          POSTGRES_HOST: localhost
+          VC_DB_USER: postgres
+          POSTGRES_PORT: 5432
+          DJANGO_DEV: 1
+          GH_TOKEN: 1 

.travis.yml

@@ -18,7 +18,7 @@ before_script:
 
 script:
   - ./manage.py collectstatic
-  - python -m pytest
+  - python -m pytest -v -m "not webtest"
 
 notifications:
   email: false

vulnerabilities/tests/test_upstream.py

@@ -0,0 +1,18 @@
+import pytest
+from vulnerabilities import importers
+from vulnerabilities.importer_yielder import IMPORTER_REGISTRY
+
+
+@pytest.mark.webtest
+@pytest.mark.parametrize(
+    ("data_source", "config"),
+    ((data["data_source"], data["data_source_cfg"]) for data in IMPORTER_REGISTRY),
+)
+def test_updated_advisories(data_source, config):
+
+    if not data_source == "GitHubAPIDataSource":
+        data_src = getattr(importers, data_source)
+        data_src = data_src(batch_size=1, config=config)
+        with data_src:
+            for i in data_src.updated_advisories():
+                pass