From 84dc814a5b7ac7871d1685174522561b2a4c9719 Mon Sep 17 00:00:00 2001 From: Fabio Pinto Date: Mon, 16 Sep 2024 11:41:56 +0100 Subject: [PATCH 1/9] version needed for sbom-repo dependency --- surface/sca/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surface/sca/requirements.txt b/surface/sca/requirements.txt index a9e77c76..94de8748 100644 --- a/surface/sca/requirements.txt +++ b/surface/sca/requirements.txt @@ -1,3 +1,3 @@ CVSS==2.6 -packageurl-python==0.15.1 +packageurl-python==0.15.0 semver==3.0.1 From 4e417acd7c00bc4a3e0f4f1ecc8f80e4a0b82b94 Mon Sep 17 00:00:00 2001 From: Fabio Pinto Date: Mon, 16 Sep 2024 11:42:32 +0100 Subject: [PATCH 2/9] exclude empty fixed in from vuln counters --- surface/requirements.txt | 9 ++++----- surface/sca/admin.py | 6 +++--- .../0002_alter_scafinding_fixed_in.py | 18 ++++++++++++++++++ surface/sca/models.py | 5 ++--- surface/sca/tests/test_admin.py | 2 +- surface/sca/tests/test_resync_sbom_repo.py | 2 +- 6 files changed, 29 insertions(+), 13 deletions(-) create mode 100644 surface/sca/migrations/0002_alter_scafinding_fixed_in.py diff --git a/surface/requirements.txt b/surface/requirements.txt index 3cecc04e..6e8c0807 100644 --- a/surface/requirements.txt +++ b/surface/requirements.txt @@ -1,7 +1,6 @@ # Core Libraries - -Django==3.2.25 -django-admin-rangefilter==0.11.0 +Django==3.2.23 +django-admin-rangefilter==0.10.0 django-after-response==0.2.2 django-object-actions==4.2.0 djangorestframework==3.14.0 @@ -25,9 +24,9 @@ django-olympus==0.0.5 django-environ-ppb[vault]==1.0.1 django-impersonator==0.0.2 django-apitokens==0.0.2 -django-sbomrepo==0.0.6 +django-sbomrepo==0.0.4 -mysqlclient==2.0.3 +mysqlclient==2.2.4 tqdm==4.65.0 # for core_utils that is not really a app/package ..? django-database-locks==0.5 # distributed locks (on mysql) django-bulk-update-or-create==0.3.0 # for faster batch operations with update_or_create diff --git a/surface/sca/admin.py b/surface/sca/admin.py index 121deb1f..4e9c9d0e 100644 --- a/surface/sca/admin.py +++ b/surface/sca/admin.py @@ -17,10 +17,10 @@ from core_utils.admin_filters import DefaultFilterMixin from core_utils.utils import admin_reverse from dkron.utils import run_async +from inventory.models import GitSource from sca import models from sca.utils import only_highest_version_dependencies from theme.filters import RelatedFieldAjaxListFilter -from inventory.models import GitSource logger = logging.getLogger(__name__) @@ -190,9 +190,9 @@ class Meta: def filter_fixed_in(self, queryset, name, value): if value == "true": - return queryset.exclude(Q(fixed_in="") | Q(fixed_in__isnull=True)) + return queryset.exclude(fixed_in="") elif value == "false": - return queryset.filter(Q(fixed_in="") | Q(fixed_in__isnull=True)) + return queryset.filter(fixed_in="") return queryset diff --git a/surface/sca/migrations/0002_alter_scafinding_fixed_in.py b/surface/sca/migrations/0002_alter_scafinding_fixed_in.py new file mode 100644 index 00000000..5bc2ba30 --- /dev/null +++ b/surface/sca/migrations/0002_alter_scafinding_fixed_in.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.23 on 2024-09-16 10:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sca', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='scafinding', + name='fixed_in', + field=models.TextField(default=''), + ), + ] diff --git a/surface/sca/models.py b/surface/sca/models.py index ba6ac14a..04c0957d 100644 --- a/surface/sca/models.py +++ b/surface/sca/models.py @@ -2,7 +2,6 @@ from enum import Enum from typing import Union -from bulk_update_or_create import BulkUpdateOrCreateQuerySet from django.db import models from django.db.models import Case, Count, Q, When @@ -164,7 +163,7 @@ def get_dependencies(root_dependency: "SCADependency") -> list: def update_vulnerability_counters(self) -> "SCAFindingCounter": severity_counters = ( SCAFinding.objects.filter( - (Q(fixed_in__isnull=False) | Q(finding_type=SCAFinding.FindingType.EOL)), + (Q(fixed_in__gt="") | Q(finding_type=SCAFinding.FindingType.EOL)), dependency__purl__in=self.dependencies, state__in=(SCAFinding.State.NEW, SCAFinding.State.OPEN), ) @@ -247,7 +246,7 @@ class FindingType(models.IntegerChoices): vuln_id = models.CharField(max_length=128) published = models.DateTimeField() aliases = models.TextField(default="") - fixed_in = models.TextField(default=None, null=True) + fixed_in = models.TextField(default="") cvss_vector = models.CharField(max_length=128, default="") ecosystem = models.CharField(max_length=20) finding_type = models.IntegerField(choices=FindingType.choices, default=FindingType.VULN) diff --git a/surface/sca/tests/test_admin.py b/surface/sca/tests/test_admin.py index 71d7e08d..d6312143 100644 --- a/surface/sca/tests/test_admin.py +++ b/surface/sca/tests/test_admin.py @@ -73,7 +73,7 @@ def test_admin_changelist(self): # Assert Vulnerabilities Counters assert "1 Critical" in content - assert "4 High" in content + assert "3 High" in content assert "3 Medium" in content assert "0 Low" in content assert "0 End of Life" in content diff --git a/surface/sca/tests/test_resync_sbom_repo.py b/surface/sca/tests/test_resync_sbom_repo.py index 56ae9d44..849227cd 100644 --- a/surface/sca/tests/test_resync_sbom_repo.py +++ b/surface/sca/tests/test_resync_sbom_repo.py @@ -50,7 +50,7 @@ def test_resync_sbom_repo(self, now): assert SCAFindingCounter.objects.filter(dependency=main_dependency).exists() counter = SCAFindingCounter.objects.filter(dependency=main_dependency).first() assert counter.critical == 1 - assert counter.high == 4 + assert counter.high == 3 assert counter.medium == 3 # Asserts main dependency has only one git source "https://github.com/test/repo" From 6bb41896d23535a59c5d9d7af4458b00d948ac38 Mon Sep 17 00:00:00 2001 From: Fabio Pinto Date: Mon, 16 Sep 2024 12:03:34 +0100 Subject: [PATCH 3/9] pkg-config, mysqlclient 2.2.4 --- dev/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/Dockerfile b/dev/Dockerfile index b6da0638..2f63cd36 100644 --- a/dev/Dockerfile +++ b/dev/Dockerfile @@ -17,6 +17,7 @@ RUN apt-get update \ libldap2-dev \ libsasl2-dev \ git \ + pkg-config \ && rm -rf /var/lib/apt/lists/* WORKDIR /wheels From 317830ab9f9b7e3381e5532ccf862ad6e108d43c Mon Sep 17 00:00:00 2001 From: Fabio Pinto Date: Mon, 16 Sep 2024 12:15:57 +0100 Subject: [PATCH 4/9] . --- dev/Dockerfile-IN-A-BOX | 1 + pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/Dockerfile-IN-A-BOX b/dev/Dockerfile-IN-A-BOX index 86922c07..249b5289 100644 --- a/dev/Dockerfile-IN-A-BOX +++ b/dev/Dockerfile-IN-A-BOX @@ -7,6 +7,7 @@ RUN apt-get update \ libldap2-dev \ libsasl2-dev \ git \ + pkg-config \ && rm -rf /var/lib/apt/lists/* RUN --mount=type=bind,target=/tmpapp \ diff --git a/pyproject.toml b/pyproject.toml index 780912a0..8d6f161a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,4 +38,4 @@ select = [ src = ['surface', 'e2e'] [tool.ruff.isort] -known-first-party = ["theme", "dkron", "django_restful_admin", "slackbot", "dbcleanup", "olympus", "notifications", "ppbenviron", "logbasecommand", "impersonate", "apitokens"] +known-first-party = ["theme", "dkron", "django_restful_admin", "slackbot", "dbcleanup", "olympus", "notifications", "ppbenviron", "logbasecommand", "impersonate", "apitokens", "sbomrepo"] From 734cf3cca409f6bb6cb425071709672a66c0856b Mon Sep 17 00:00:00 2001 From: Fabio Pinto Date: Mon, 16 Sep 2024 12:19:04 +0100 Subject: [PATCH 5/9] dep versions --- surface/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/surface/requirements.txt b/surface/requirements.txt index 6e8c0807..0c56f276 100644 --- a/surface/requirements.txt +++ b/surface/requirements.txt @@ -1,6 +1,6 @@ # Core Libraries -Django==3.2.23 -django-admin-rangefilter==0.10.0 +Django==3.2.25 +django-admin-rangefilter==0.11.0 django-after-response==0.2.2 django-object-actions==4.2.0 djangorestframework==3.14.0 @@ -24,7 +24,7 @@ django-olympus==0.0.5 django-environ-ppb[vault]==1.0.1 django-impersonator==0.0.2 django-apitokens==0.0.2 -django-sbomrepo==0.0.4 +django-sbomrepo==0.0.6 mysqlclient==2.2.4 tqdm==4.65.0 # for core_utils that is not really a app/package ..? From 9329264c3146dac4f3e1a89e7d1b475720cc0f03 Mon Sep 17 00:00:00 2001 From: Fabio Pinto Date: Mon, 16 Sep 2024 12:22:11 +0100 Subject: [PATCH 6/9] fixes sbomrepor deps requirement --- surface/sca/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surface/sca/requirements.txt b/surface/sca/requirements.txt index 94de8748..a9e77c76 100644 --- a/surface/sca/requirements.txt +++ b/surface/sca/requirements.txt @@ -1,3 +1,3 @@ CVSS==2.6 -packageurl-python==0.15.0 +packageurl-python==0.15.1 semver==3.0.1 From 8bec1e7966d09136e7b66642df49cff376efe546 Mon Sep 17 00:00:00 2001 From: Fabio Pinto Date: Mon, 16 Sep 2024 15:11:51 +0100 Subject: [PATCH 7/9] dependencies --- .github/workflows/integration.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 4d9c99ce..3067e19f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -12,6 +12,10 @@ jobs: steps: - uses: actions/checkout@v4 + + - name: install mysql dependencies + run: sudo apt-get install python3-dev default-libmysqlclient-dev build-essential pkg-config + - name: Set-up environment run: pip install -r surface/requirements_test.txt From cfc8984e388cadb64b691949bbd9f8fbd6340a3b Mon Sep 17 00:00:00 2001 From: Fabio Pinto Date: Mon, 16 Sep 2024 15:19:28 +0100 Subject: [PATCH 8/9] . --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 3067e19f..c8d1aba5 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v4 - name: install mysql dependencies - run: sudo apt-get install python3-dev default-libmysqlclient-dev build-essential pkg-config + run: sudo apt-get install python3-dev libmysqlclient-dev build-essential pkg-config - name: Set-up environment run: pip install -r surface/requirements_test.txt From 12bbaa8d959b7f67e7426ac735a5291558de1d1d Mon Sep 17 00:00:00 2001 From: Fabio Pinto Date: Mon, 16 Sep 2024 15:30:02 +0100 Subject: [PATCH 9/9] python 3.9 on integration test --- .github/workflows/integration.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c8d1aba5..6d6980d6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -12,9 +12,9 @@ jobs: steps: - uses: actions/checkout@v4 - - - name: install mysql dependencies - run: sudo apt-get install python3-dev libmysqlclient-dev build-essential pkg-config + - uses: actions/setup-python@v5 + with: + python-version: '3.9' - name: Set-up environment run: pip install -r surface/requirements_test.txt