From d91317666d82dde8e0f38b7ab6ed527fc91c95c1 Mon Sep 17 00:00:00 2001 From: Gustavo Silva Date: Fri, 13 Jan 2023 11:22:14 +0000 Subject: [PATCH 1/4] dns_ips: Deprecate iprangethirdparty w tags in iprange --- surface/dns_ips/admin.py | 10 +---- .../0004_deprecate_iprangethird_party.py | 44 +++++++++++++++++++ surface/dns_ips/models.py | 18 +------- surface/dns_ips/slack.py | 1 - 4 files changed, 46 insertions(+), 27 deletions(-) create mode 100644 surface/dns_ips/migrations/0004_deprecate_iprangethird_party.py diff --git a/surface/dns_ips/admin.py b/surface/dns_ips/admin.py index 2ca19be4..53d7bdb5 100644 --- a/surface/dns_ips/admin.py +++ b/surface/dns_ips/admin.py @@ -55,7 +55,7 @@ class IPRangeAdmin(DefaultFilterMixin, admin.ModelAdmin): ] list_display_links = ('range',) search_fields = ('range', 'zone', 'datacenter', 'description', 'notes') - list_filter = ('source', 'active', 'zone', 'datacenter') + list_filter = ('source', 'active', 'zone', 'datacenter', 'tags') exclude = ('range_min', 'range_max') def get_search_results(self, request, queryset, search_term): @@ -78,14 +78,6 @@ def get_default_filters(self, request): return {'active__exact': 1} -@admin.register(models.IPRangeThirdParty) -class IPRangeThirdParty(admin.ModelAdmin): - list_display = [field.name for field in models.IPRangeThirdParty._meta.fields if field.name not in ('id',)] - search_fields = ('range__range', 'description', 'expected_traffic', 'expected_ports', 'expected_protocol', 'sn_ref') - list_display_links = ('range',) - list_filter = ('expected_traffic', 'expected_ports', 'expected_protocol') - - @admin.register(models.IPAddress) class IPAddress(DefaultFilterMixin, admin.ModelAdmin): list_display = ( diff --git a/surface/dns_ips/migrations/0004_deprecate_iprangethird_party.py b/surface/dns_ips/migrations/0004_deprecate_iprangethird_party.py new file mode 100644 index 00000000..cdc674f8 --- /dev/null +++ b/surface/dns_ips/migrations/0004_deprecate_iprangethird_party.py @@ -0,0 +1,44 @@ +# Generated by Django 3.2.12 on 2023-01-06 16:21 + +from ipaddress import ip_network + +from django.db import migrations, models +from django.db.utils import IntegrityError + + +def tag_existing_ranges(apps, db_schema): + iprange = apps.get_model("dns_ips", "iprange") + iprange3rd = apps.get_model("dns_ips", "iprangethirdparty") + internal_tag = apps.get_model("dns_ips", "tag").objects.filter(name="is_internal").first() + external_tag = apps.get_model("dns_ips", "tag").objects.filter(name="is_external").first() + trd_party_tag = apps.get_model("dns_ips", "tag").objects.filter(name="is_third_party").first() + + for thrdpartyrange in iprange3rd.objects.all(): + for rng in iprange.objects.filter(range=thrdpartyrange.range.range).exclude(tags__id=trd_party_tag.id): + if rng.description is None and thrdpartyrange.description is not None: + rng.description = thrdpartyrange.description + + try: + if ip_network(rng.range).is_private: + rng.tags.add(internal_tag.id, trd_party_tag.id) + else: + rng.tags.add(external_tag.id, trd_party_tag.id) + except ValueError or IntegrityError: + continue + + +class Migration(migrations.Migration): + + dependencies = [ + ("dns_ips", "0003_alter_dnsrecord_tla"), + ] + + operations = [ + migrations.AddField( + model_name="iprange", + name="tags", + field=models.ManyToManyField(blank=True, to="dns_ips.Tag"), + ), + migrations.RunPython(tag_existing_ranges, reverse_code=migrations.RunPython.noop), + migrations.DeleteModel("iprangethirdparty"), + ] diff --git a/surface/dns_ips/models.py b/surface/dns_ips/models.py index 115f9fb4..b4c4f840 100644 --- a/surface/dns_ips/models.py +++ b/surface/dns_ips/models.py @@ -91,6 +91,7 @@ class IPRange(RangeModel): datacenter = models.CharField(max_length=255, null=True, blank=True) description = models.TextField(null=True, blank=True) notes = models.TextField(null=True, blank=True) + tags = models.ManyToManyField("dns_ips.Tag", blank=True) def __str__(self): return self.range @@ -103,23 +104,6 @@ class Meta: verbose_name_plural = "IP Ranges" -class IPRangeThirdParty(models.Model): - range = models.ForeignKey("dns_ips.IPRange", blank=True, null=True, on_delete=models.CASCADE) - organisation = models.ForeignKey("Organisation", blank=True, null=True, on_delete=models.CASCADE) - sn_ref = models.CharField(max_length=64, blank=True, null=True) - expected_traffic = models.CharField(max_length=255, blank=True, null=True, db_index=True) - expected_ports = models.CharField(max_length=255, blank=True, null=True, db_index=True) - expected_protocol = models.CharField(max_length=255, blank=True, null=True, db_index=True) - description = models.TextField(blank=True, null=True) - - def __str__(self): - return f"{self.sn_ref} ({self.expected_traffic})" - - class Meta: - verbose_name = "IP Range - Third Party (SN)" - verbose_name_plural = "IP Ranges - Third Parties (SN)" - - class IPAddress(models.Model): objects = BulkUpdateOrCreateQuerySet.as_manager() diff --git a/surface/dns_ips/slack.py b/surface/dns_ips/slack.py index 5b21f278..e3afb939 100644 --- a/surface/dns_ips/slack.py +++ b/surface/dns_ips/slack.py @@ -22,7 +22,6 @@ class DNSIPSProcessor(MessageProcessor): IGNORED_TERMS = ['1.1.1.1', '127.0.0.1', 'internal', 'localhost'] IPS_MODELS = [ 'dns_ips.iprange', - 'dns_ips.iprangethirdparty', 'dns_ips.ipaddress', 'dns_ips.dnsrecord', ] From 436df4c96036763216e96894e9a9f53aeca1779b Mon Sep 17 00:00:00 2001 From: Gustavo Silva Date: Thu, 19 Oct 2023 15:27:02 +0100 Subject: [PATCH 2/4] misc: upgrade notifications-sender --- surface/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surface/requirements.txt b/surface/requirements.txt index 600b122d..e2732ec6 100644 --- a/surface/requirements.txt +++ b/surface/requirements.txt @@ -17,7 +17,7 @@ netaddr==0.8.0 django-surface-theme==0.0.10 django-dbcleanup==0.1.4 django-logbasecommand==0.0.2 -django-notification-sender[slack]==0.0.5 +django-notification-sender[slack]==0.0.6 django-dkron==1.1.1 django-slack-processor==0.0.5 django-olympus==0.0.5 From 09af3c9caf0e3fdc6d7fe6e6a2b1532957e5227c Mon Sep 17 00:00:00 2001 From: Duarte Duarte Date: Tue, 16 Jan 2024 16:54:41 +0000 Subject: [PATCH 3/4] core_utils: deprecate make_dt_aware replace by django.utils.timezone.make_aware django is deprecating pytz --- surface/core_utils/admin_filters.py | 48 ++++++++++------------------- 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/surface/core_utils/admin_filters.py b/surface/core_utils/admin_filters.py index 503b1245..8147f74f 100644 --- a/surface/core_utils/admin_filters.py +++ b/surface/core_utils/admin_filters.py @@ -1,39 +1,34 @@ import datetime -from urllib.parse import urlencode import operator -import pytz +from urllib.parse import urlencode +from django import forms +from django.contrib import admin from django.contrib.admin.filters import RelatedFieldListFilter from django.contrib.admin.options import ModelAdmin +from django.contrib.admin.widgets import AdminDateWidget from django.core.exceptions import ImproperlyConfigured -from django.db.models import Model -from django.core.handlers.wsgi import WSGIRequest -from django.contrib.admin.filters import RelatedFieldListFilter -from django.contrib.admin.options import ModelAdmin from django.core.handlers.wsgi import WSGIRequest +from django.db.models import Model from django.db.models.base import Model from django.db.models.fields import BLANK_CHOICE_DASH, Field from django.db.models.fields.related import RelatedField -from django.contrib import admin from django.shortcuts import redirect from django.utils import timezone -from django.conf import settings -from django import forms -from django.contrib.admin.widgets import AdminDateWidget try: from theme.filters import DateRangeFilter as OriginalDateRangeFilter class CalendarFilter(OriginalDateRangeFilter): def __init__(self, field, request, params, model, model_admin, field_path): - self.lookup_kwarg_within = f'{field_path}__within' + self.lookup_kwarg_within = f"{field_path}__within" super().__init__(field, request, params, model, model_admin, field_path) def _get_form_fields(self): return { self.lookup_kwarg_within: forms.DateField( - label='', - widget=AdminDateWidget(attrs={'placeholder': self.field_path.replace("_", " ").title()}), + label="", + widget=AdminDateWidget(attrs={"placeholder": self.field_path.replace("_", " ").title()}), localize=True, required=False, ) @@ -47,26 +42,17 @@ def _make_query_filter(self, request, validated_data): date_value = validated_data.get(self.lookup_kwarg_within, None) if date_value: - date_gte = self.make_dt_aware( + date_gte = timezone.make_aware( datetime.datetime.combine(date_value, datetime.time.min), self.get_timezone(request) ) - query_params[f'{self.field_path}__gte'] = date_gte - query_params[f'{self.field_path}__lt'] = date_gte + datetime.timedelta(days=1) + query_params[f"{self.field_path}__gte"] = date_gte + query_params[f"{self.field_path}__lt"] = date_gte + datetime.timedelta(days=1) return query_params def get_timezone(self, request): return timezone.get_default_timezone() - @staticmethod - def make_dt_aware(value, timezone): - if settings.USE_TZ and pytz is not None: - default_tz = timezone - if value.tzinfo is not None: - value = default_tz.normalize(value) - else: - value = default_tz.localize(value) - return value except ImportError: pass @@ -83,7 +69,7 @@ def __init__( field_path: str, ) -> None: # validate select_related is defined now for early errors - if not hasattr(model_admin, 'list_filter_select_related'): + if not hasattr(model_admin, "list_filter_select_related"): raise ImproperlyConfigured( "The list filter '%s' requires '%s' to define 'list_filter_select_related'." % ( @@ -113,7 +99,7 @@ def _custom_field_get_choices( if field.choices is not None: choices = list(field.choices) if include_blank: - blank_defined = any(choice in ('', None) for choice, _ in field.flatchoices) + blank_defined = any(choice in ("", None) for choice, _ in field.flatchoices) if not blank_defined: choices = blank_choice + choices return choices @@ -130,7 +116,7 @@ def field_choices( # re-implement Field get_choices as it doesn't return the queryset itself, but the strings already built... ordering = self.field_admin_ordering(field, request, model_admin) choice_func = operator.attrgetter( - field.remote_field.get_related_field().attname if hasattr(field.remote_field, 'get_related_field') else 'pk' + field.remote_field.get_related_field().attname if hasattr(field.remote_field, "get_related_field") else "pk" ) qs = self._custom_field_get_choices(field, include_blank=False, ordering=ordering).select_related( *model_admin.list_filter_select_related[field.name] @@ -152,14 +138,14 @@ def get_default_filters(self, request): raise NotImplementedError() def changelist_view(self, request, extra_context=None): - ref = request.META.get('HTTP_REFERER', '') - path = request.META.get('PATH_INFO', '') + ref = request.META.get("HTTP_REFERER", "") + path = request.META.get("PATH_INFO", "") # if any existing GET parameters, do not apply default filters # if referrer comes from changelist itself (or an action such as /add/), do not apply default filters if request.GET or path in ref: return super().changelist_view(request, extra_context=extra_context) query = urlencode(self.get_default_filters(request)) - return redirect(f'{path}?{query}') + return redirect(f"{path}?{query}") def custom_titled_filter(title): From 1d190de5ed5a7dcea02ae500e922c7d99de6eb7f Mon Sep 17 00:00:00 2001 From: Duarte Duarte Date: Tue, 16 Jan 2024 17:00:45 +0000 Subject: [PATCH 4/4] format --- surface/core_utils/admin_filters.py | 1 - 1 file changed, 1 deletion(-) diff --git a/surface/core_utils/admin_filters.py b/surface/core_utils/admin_filters.py index 8147f74f..9070c133 100644 --- a/surface/core_utils/admin_filters.py +++ b/surface/core_utils/admin_filters.py @@ -53,7 +53,6 @@ def _make_query_filter(self, request, validated_data): def get_timezone(self, request): return timezone.get_default_timezone() - except ImportError: pass