From a8f57ccc39efb6e6774be951e1e288b805ae29c3 Mon Sep 17 00:00:00 2001 From: JgtAman <41839176+DragnEmperor@users.noreply.github.com> Date: Mon, 30 Jun 2025 23:56:49 +0530 Subject: [PATCH 1/5] [feature] Base logic for WHOIS lookup feature #1032 #1033 #1037 #1045 Closes #1032 Closes #1033 Closes #1037 Closes #1045 Signed-off-by: DragnEmperor Co-authored-by: Federico Capoano --- docs/developer/extending.rst | 1 + docs/index.rst | 1 + docs/user/settings.rst | 51 ++ docs/user/whois.rst | 80 +++ openwisp_controller/config/admin.py | 31 +- openwisp_controller/config/apps.py | 2 + openwisp_controller/config/base/device.py | 28 +- .../config/base/multitenancy.py | 20 + openwisp_controller/config/base/whois.py | 115 ++++ .../config/controller/views.py | 9 + .../config/migrations/0060_whoisinfo.py | 83 +++ openwisp_controller/config/models.py | 11 + openwisp_controller/config/settings.py | 10 + .../config/tests/test_device.py | 2 + openwisp_controller/config/whois/__init__.py | 0 openwisp_controller/config/whois/handlers.py | 29 + openwisp_controller/config/whois/service.py | 118 +++++ openwisp_controller/config/whois/tasks.py | 160 ++++++ .../config/whois/test_whois.py | 501 ++++++++++++++++++ openwisp_controller/config/whois/utils.py | 36 ++ requirements.txt | 1 + ...rganizationconfigsettings_whois_enabled.py | 79 +++ tests/openwisp2/sample_config/models.py | 10 + tests/openwisp2/settings.py | 9 + 24 files changed, 1376 insertions(+), 11 deletions(-) create mode 100644 docs/user/whois.rst create mode 100644 openwisp_controller/config/base/whois.py create mode 100644 openwisp_controller/config/migrations/0060_whoisinfo.py create mode 100644 openwisp_controller/config/whois/__init__.py create mode 100644 openwisp_controller/config/whois/handlers.py create mode 100644 openwisp_controller/config/whois/service.py create mode 100644 openwisp_controller/config/whois/tasks.py create mode 100644 openwisp_controller/config/whois/test_whois.py create mode 100644 openwisp_controller/config/whois/utils.py create mode 100644 tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py diff --git a/docs/developer/extending.rst b/docs/developer/extending.rst index eb29af94a..eda5c0b05 100644 --- a/docs/developer/extending.rst +++ b/docs/developer/extending.rst @@ -344,6 +344,7 @@ Once you have created the models, add the following to your CONFIG_VPNCLIENT_MODEL = "sample_config.VpnClient" CONFIG_ORGANIZATIONCONFIGSETTINGS_MODEL = "sample_config.OrganizationConfigSettings" CONFIG_ORGANIZATIONLIMITS_MODEL = "sample_config.OrganizationLimits" + CONFIG_WHOISINFO_MODEL = "sample_config.WHOISInfo" DJANGO_X509_CA_MODEL = "sample_pki.Ca" DJANGO_X509_CERT_MODEL = "sample_pki.Cert" GEO_LOCATION_MODEL = "sample_geo.Location" diff --git a/docs/index.rst b/docs/index.rst index 3372e6eae..6895e1c48 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -48,6 +48,7 @@ the OpenWISP architecture. user/zerotier.rst user/openvpn.rst user/subnet-division-rules.rst + user/whois.rst user/rest-api.rst user/settings.rst diff --git a/docs/user/settings.rst b/docs/user/settings.rst index ee32e4875..948b82ec8 100644 --- a/docs/user/settings.rst +++ b/docs/user/settings.rst @@ -761,3 +761,54 @@ recoverable failures, improving the reliability of the system. For more information on these settings, you can refer to the `the celery documentation regarding automatic retries for known errors. `_ + +.. _openwisp_controller_whois_enabled: + +``OPENWISP_CONTROLLER_WHOIS_ENABLED`` +------------------------------------- + +============ ========= +**type**: ``bool`` +**default**: ``False`` +============ ========= + +Allows enabling the optional :doc:`WHOIS Lookup feature `. + +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.2/whois-admin-setting.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.2/whois-admin-setting.png + :alt: WHOIS admin setting + +After enabling this feature, you have to set +:ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT +` and +:ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY +`. + +.. warning:: + + If these three settings are not configured as expected, an + ``ImproperlyConfigured`` exception will be raised. + +.. _openwisp_controller_whois_geoip_account: + +``OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT`` +------------------------------------------- + +============ ======= +**type**: ``str`` +**default**: None +============ ======= + +Maxmind Account ID required for the :doc:`WHOIS Lookup feature `. + +.. _openwisp_controller_whois_geoip_key: + +``OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY`` +--------------------------------------- + +============ ======= +**type**: ``str`` +**default**: None +============ ======= + +Maxmind License Key required for the :doc:`WHOIS Lookup feature `. diff --git a/docs/user/whois.rst b/docs/user/whois.rst new file mode 100644 index 000000000..3039569a4 --- /dev/null +++ b/docs/user/whois.rst @@ -0,0 +1,80 @@ +WHOIS Lookup +============ + +.. important:: + + The **WHOIS Lookup** feature is **disabled by default**. + + To enable it, follow the `setup steps + `_ below. + +.. contents:: **Table of contents**: + :depth: 1 + :local: + +Overview +-------- + +The WHOIS Lookup feature displays information about the public IP address +used by devices to communicate with OpenWISP (via the ``last_ip`` field). +It helps identify the geographic location and ISP associated with the IP +address, which can be useful for troubleshooting network issues. + +The retrieved information pertains to the Autonomous System (ASN) +associated with the device's public IP address and includes: + +- ASN (Autonomous System Number) +- Organization name that owns the ASN +- CIDR block assigned to the ASN +- Physical address registered to the ASN +- Timezone of the ASN's registered location + +Trigger Conditions +------------------ + +A WHOIS lookup is triggered automatically when: + +- A new device is registered. +- A device fetches its checksum. + +However, the lookup will only run if **all** the following conditions are +met: + +- The device's last IP address is **public**. +- There is **no existing WHOIS record** for that IP. +- WHOIS lookup is **enabled** for the device's organization. + +Behavior with Shared IP Addresses +--------------------------------- + +If multiple devices share the same public IP address and one of them +switches to a different IP, the following occurs: + +- A lookup is triggered for the **new IP**. +- The WHOIS record for the **old IP** is deleted. +- The next time a device still using the old IP fetches its checksum, a + new lookup is triggered, ensuring up-to-date data. + +.. note:: + + When a device with an associated WHOIS record is deleted, its WHOIS + record is automatically removed. + +.. _controller_setup_whois_lookup: + +Setup Instructions +------------------ + +1. Create a MaxMind account: `Sign up here + `_. + + If you already have an account, just **Sign In**. + +2. Go to **Manage License Keys** in your MaxMind dashboard. +3. Generate a new license key and name it as you prefer. +4. Copy both the **Account ID** and **License Key**. +5. Set the following settings accordingly: + + - Set :ref:`OPENWISP_CONTROLLER_WHOIS_ENABLED` to ``True``. + - Set :ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT` to **Account ID**. + - Set :ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY` to **License Key**. diff --git a/openwisp_controller/config/admin.py b/openwisp_controller/config/admin.py index 9c758fd06..4ba8ea83b 100644 --- a/openwisp_controller/config/admin.py +++ b/openwisp_controller/config/admin.py @@ -1362,18 +1362,29 @@ def has_delete_permission(self, request, obj): limits_inline_position = 0 -if getattr(app_settings, "REGISTRATION_ENABLED", True): - class ConfigSettingsForm(AlwaysHasChangedMixin, forms.ModelForm): - class Meta: - widgets = {"context": FlatJsonWidget} - class ConfigSettingsInline(admin.StackedInline): - model = OrganizationConfigSettings - form = ConfigSettingsForm +class ConfigSettingsForm(AlwaysHasChangedMixin, forms.ModelForm): + class Meta: + widgets = {"context": FlatJsonWidget} + + +class ConfigSettingsInline(admin.StackedInline): + model = OrganizationConfigSettings + form = ConfigSettingsForm + + def get_fields(self, request, obj=None): + fields = [] + if app_settings.REGISTRATION_ENABLED: + fields += ["registration_enabled", "shared_secret"] + if app_settings.WHOIS_CONFIGURED: + fields += ["whois_enabled"] + fields += ["context"] + return fields + - OrganizationAdmin.save_on_top = True - OrganizationAdmin.inlines.insert(0, ConfigSettingsInline) - limits_inline_position = 1 +OrganizationAdmin.save_on_top = True +OrganizationAdmin.inlines.insert(0, ConfigSettingsInline) +limits_inline_position = 1 OrganizationAdmin.inlines.insert(limits_inline_position, OrganizationLimitsInline) diff --git a/openwisp_controller/config/apps.py b/openwisp_controller/config/apps.py index 973b46c63..4354fcd10 100644 --- a/openwisp_controller/config/apps.py +++ b/openwisp_controller/config/apps.py @@ -29,6 +29,7 @@ vpn_peers_changed, vpn_server_modified, ) +from .whois.handlers import connect_whois_handlers # ensure Device.hardware_id field is not flagged as unique # (because it's flagged as unique_together with organization) @@ -50,6 +51,7 @@ def ready(self, *args, **kwargs): self.register_dashboard_charts() self.register_menu_groups() self.notification_cache_update() + connect_whois_handlers() def __setmodels__(self): self.device_model = load_model("config", "Device") diff --git a/openwisp_controller/config/base/device.py b/openwisp_controller/config/base/device.py index 763f25ecc..1921d5ad3 100644 --- a/openwisp_controller/config/base/device.py +++ b/openwisp_controller/config/base/device.py @@ -1,3 +1,4 @@ +from functools import cached_property from hashlib import md5 from django.core.exceptions import ObjectDoesNotExist, PermissionDenied, ValidationError @@ -18,6 +19,7 @@ management_ip_changed, ) from ..validators import device_name_validator, mac_address_validator +from ..whois.service import WHOISService from .base import BaseModel @@ -118,6 +120,11 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + # Initial value for last_ip is required in WHOIS + # to remove WHOIS info related to that ip address. + if app_settings.WHOIS_CONFIGURED: + self._changed_checked_fields.append("last_ip") + self._set_initial_values_for_changed_checked_fields() def _set_initial_values_for_changed_checked_fields(self): @@ -279,6 +286,8 @@ def save(self, *args, **kwargs): self.key = self.generate_key(shared_secret) state_adding = self._state.adding super().save(*args, **kwargs) + if app_settings.WHOIS_CONFIGURED: + self._check_last_ip() if state_adding and self.group and self.group.templates.exists(): self.create_default_config() # The value of "self._state.adding" will always be "False" @@ -299,7 +308,9 @@ def _check_changed_fields(self): self._get_initial_values_for_checked_fields() # Execute method for checked for each field in self._changed_checked_fields for field in self._changed_checked_fields: - getattr(self, f"_check_{field}_changed")() + method = getattr(self, f"_check_{field}_changed", None) + if callable(method): + method() def _is_deferred(self, field): """ @@ -509,3 +520,18 @@ def config_deactivated_clear_management_ip(cls, instance, *args, **kwargs): is changed to 'deactivated'. """ cls.objects.filter(pk=instance.device_id).update(management_ip="") + + @cached_property + def whois_service(self): + """ + Used as a shortcut to get WHOISService instance + for the device. + """ + return WHOISService(self) + + def _check_last_ip(self): + """Trigger WHOIS lookup if last_ip is not deferred.""" + if self._initial_last_ip == models.DEFERRED: + return + self.whois_service.trigger_whois_lookup() + self._initial_last_ip = self.last_ip diff --git a/openwisp_controller/config/base/multitenancy.py b/openwisp_controller/config/base/multitenancy.py index 9cdf15736..3095ab779 100644 --- a/openwisp_controller/config/base/multitenancy.py +++ b/openwisp_controller/config/base/multitenancy.py @@ -2,12 +2,15 @@ from copy import deepcopy import swapper +from django.core.exceptions import ValidationError from django.db import models from django.utils.translation import gettext_lazy as _ from jsonfield import JSONField from openwisp_utils.base import KeyField, UUIDModel +from openwisp_utils.fields import FallbackBooleanChoiceField +from .. import settings as app_settings from ..exceptions import OrganizationDeviceLimitExceeded from ..tasks import bulk_invalidate_config_get_cached_checksum @@ -31,6 +34,11 @@ class AbstractOrganizationConfigSettings(UUIDModel): verbose_name=_("shared secret"), help_text=_("used for automatic registration of devices"), ) + whois_enabled = FallbackBooleanChoiceField( + help_text=_("Whether the WHOIS lookup feature is enabled"), + fallback=app_settings.WHOIS_ENABLED, + verbose_name=_("WHOIS Enabled"), + ) context = JSONField( blank=True, default=dict, @@ -53,6 +61,18 @@ def __str__(self): def get_context(self): return deepcopy(self.context) + def clean(self): + if not app_settings.WHOIS_CONFIGURED and self.whois_enabled: + raise ValidationError( + { + "whois_enabled": _( + "WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY must be set " + + "before enabling WHOIS feature." + ) + } + ) + return super().clean() + def save( self, force_insert=False, force_update=False, using=None, update_fields=None ): diff --git a/openwisp_controller/config/base/whois.py b/openwisp_controller/config/base/whois.py new file mode 100644 index 000000000..80af038f2 --- /dev/null +++ b/openwisp_controller/config/base/whois.py @@ -0,0 +1,115 @@ +from ipaddress import ip_address, ip_network + +from django.core.cache import cache +from django.core.exceptions import ValidationError +from django.db import models, transaction +from django.utils.translation import gettext_lazy as _ +from jsonfield import JSONField + +from openwisp_utils.base import TimeStampedEditableModel + +from ..whois.service import WHOISService +from ..whois.tasks import delete_whois_record + + +class AbstractWHOISInfo(TimeStampedEditableModel): + """ + Abstract model to store WHOIS information + for a device. + """ + + id = None + # Using ip_address as primary key to avoid redundant lookups + # and storage of duplicate WHOIS information per IP address. + # Whenever a device's last ip address changes, data related to + # previous IP address is deleted. If any device still has the + # previous IP address, they will trigger the lookup again + # ensuring latest WHOIS information is always available. + ip_address = models.GenericIPAddressField(db_index=True, primary_key=True) + isp = models.CharField( + max_length=100, + blank=True, + help_text=_("Organization for ASN"), + ) + asn = models.CharField( + max_length=6, + blank=True, + help_text=_("ASN"), + ) + timezone = models.CharField( + max_length=35, + blank=True, + help_text=_("Time zone"), + ) + address = JSONField( + default=dict, + help_text=_("Address"), + blank=True, + ) + cidr = models.CharField( + max_length=20, + blank=True, + help_text=_("CIDR"), + ) + + class Meta: + abstract = True + + def clean(self): + if ip_address(self.ip_address).is_private: + raise ValidationError( + { + "ip_address": _( + "WHOIS information cannot be created for private IP addresses." + ) + } + ) + if self.cidr: + try: + # strict is set to False to allow CIDR without a mask + # e.g. 192.168.1.12/24 with strict False normalizes to + # 192.168.1.0/24 else it would raise an error. + ip_network(self.cidr, strict=False) + except ValueError as e: + raise ValidationError( + {"cidr": _("Invalid CIDR format: %(error)s") % {"error": str(e)}} + ) + return super().clean() + + @staticmethod + def device_whois_info_delete_handler(instance, **kwargs): + """ + Delete WHOIS information for a device when the last IP address is removed or + when device is deleted. + """ + if instance._get_organization__config_settings().whois_enabled: + transaction.on_commit(lambda: delete_whois_record.delay(instance.last_ip)) + + # this method is kept here instead of in OrganizationConfigSettings because + # currently the caching is used only for WHOIS feature + @staticmethod + def invalidate_org_settings_cache(instance, **kwargs): + """ + Invalidate the cache for Organization settings on update/delete of + Organization settings instance. + """ + org_id = instance.organization_id + cache.delete(WHOISService.get_cache_key(org_id)) + + @property + def formatted_address(self): + """ + Used as default formatter for address field. + 'filter' is used to remove any None values + """ + return ", ".join( + filter( + None, + [ + self.address.get("city"), + self.address.get("country"), + self.address.get("continent"), + self.address.get("postal"), + ], + ) + ) diff --git a/openwisp_controller/config/controller/views.py b/openwisp_controller/config/controller/views.py index 3dc714c0c..68023ea6d 100644 --- a/openwisp_controller/config/controller/views.py +++ b/openwisp_controller/config/controller/views.py @@ -153,6 +153,15 @@ def get(self, request, pk): # updates cache if ip addresses changed if updated: self.update_device_cache(device) + # When update fields are present then save() will run the WHOIS + # lookup. But if there are no update fields, we still want to + # trigger the WHOIS lookup if there is no record for the device's + # last_ip. + elif ( + app_settings.WHOIS_CONFIGURED + and not device.whois_service.get_device_whois_info() + ): + device.whois_service.trigger_whois_lookup() checksum_requested.send( sender=device.__class__, instance=device, request=request ) diff --git a/openwisp_controller/config/migrations/0060_whoisinfo.py b/openwisp_controller/config/migrations/0060_whoisinfo.py new file mode 100644 index 000000000..abacbdb8c --- /dev/null +++ b/openwisp_controller/config/migrations/0060_whoisinfo.py @@ -0,0 +1,83 @@ +# Generated by Django 5.2.1 on 2025-06-26 02:13 + +import django.utils.timezone +import jsonfield.fields +import model_utils.fields +from django.db import migrations, models + +import openwisp_utils.fields + + +class Migration(migrations.Migration): + dependencies = [ + ("config", "0059_zerotier_templates_ow_zt_to_global"), + ] + + operations = [ + migrations.CreateModel( + name="WHOISInfo", + fields=[ + ( + "created", + model_utils.fields.AutoCreatedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="created", + ), + ), + ( + "modified", + model_utils.fields.AutoLastModifiedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="modified", + ), + ), + ( + "ip_address", + models.GenericIPAddressField( + db_index=True, primary_key=True, serialize=False + ), + ), + ( + "asn", + models.CharField(blank=True, help_text="ASN", max_length=6), + ), + ( + "timezone", + models.CharField(blank=True, help_text="Time zone", max_length=35), + ), + ( + "address", + jsonfield.fields.JSONField( + blank=True, default=dict, help_text="Address" + ), + ), + ("cidr", models.CharField(blank=True, help_text="CIDR", max_length=20)), + ( + "isp", + models.CharField( + blank=True, + help_text="Organization for ASN", + max_length=100, + ), + ), + ], + options={ + "abstract": False, + "swappable": "CONFIG_WHOISINFO_MODEL", + }, + ), + migrations.AddField( + model_name="organizationconfigsettings", + name="whois_enabled", + field=openwisp_utils.fields.FallbackBooleanChoiceField( + blank=True, + default=None, + fallback=False, + help_text="Whether the WHOIS lookup feature is enabled", + null=True, + verbose_name="WHOIS Enabled", + ), + ), + ] diff --git a/openwisp_controller/config/models.py b/openwisp_controller/config/models.py index 9795abba8..e36280319 100644 --- a/openwisp_controller/config/models.py +++ b/openwisp_controller/config/models.py @@ -10,6 +10,7 @@ from .base.tag import AbstractTaggedTemplate, AbstractTemplateTag from .base.template import AbstractTemplate from .base.vpn import AbstractVpn, AbstractVpnClient +from .base.whois import AbstractWHOISInfo class Device(AbstractDevice): @@ -111,3 +112,13 @@ class OrganizationLimits(AbstractOrganizationLimits): class Meta(AbstractOrganizationLimits.Meta): abstract = False swappable = swapper.swappable_setting("config", "OrganizationLimits") + + +class WHOISInfo(AbstractWHOISInfo): + """ + Stores WHOIS information for devices. + """ + + class Meta(AbstractWHOISInfo.Meta): + abstract = False + swappable = swapper.swappable_setting("config", "WHOISInfo") diff --git a/openwisp_controller/config/settings.py b/openwisp_controller/config/settings.py index 783bdcd49..a94617b34 100644 --- a/openwisp_controller/config/settings.py +++ b/openwisp_controller/config/settings.py @@ -1,6 +1,7 @@ import logging from django.conf import settings +from django.core.exceptions import ImproperlyConfigured from django.utils.translation import gettext_lazy as _ logger = logging.getLogger(__name__) @@ -67,3 +68,12 @@ def get_setting(option, default): "API_TASK_RETRY_OPTIONS", dict(max_retries=5, retry_backoff=True, retry_backoff_max=600, retry_jitter=True), ) +WHOIS_GEOIP_ACCOUNT = get_setting("WHOIS_GEOIP_ACCOUNT", None) +WHOIS_GEOIP_KEY = get_setting("WHOIS_GEOIP_KEY", None) +WHOIS_ENABLED = get_setting("WHOIS_ENABLED", False) +WHOIS_CONFIGURED = WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY +if WHOIS_ENABLED and not WHOIS_CONFIGURED: + raise ImproperlyConfigured( + "WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY must be set " + + "when WHOIS_ENABLED is True." + ) diff --git a/openwisp_controller/config/tests/test_device.py b/openwisp_controller/config/tests/test_device.py index eb053f237..f40950440 100644 --- a/openwisp_controller/config/tests/test_device.py +++ b/openwisp_controller/config/tests/test_device.py @@ -524,6 +524,8 @@ def test_device_field_changed_checks(self): device.management_ip = "10.0.0.1" device.group_id = device_group.id device.organization_id = self._create_org().id + # assigning a random ip to last_ip + device.last_ip = "172.217.22.14" # Another query is generated due to "config.set_status_modified" # on name change with self.assertNumQueries(3): diff --git a/openwisp_controller/config/whois/__init__.py b/openwisp_controller/config/whois/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/config/whois/handlers.py b/openwisp_controller/config/whois/handlers.py new file mode 100644 index 000000000..6a785c4b6 --- /dev/null +++ b/openwisp_controller/config/whois/handlers.py @@ -0,0 +1,29 @@ +from django.db.models.signals import post_delete, post_save +from swapper import load_model + +from .. import settings as app_settings + + +def connect_whois_handlers(): + if not app_settings.WHOIS_CONFIGURED: + return + + Device = load_model("config", "Device") + WHOISInfo = load_model("config", "WHOISInfo") + OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + + post_delete.connect( + WHOISInfo.device_whois_info_delete_handler, + sender=Device, + dispatch_uid="device.delete_whois_info", + ) + post_save.connect( + WHOISInfo.invalidate_org_settings_cache, + sender=OrganizationConfigSettings, + dispatch_uid="invalidate_org_config_cache_on_org_config_save", + ) + post_delete.connect( + WHOISInfo.invalidate_org_settings_cache, + sender=OrganizationConfigSettings, + dispatch_uid="invalidate_org_config_cache_on_org_config_delete", + ) diff --git a/openwisp_controller/config/whois/service.py b/openwisp_controller/config/whois/service.py new file mode 100644 index 000000000..81275a8e3 --- /dev/null +++ b/openwisp_controller/config/whois/service.py @@ -0,0 +1,118 @@ +from ipaddress import ip_address + +from django.core.cache import cache +from django.db import transaction +from swapper import load_model + +from openwisp_controller.config import settings as app_settings + +from .tasks import fetch_whois_details + + +class WHOISService: + """ + A handler class for managing the WHOIS functionality. + """ + + def __init__(self, device): + self.device = device + + @staticmethod + def get_cache_key(org_id): + """ + Used to get cache key for caching org settings of a device. + """ + return f"organization_config_{org_id}" + + @staticmethod + def is_valid_public_ip_address(ip): + """ + Check if given IP address is a valid public IP address. + """ + try: + return ip and ip_address(ip).is_global + except ValueError: + return False + + @staticmethod + def _get_whois_info_from_db(ip_address): + """ + For getting existing WHOISInfo for given IP from db if present. + """ + WHOISInfo = load_model("config", "WHOISInfo") + + return WHOISInfo.objects.filter(ip_address=ip_address) + + @property + def is_whois_enabled(self): + """ + Check if the WHOIS lookup feature is enabled. + The OrganizationConfigSettings are cached as these settings + are not expected to change frequently. The timeout for the cache + is set to the same as the checksum cache timeout for consistency + with DeviceChecksumView. + """ + OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + Config = load_model("config", "Config") + + org_id = self.device.organization.pk + org_settings = cache.get(self.get_cache_key(org_id=org_id)) + if org_settings is None: + try: + org_settings = OrganizationConfigSettings.objects.get( + organization=org_id + ) + except OrganizationConfigSettings.DoesNotExist: + # If organization settings do not exist, fall back to global setting + return app_settings.WHOIS_ENABLED + cache.set( + self.get_cache_key(org_id=org_id), + org_settings, + timeout=Config._CHECKSUM_CACHE_TIMEOUT, + ) + return getattr(org_settings, "whois_enabled", app_settings.WHOIS_ENABLED) + + def _need_whois_lookup(self, new_ip): + """ + This is used to determine if the WHOIS lookup should be triggered + when the device is saved. + + The lookup is not triggered if: + - The new IP address is None or it is a private IP address. + - The WHOIS information of new ip is already present. + - WHOIS is disabled in the organization settings. (query from db) + """ + + # Check cheap conditions first before hitting the database + if not self.is_valid_public_ip_address(new_ip): + return False + + if self._get_whois_info_from_db(new_ip).exists(): + return False + + return self.is_whois_enabled + + def get_device_whois_info(self): + """ + If the WHOIS lookup feature is enabled and the device ``last_ip`` + is a public IP, it fetches WHOIS information for the network device. + """ + ip_address = self.device.last_ip + if not (self.is_valid_public_ip_address(ip_address) and self.is_whois_enabled): + return None + + return self._get_whois_info_from_db(ip_address=ip_address).first() + + def trigger_whois_lookup(self): + """ + Trigger WHOIS lookup based on the conditions of `_need_whois_lookup`. + Task is triggered on commit to ensure redundant data is not created. + """ + if self._need_whois_lookup(self.device.last_ip): + transaction.on_commit( + lambda: fetch_whois_details.delay( + device_pk=self.device.pk, + initial_ip_address=self.device._initial_last_ip, + new_ip_address=self.device.last_ip, + ) + ) diff --git a/openwisp_controller/config/whois/tasks.py b/openwisp_controller/config/whois/tasks.py new file mode 100644 index 000000000..2ae434cd7 --- /dev/null +++ b/openwisp_controller/config/whois/tasks.py @@ -0,0 +1,160 @@ +import logging + +import requests +from celery import shared_task +from django.utils.translation import gettext as _ +from geoip2 import errors +from geoip2 import webservice as geoip2_webservice +from openwisp_notifications.signals import notify +from swapper import load_model + +from openwisp_utils.tasks import OpenwispCeleryTask + +from .. import settings as app_settings + +logger = logging.getLogger(__name__) + +EXCEPTION_MESSAGES = { + errors.AddressNotFoundError: _( + "No WHOIS information found for IP address {ip_address}" + ), + errors.AuthenticationError: _( + "Authentication failed for GeoIP2 service. " + "Check your OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT and " + "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY settings." + ), + errors.OutOfQueriesError: _( + "Your account has run out of queries for the GeoIP2 service." + ), + errors.PermissionRequiredError: _( + "Your account does not have permission to access this service." + ), +} + + +class WHOISCeleryRetryTask(OpenwispCeleryTask): + """ + Base class for OpenWISP Celery tasks with retry support on failure. + """ + + # this is the exception related to networking errors + # that should trigger a retry of the task. + autoretry_for = (errors.HTTPError,) + + def on_failure(self, exc, task_id, args, kwargs, einfo): + """Notify the user about the failure of the WHOIS task.""" + Device = load_model("config", "Device") + + device_pk = kwargs.get("device_pk") + new_ip_address = kwargs.get("new_ip_address") + device = Device.objects.get(pk=device_pk) + + notify.send( + sender=device, + type="generic_message", + target=device, + action_object=device, + level="error", + message=_( + "Failed to fetch WHOIS details for device" + " [{notification.target}]({notification.target_link})" + ), + description=_( + f"WHOIS details could not be fetched for ip: {new_ip_address}." + ), + ) + logger.error(f"WHOIS lookup failed. Details: {exc}") + return super().on_failure(exc, task_id, args, kwargs, einfo) + + +# device_pk is used when task fails to report for which device failure occurred +@shared_task( + bind=True, + base=WHOISCeleryRetryTask, + **app_settings.API_TASK_RETRY_OPTIONS, +) +def fetch_whois_details(self, device_pk, initial_ip_address, new_ip_address): + """ + Fetches the WHOIS details of the given IP address + and creates/updates the WHOIS record. + """ + WHOISInfo = load_model("config", "WHOISInfo") + + # The task can be triggered for same ip address multiple times + # so we need to return early if WHOIS is already created. + if WHOISInfo.objects.filter(ip_address=new_ip_address).exists(): + return + + # Host is based on the db that is used to fetch the details. + # As we are using GeoLite2, 'geolite.info' host is used. + # Refer: https://geoip2.readthedocs.io/en/latest/#sync-web-service-example + ip_client = geoip2_webservice.Client( + account_id=app_settings.WHOIS_GEOIP_ACCOUNT, + license_key=app_settings.WHOIS_GEOIP_KEY, + host="geolite.info", + ) + + try: + data = ip_client.city(ip_address=new_ip_address) + + # Catching all possible exceptions raised by the geoip2 client + # and raising them with appropriate messages to be handled by the task + # retry mechanism. + except ( + errors.AddressNotFoundError, + errors.AuthenticationError, + errors.OutOfQueriesError, + errors.PermissionRequiredError, + ) as e: + exc_type = type(e) + message = EXCEPTION_MESSAGES.get(exc_type) + if exc_type is errors.AddressNotFoundError: + message = message.format(ip_address=new_ip_address) + raise exc_type(message) + except requests.RequestException as e: + raise e + + else: + # The attributes are always present in the response, + # but they can be None, so added fallbacks. + address = { + "city": data.city.name or "", + "country": data.country.name or "", + "continent": data.continent.name or "", + "postal": str(data.postal.code or ""), + } + + whois_obj = WHOISInfo( + isp=data.traits.autonomous_system_organization, + asn=data.traits.autonomous_system_number, + timezone=data.location.time_zone, + address=address, + cidr=data.traits.network, + ip_address=new_ip_address, + ) + whois_obj.full_clean() + whois_obj.save() + logger.info(f"Successfully fetched WHOIS details for {new_ip_address}.") + + # the following check ensures that for a case when device last_ip + # is not changed and there is no related WHOIS record, we do not + # delete the newly created record as both `initial_ip_address` and + # `new_ip_address` would be same for such case. + if initial_ip_address != new_ip_address: + # If any active devices are linked to the following record, + # then they will trigger this task and new record gets created + # with latest data. + delete_whois_record(ip_address=initial_ip_address) + + +@shared_task +def delete_whois_record(ip_address): + """ + Deletes the WHOIS record for the device's last IP address. + This is used when the device is deleted or its last IP address is changed. + """ + WHOISInfo = load_model("config", "WHOISInfo") + + queryset = WHOISInfo.objects.filter(ip_address=ip_address) + if queryset.exists(): + queryset.delete() diff --git a/openwisp_controller/config/whois/test_whois.py b/openwisp_controller/config/whois/test_whois.py new file mode 100644 index 000000000..f5c93b6d3 --- /dev/null +++ b/openwisp_controller/config/whois/test_whois.py @@ -0,0 +1,501 @@ +import importlib +from unittest import mock + +from django.core.exceptions import ImproperlyConfigured, ValidationError +from django.db.models.signals import post_delete, post_save +from django.test import TestCase, TransactionTestCase, override_settings +from django.urls import reverse +from geoip2 import errors +from swapper import load_model + +from ...tests.utils import TestAdminMixin +from .. import settings as app_settings +from .handlers import connect_whois_handlers +from .utils import CreateWHOISMixin + +Device = load_model("config", "Device") +WHOISInfo = load_model("config", "WHOISInfo") +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") +Notification = load_model("openwisp_notifications", "Notification") + +notification_qs = Notification.objects.all() + + +class TestWHOIS(CreateWHOISMixin, TestAdminMixin, TestCase): + # Signals are connected when apps are loaded, + # and if WHOIS is Configured all related WHOIS + # handlers are also connected. Thus we need to + # disconnect them. + def _disconnect_signals(self): + post_delete.disconnect( + WHOISInfo.device_whois_info_delete_handler, + sender=Device, + dispatch_uid="device.delete_whois_info", + ) + post_save.disconnect( + WHOISInfo.invalidate_org_settings_cache, + sender=OrganizationConfigSettings, + dispatch_uid="invalidate_org_config_cache_on_org_config_save", + ) + post_delete.disconnect( + WHOISInfo.invalidate_org_settings_cache, + sender=OrganizationConfigSettings, + dispatch_uid="invalidate_org_config_cache_on_org_config_delete", + ) + + @override_settings( + OPENWISP_CONTROLLER_WHOIS_ENABLED=False, + OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT=None, + OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY=None, + ) + def test_whois_configuration_setting(self): + self._disconnect_signals() + org = self._get_org() + # reload app_settings to apply the overridden settings + importlib.reload(app_settings) + + with self.subTest("Test Signals not connected when WHOIS_CONFIGURED is False"): + # should not connect any handlers since WHOIS_CONFIGURED is False + connect_whois_handlers() + + assert not any( + "device.delete_whois_info" in str(r[0]) for r in post_delete.receivers + ) + assert not any( + "invalidate_org_config_cache_on_org_config_save" in str(r[0]) + for r in post_save.receivers + ) + assert not any( + "invalidate_org_config_cache_on_org_config_delete" in str(r[0]) + for r in post_delete.receivers + ) + + with self.subTest( + "Test WHOIS field hidden on admin when WHOIS_CONFIGURED is False" + ): + self._login() + url = reverse( + "admin:openwisp_users_organization_change", + args=[org.pk], + ) + response = self.client.get(url) + self.assertNotContains(response, 'name="config_settings-0-whois_enabled"') + + with self.subTest( + "Test ImproperlyConfigured raised when WHOIS_CONFIGURED is False" + ): + with override_settings(OPENWISP_CONTROLLER_WHOIS_ENABLED=True): + with self.assertRaises(ImproperlyConfigured): + # reload app_settings to apply the overridden settings + importlib.reload(app_settings) + + with override_settings( + OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT="test_account_id", + OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY="test_license_key", + ): + importlib.reload(app_settings) + with self.subTest( + "Test WHOIS_CONFIGURED is True when both settings are set" + ): + self.assertTrue(app_settings.WHOIS_CONFIGURED) + + with self.subTest("Test Signals connected when WHOIS_CONFIGURED is True"): + connect_whois_handlers() + + assert any( + "device.delete_whois_info" in str(r[0]) + for r in post_delete.receivers + ) + assert any( + "invalidate_org_config_cache_on_org_config_save" in str(r[0]) + for r in post_save.receivers + ) + assert any( + "invalidate_org_config_cache_on_org_config_delete" in str(r[0]) + for r in post_delete.receivers + ) + + with self.subTest( + "Test WHOIS field visible on admin when WHOIS_CONFIGURED is True" + ): + self._login() + url = reverse( + "admin:openwisp_users_organization_change", + args=[org.pk], + ) + response = self.client.get(url) + self.assertContains(response, 'name="config_settings-0-whois_enabled"') + + def test_whois_enabled(self): + OrganizationConfigSettings.objects.all().delete() + device = self._create_device() + with self.subTest( + "Test WHOIS fallback when Organization settings do not exist" + ): + self.assertEqual( + device.whois_service.is_whois_enabled, app_settings.WHOIS_ENABLED + ) + + org_settings_obj = OrganizationConfigSettings( + organization=self._get_org(), whois_enabled=True + ) + + with self.subTest("Test WHOIS not configured does not allow enabling WHOIS"): + with mock.patch.object( + app_settings, "WHOIS_CONFIGURED", False + ), self.assertRaises(ValidationError) as context_manager: + org_settings_obj.full_clean() + try: + self.assertEqual( + context_manager.exception.message_dict["whois_enabled"][0], + "WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY must be set " + + "before enabling WHOIS feature.", + ) + except AssertionError: + self.fail("ValidationError message not equal to expected message.") + + with mock.patch.object(app_settings, "WHOIS_CONFIGURED", True): + org_settings_obj.full_clean() + org_settings_obj.save() + + with self.subTest("Test setting WHOIS enabled to True"): + org_settings_obj.whois_enabled = True + org_settings_obj.save(update_fields=["whois_enabled"]) + org_settings_obj.refresh_from_db(fields=["whois_enabled"]) + self.assertEqual(getattr(org_settings_obj, "whois_enabled"), True) + + with self.subTest("Test setting WHOIS enabled to False"): + org_settings_obj.whois_enabled = False + org_settings_obj.save(update_fields=["whois_enabled"]) + org_settings_obj.refresh_from_db(fields=["whois_enabled"]) + self.assertEqual(getattr(org_settings_obj, "whois_enabled"), False) + + with self.subTest( + "Test setting WHOIS enabled to None fallbacks to global setting" + ): + # reload app_settings to ensure latest settings are applied + importlib.reload(app_settings) + org_settings_obj.whois_enabled = None + org_settings_obj.save(update_fields=["whois_enabled"]) + org_settings_obj.refresh_from_db(fields=["whois_enabled"]) + self.assertEqual( + getattr(org_settings_obj, "whois_enabled"), + app_settings.WHOIS_ENABLED, + ) + + +class TestWHOISInfoModel(CreateWHOISMixin, TestCase): + def test_whois_model_fields_validation(self): + """ + Test db_constraints and validators for WHOISInfo model fields. + """ + with self.assertRaises(ValidationError): + self._create_whois_info(isp="a" * 101) + + with self.assertRaises(ValidationError) as context_manager: + self._create_whois_info(ip_address="127.0.0.1") + try: + self.assertEqual( + context_manager.exception.message_dict["ip_address"][0], + "WHOIS information cannot be created for private IP addresses.", + ) + except AssertionError: + self.fail("ValidationError message not equal to expected message.") + + with self.assertRaises(ValidationError): + self._create_whois_info(timezone="a" * 36) + + with self.assertRaises(ValidationError) as context_manager: + self._create_whois_info(cidr="InvalidCIDR") + try: + # Not using assertEqual here because we are adding error message raised by + # ipaddress module to the ValidationError message. + self.assertIn( + "Invalid CIDR format: 'InvalidCIDR'", + context_manager.exception.message_dict["cidr"][0], + ) + except AssertionError: + self.fail("ValidationError message not equal to expected message.") + + with self.assertRaises(ValidationError): + self._create_whois_info(asn="InvalidASN") + + +class TestWHOISTransaction(CreateWHOISMixin, TransactionTestCase): + _WHOIS_GEOIP_CLIENT = ( + "openwisp_controller.config.whois.tasks.geoip2_webservice.Client" + ) + _WHOIS_TASKS_INFO_LOGGER = "openwisp_controller.config.whois.tasks.logger.info" + _WHOIS_TASKS_WARN_LOGGER = "openwisp_controller.config.whois.tasks.logger.warning" + _WHOIS_TASKS_ERR_LOGGER = "openwisp_controller.config.whois.tasks.logger.error" + + def setUp(self): + super().setUp() + self.admin = self._get_admin() + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch("openwisp_controller.config.whois.tasks.fetch_whois_details.delay") + def test_whois_task_called(self, mocked_task): + org = self._get_org() + connect_whois_handlers() + + with self.subTest("task called when last_ip is public"): + with mock.patch("django.core.cache.cache.set") as mocked_set: + device = self._create_device(last_ip="172.217.22.14") + mocked_task.assert_called() + mocked_set.assert_called_once() + mocked_task.reset_mock() + + with self.subTest("task called when last_ip is changed and is public"): + with mock.patch("django.core.cache.cache.get") as mocked_get: + device.last_ip = "172.217.22.10" + device.save() + mocked_task.assert_called() + mocked_get.assert_called_once() + mocked_task.reset_mock() + + with self.subTest("task not called when last_ip is private"): + device.last_ip = "10.0.0.1" + device.save() + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest("task not called when last_ip has related WHOISInfo"): + device.last_ip = "172.217.22.10" + self._create_whois_info(ip_address=device.last_ip) + device.save() + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest("task not called when WHOIS is disabled"): + Device.objects.all().delete() + org.config_settings.whois_enabled = False + # Invalidates old org config settings cache + org.config_settings.save(update_fields=["whois_enabled"]) + device = self._create_device(last_ip="172.217.22.14") + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest("task called via DeviceChecksumView when WHOIS is enabled"): + org.config_settings.whois_enabled = True + # Invalidates old org config settings cache + org.config_settings.save(update_fields=["whois_enabled"]) + # config is required for checksum view to work + self._create_config(device=device) + # setting remote address field to a public IP to trigger WHOIS task + # since the view uses this header for tracking the device's IP + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR="172.217.22.10", + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_called() + mocked_task.reset_mock() + + with self.subTest( + "task called via DeviceChecksumView when a device has no WHOIS record" + ): + WHOISInfo.objects.all().delete() + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_called() + mocked_task.reset_mock() + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch("openwisp_controller.config.whois.tasks.fetch_whois_details.delay") + def test_whois_multiple_orgs(self, mocked_task): + org2 = self._create_org(name="test org2", slug="test-org2") + OrganizationConfigSettings.objects.create( + organization=org2, whois_enabled=False + ) + + with self.subTest("Test task calls when device created with public last_ip"): + device1 = self._create_device( + last_ip="172.217.22.10", organization=self._get_org() + ) + mocked_task.assert_called() + mocked_task.reset_mock() + device2 = self._create_device(last_ip="172.217.22.11", organization=org2) + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest("Test task calls when last_ip is changed and is public"): + device1.last_ip = "172.217.22.12" + device1.save() + mocked_task.assert_called() + mocked_task.reset_mock() + device2.last_ip = "172.217.22.13" + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest("Test fetching WHOIS details"): + whois_obj1 = self._create_whois_info(ip_address=device1.last_ip) + self._create_whois_info(ip_address=device2.last_ip) + self.assertEqual(whois_obj1, device1.whois_service.get_device_whois_info()) + self.assertIsNone(device2.whois_service.get_device_whois_info()) + + with self.subTest("Test task calls in DeviceChecksumView when last_ip changes"): + # config is required for checksum view to work + self._create_config(device=device1) + self._create_config(device=device2) + response = self.client.get( + reverse("controller:device_checksum", args=[device1.pk]), + {"key": device1.key}, + REMOTE_ADDR="172.217.22.20", + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_called() + mocked_task.reset_mock() + response = self.client.get( + reverse("controller:device_checksum", args=[device2.pk]), + {"key": device2.key}, + REMOTE_ADDR="172.217.22.30", + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest( + "task called via DeviceChecksumView when a device has no WHOIS record" + ): + WHOISInfo.objects.all().delete() + response = self.client.get( + reverse("controller:device_checksum", args=[device1.pk]), + {"key": device1.key}, + REMOTE_ADDR=device1.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_called() + mocked_task.reset_mock() + response = self.client.get( + reverse("controller:device_checksum", args=[device2.pk]), + {"key": device2.key}, + REMOTE_ADDR=device2.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_not_called() + mocked_task.reset_mock() + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_TASKS_INFO_LOGGER) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_whois_creation(self, mock_client, mock_info): + # helper function for asserting the model details with + # mocked api response + connect_whois_handlers() + + def _verify_whois_details(instance, ip_address): + self.assertEqual(instance.isp, "Google LLC") + self.assertEqual(instance.asn, "15169") + self.assertEqual(instance.timezone, "America/Los_Angeles") + self.assertEqual( + instance.address, + { + "city": "Mountain View", + "country": "United States", + "continent": "North America", + "postal": "94043", + }, + ) + self.assertEqual(instance.cidr, "172.217.22.0/24") + self.assertEqual(instance.ip_address, ip_address) + self.assertEqual( + instance.formatted_address, + "Mountain View, United States, North America, 94043", + ) + + # mocking the response from the geoip2 client + mock_response = mock.MagicMock() + mock_response.city.name = "Mountain View" + mock_response.country.name = "United States" + mock_response.continent.name = "North America" + mock_response.postal.code = "94043" + mock_response.traits.autonomous_system_organization = "Google LLC" + mock_response.traits.autonomous_system_number = 15169 + mock_response.traits.network = "172.217.22.0/24" + mock_response.location.time_zone = "America/Los_Angeles" + mock_client.return_value.city.return_value = mock_response + + with self.subTest("Test WHOIS create when device is created"): + device = self._create_device(last_ip="172.217.22.14") + self.assertEqual(mock_info.call_count, 1) + mock_info.reset_mock() + device.refresh_from_db() + + _verify_whois_details( + device.whois_service.get_device_whois_info(), device.last_ip + ) + + with self.subTest( + "Test WHOIS create & deletion of old record when last ip is updated" + ): + old_ip_address = device.last_ip + device.last_ip = "172.217.22.10" + device.save() + self.assertEqual(mock_info.call_count, 1) + mock_info.reset_mock() + device.refresh_from_db() + + _verify_whois_details( + device.whois_service.get_device_whois_info(), device.last_ip + ) + + # details related to old ip address should be deleted + self.assertEqual( + WHOISInfo.objects.filter(ip_address=old_ip_address).count(), 0 + ) + + with self.subTest("Test WHOIS delete when device is deleted"): + ip_address = device.last_ip + device.delete(check_deactivated=False) + self.assertEqual(mock_info.call_count, 0) + mock_info.reset_mock() + + # WHOIS related to the device's last_ip should be deleted + self.assertEqual(WHOISInfo.objects.filter(ip_address=ip_address).count(), 0) + + # we need to allow the task to propagate exceptions to ensure + # `on_failure` method is called and notifications are executed + @override_settings(CELERY_TASK_EAGER_PROPAGATES=False) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_TASKS_ERR_LOGGER) + @mock.patch(_WHOIS_TASKS_WARN_LOGGER) + @mock.patch(_WHOIS_TASKS_INFO_LOGGER) + def test_whois_task_failure_notification(self, mock_info, mock_warn, mock_error): + def assert_logging_on_exception( + exception, info_calls=0, warn_calls=0, error_calls=1 + ): + with self.subTest( + f"Test notifications and logging when {exception.__name__} is raised" + ), mock.patch(self._WHOIS_GEOIP_CLIENT, side_effect=exception("test")): + Device.objects.all().delete() # Clear existing devices + device = self._create_device(last_ip="172.217.22.14") + self.assertEqual(mock_info.call_count, info_calls) + self.assertEqual(mock_warn.call_count, warn_calls) + self.assertEqual(mock_error.call_count, error_calls) + self.assertEqual(notification_qs.count(), 1) + notification = notification_qs.first() + self.assertEqual(notification.actor, device) + self.assertEqual(notification.target, device) + self.assertEqual(notification.type, "generic_message") + self.assertIn( + "Failed to fetch WHOIS details for device", + notification.message, + ) + self.assertIn(device.last_ip, notification.description) + + mock_info.reset_mock() + mock_warn.reset_mock() + mock_error.reset_mock() + notification_qs.delete() + + # Test for all possible exceptions that can be raised by the geoip2 client + assert_logging_on_exception(errors.OutOfQueriesError) + assert_logging_on_exception(errors.AddressNotFoundError) + assert_logging_on_exception(errors.AuthenticationError) + assert_logging_on_exception(errors.PermissionRequiredError) diff --git a/openwisp_controller/config/whois/utils.py b/openwisp_controller/config/whois/utils.py new file mode 100644 index 000000000..fd31ff6cf --- /dev/null +++ b/openwisp_controller/config/whois/utils.py @@ -0,0 +1,36 @@ +from swapper import load_model + +from ..tests.utils import CreateConfigMixin + +Device = load_model("config", "Device") +WHOISInfo = load_model("config", "WHOISInfo") +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + + +class CreateWHOISMixin(CreateConfigMixin): + def _create_whois_info(self, **kwargs): + options = dict( + ip_address="172.217.22.14", + address={ + "city": "Mountain View", + "country": "United States", + "continent": "North America", + "postal": "94043", + }, + asn="15169", + isp="Google LLC", + timezone="America/Los_Angeles", + cidr="172.217.22.0/24", + ) + + options.update(kwargs) + w = WHOISInfo(**options) + w.full_clean() + w.save() + return w + + def setUp(self): + super().setUp() + OrganizationConfigSettings.objects.create( + organization=self._get_org(), whois_enabled=True + ) diff --git a/requirements.txt b/requirements.txt index 9775ceb98..5a8d14148 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,4 @@ shortuuid~=1.0.13 netaddr~=1.3.0 django-import-export~=4.3.14 jsonfield>=3.1.0,<4.0.0 +geoip2~=5.1.0 diff --git a/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py b/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py new file mode 100644 index 000000000..707a44bf5 --- /dev/null +++ b/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py @@ -0,0 +1,79 @@ +# Generated by Django 5.2.1 on 2025-06-26 17:48 + +import django.utils.timezone +import jsonfield.fields +import model_utils.fields +from django.db import migrations, models + +import openwisp_utils.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("sample_config", "0007_alter_config_status"), + ] + + operations = [ + migrations.CreateModel( + name="WHOISInfo", + fields=[ + ( + "created", + model_utils.fields.AutoCreatedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="created", + ), + ), + ( + "modified", + model_utils.fields.AutoLastModifiedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="modified", + ), + ), + ( + "ip_address", + models.GenericIPAddressField( + db_index=True, primary_key=True, serialize=False + ), + ), + ( + "isp", + models.CharField( + blank=True, help_text="Organization for ASN", max_length=100 + ), + ), + ("asn", models.CharField(blank=True, help_text="ASN", max_length=6)), + ( + "timezone", + models.CharField(blank=True, help_text="Time zone", max_length=35), + ), + ( + "address", + jsonfield.fields.JSONField( + blank=True, default=dict, help_text="Address" + ), + ), + ("cidr", models.CharField(blank=True, help_text="CIDR", max_length=20)), + ("details", models.CharField(blank=True, max_length=64, null=True)), + ], + options={ + "abstract": False, + }, + ), + migrations.AddField( + model_name="organizationconfigsettings", + name="whois_enabled", + field=openwisp_utils.fields.FallbackBooleanChoiceField( + blank=True, + default=None, + fallback=False, + help_text="Whether the WHOIS lookup feature is enabled", + null=True, + verbose_name="WHOIS Enabled", + ), + ), + ] diff --git a/tests/openwisp2/sample_config/models.py b/tests/openwisp2/sample_config/models.py index 8fec929db..e0e6448a5 100644 --- a/tests/openwisp2/sample_config/models.py +++ b/tests/openwisp2/sample_config/models.py @@ -13,6 +13,7 @@ ) from openwisp_controller.config.base.template import AbstractTemplate from openwisp_controller.config.base.vpn import AbstractVpn, AbstractVpnClient +from openwisp_controller.config.base.whois import AbstractWHOISInfo class DetailsModel(models.Model): @@ -111,3 +112,12 @@ class OrganizationLimits(DetailsModel, AbstractOrganizationLimits): class Meta(AbstractOrganizationLimits.Meta): abstract = False + + +class WHOISInfo(DetailsModel, AbstractWHOISInfo): + """ + Stores WHOIS information for devices. + """ + + class Meta(AbstractWHOISInfo.Meta): + abstract = False diff --git a/tests/openwisp2/settings.py b/tests/openwisp2/settings.py index c46e3e50a..ea1a79a56 100644 --- a/tests/openwisp2/settings.py +++ b/tests/openwisp2/settings.py @@ -208,6 +208,14 @@ OPENWISP_CONTROLLER_CONTEXT = {"vpnserver1": "vpn.testdomain.com"} OPENWISP_USERS_AUTH_API = True +# GEOIP Related Settings +OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT = os.getenv( + "OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT", "" +) +OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY = os.getenv( + "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY", "" +) + TEST_RUNNER = "openwisp_utils.tests.TimeLoggingTestRunner" if os.environ.get("SAMPLE_APP", False): @@ -266,6 +274,7 @@ CONFIG_VPNCLIENT_MODEL = "sample_config.VpnClient" CONFIG_ORGANIZATIONCONFIGSETTINGS_MODEL = "sample_config.OrganizationConfigSettings" CONFIG_ORGANIZATIONLIMITS_MODEL = "sample_config.OrganizationLimits" + CONFIG_WHOISINFO_MODEL = "sample_config.WHOISInfo" DJANGO_X509_CA_MODEL = "sample_pki.Ca" DJANGO_X509_CERT_MODEL = "sample_pki.Cert" GEO_LOCATION_MODEL = "sample_geo.Location" From 5e1d398ab348943fd4d1ff45fc7fb08230ac5c94 Mon Sep 17 00:00:00 2001 From: JgtAman <41839176+DragnEmperor@users.noreply.github.com> Date: Tue, 22 Jul 2025 23:25:24 +0530 Subject: [PATCH 2/5] [feature] Added WHOIS details to the UI #1026 Closes #1026 Signed-off-by: DragnEmperor --- docs/user/rest-api.rst | 15 ++ docs/user/settings.rst | 4 +- docs/user/whois.rst | 17 ++ openwisp_controller/config/admin.py | 17 +- openwisp_controller/config/api/serializers.py | 5 +- .../{0060_whoisinfo.py => 0061_whoisinfo.py} | 2 +- .../config/static/whois/css/whois.css | 113 +++++++++++++ .../static/whois/images/whois_globe.svg | 3 + .../config/static/whois/js/whois.js | 56 +++++++ .../templates/admin/config/change_form.html | 3 + openwisp_controller/config/tests/test_api.py | 12 +- openwisp_controller/config/whois/mixins.py | 24 +++ .../config/whois/serializers.py | 28 ++++ .../config/whois/test_whois.py | 151 +++++++++++++++++- .../config/whois/tests_utils.py | 36 +++++ openwisp_controller/config/whois/utils.py | 49 +++--- 16 files changed, 491 insertions(+), 44 deletions(-) rename openwisp_controller/config/migrations/{0060_whoisinfo.py => 0061_whoisinfo.py} (97%) create mode 100644 openwisp_controller/config/static/whois/css/whois.css create mode 100644 openwisp_controller/config/static/whois/images/whois_globe.svg create mode 100644 openwisp_controller/config/static/whois/js/whois.js create mode 100644 openwisp_controller/config/whois/mixins.py create mode 100644 openwisp_controller/config/whois/serializers.py create mode 100644 openwisp_controller/config/whois/tests_utils.py diff --git a/docs/user/rest-api.rst b/docs/user/rest-api.rst index a4e15febe..c58d1ce93 100644 --- a/docs/user/rest-api.rst +++ b/docs/user/rest-api.rst @@ -68,6 +68,14 @@ List Devices GET /api/v1/controller/device/ +.. _device_list_whois: + +**WHOIS Details** + +If :doc:`WHOIS Lookup feature ` is enabled, each device in the list +response will also include a ``whois_info`` field with related brief WHOIS +information. + **Available filters** You can filter a list of devices based on their configuration status using @@ -145,6 +153,13 @@ Get Device Detail GET /api/v1/controller/device/{id}/ +.. _device_detail_whois: + +**WHOIS Details** + +If :doc:`WHOIS Lookup feature ` is enabled, the response will also +include a ``whois_info`` field with related detailed WHOIS information. + Download Device Configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/user/settings.rst b/docs/user/settings.rst index 948b82ec8..de49b9799 100644 --- a/docs/user/settings.rst +++ b/docs/user/settings.rst @@ -774,8 +774,8 @@ documentation regarding automatic retries for known errors. Allows enabling the optional :doc:`WHOIS Lookup feature `. -.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.2/whois-admin-setting.png - :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.2/whois-admin-setting.png +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois-admin-setting.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois-admin-setting.png :alt: WHOIS admin setting After enabling this feature, you have to set diff --git a/docs/user/whois.rst b/docs/user/whois.rst index 3039569a4..8fc7d0e1c 100644 --- a/docs/user/whois.rst +++ b/docs/user/whois.rst @@ -78,3 +78,20 @@ Setup Instructions - Set :ref:`OPENWISP_CONTROLLER_WHOIS_ENABLED` to ``True``. - Set :ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT` to **Account ID**. - Set :ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY` to **License Key**. + +Viewing WHOIS Lookup Data +------------------------- + +Once the WHOIS Lookup feature is enabled and WHOIS data is available, the +retrieved details can be viewed in the following locations: + +- **Device Admin**: On the device's admin page, the WHOIS data is + displayed alongside the device's last IP address. + +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois-admin-details.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois-admin-details.png + :alt: WHOIS admin details + +- **Device REST API**: See WHOIS details in the :ref:`Device List + ` and :ref:`Device Detail ` + responses. diff --git a/openwisp_controller/config/admin.py b/openwisp_controller/config/admin.py index 4ba8ea83b..4ec225bb2 100644 --- a/openwisp_controller/config/admin.py +++ b/openwisp_controller/config/admin.py @@ -31,6 +31,7 @@ from swapper import load_model from openwisp_controller.config.views import get_default_values, get_relevant_templates +from openwisp_controller.config.whois.utils import get_whois_info from openwisp_users.admin import OrganizationAdmin from openwisp_users.multitenancy import MultitenantOrgFilter from openwisp_utils.admin import ( @@ -49,6 +50,7 @@ logger = logging.getLogger(__name__) prefix = "config/" +whois_prefix = "whois/" Config = load_model("config", "Config") Device = load_model("config", "Device") DeviceGroup = load_model("config", "DeviceGroup") @@ -569,12 +571,21 @@ class DeviceAdmin(MultitenantAdminMixin, BaseConfigAdmin, UUIDAdmin): fields.insert(0, "hardware_id") list_select_related = ("config", "organization") - class Media(BaseConfigAdmin.Media): + # overriding media property to allow testing in isolation + # as class Media is evaluated at import time making the + # settings not overridable in tests + @property + def media(self): js = BaseConfigAdmin.Media.js + [ f"{prefix}js/tabs.js", f"{prefix}js/management_ip.js", f"{prefix}js/relevant_templates.js", ] + css = BaseConfigAdmin.Media.css["all"] + if app_settings.WHOIS_CONFIGURED: + js.append(f"{whois_prefix}js/whois.js") + css += (f"{whois_prefix}css/whois.css",) + return super().media + forms.Media(js=js, css={"all": css}) def has_change_permission(self, request, obj=None): perm = super().has_change_permission(request) @@ -928,6 +939,10 @@ def get_extra_context(self, pk=None): ), } ) + # passing the whois details to the context to avoid + # the need to make an additional request in the js + if data := get_whois_info(pk): + ctx["device_whois_details"] = json.dumps(data) return ctx def add_view(self, request, form_url="", extra_context=None): diff --git a/openwisp_controller/config/api/serializers.py b/openwisp_controller/config/api/serializers.py index 7e67d8f5f..9402bbcbf 100644 --- a/openwisp_controller/config/api/serializers.py +++ b/openwisp_controller/config/api/serializers.py @@ -9,6 +9,7 @@ from ...serializers import BaseSerializer from .. import settings as app_settings +from ..whois.mixins import BriefWHOISMixin, WHOISMixin Template = load_model("config", "Template") Vpn = load_model("config", "Vpn") @@ -220,7 +221,7 @@ class DeviceListConfigSerializer(BaseConfigSerializer): templates = FilterTemplatesByOrganization(many=True, write_only=True) -class DeviceListSerializer(DeviceConfigSerializer): +class DeviceListSerializer(BriefWHOISMixin, DeviceConfigSerializer): config = DeviceListConfigSerializer(required=False) class Meta(BaseMeta): @@ -274,7 +275,7 @@ class DeviceDetailConfigSerializer(BaseConfigSerializer): templates = FilterTemplatesByOrganization(many=True) -class DeviceDetailSerializer(DeviceConfigSerializer): +class DeviceDetailSerializer(WHOISMixin, DeviceConfigSerializer): config = DeviceDetailConfigSerializer(allow_null=True) is_deactivated = serializers.BooleanField(read_only=True) diff --git a/openwisp_controller/config/migrations/0060_whoisinfo.py b/openwisp_controller/config/migrations/0061_whoisinfo.py similarity index 97% rename from openwisp_controller/config/migrations/0060_whoisinfo.py rename to openwisp_controller/config/migrations/0061_whoisinfo.py index abacbdb8c..d8148756b 100644 --- a/openwisp_controller/config/migrations/0060_whoisinfo.py +++ b/openwisp_controller/config/migrations/0061_whoisinfo.py @@ -10,7 +10,7 @@ class Migration(migrations.Migration): dependencies = [ - ("config", "0059_zerotier_templates_ow_zt_to_global"), + ("config", "0060_cleanup_api_task_notification_types"), ] operations = [ diff --git a/openwisp_controller/config/static/whois/css/whois.css b/openwisp_controller/config/static/whois/css/whois.css new file mode 100644 index 000000000..44a7b5d1f --- /dev/null +++ b/openwisp_controller/config/static/whois/css/whois.css @@ -0,0 +1,113 @@ +:root { + --whois-bg-light: #f9fafb; + --whois-text-muted: rgba(0, 0, 0, 0.45); + --whois-border-radius: 5px; + --whois-padding: 15px; +} + +.whois-table, +.whois { + margin-top: 1.25em; + max-width: 500px; + width: 100%; + border-radius: var(--whois-border-radius); +} +.whois-table { + border-spacing: 0; + box-shadow: 0 0 0 1px #ccc; + border-style: none; +} +.whois-table th, +.whois-table td { + padding: var(--whois-padding) !important; + border: none; +} +.whois-table tr:first-child { + background-color: var(--whois-bg-light); +} +.whois-table tr:not(:first-child) { + background-color: white; +} +.whois-table th:first-of-type > div { + display: flex; + align-items: center; +} +/* For rounding the individual corners */ +.whois-table th:first-of-type { + border-top-left-radius: var(--whois-border-radius); + border-right: 1px solid var(--border-color); + border-bottom: 1px solid var(--border-color); +} +.whois-table th:last-of-type { + border-top-right-radius: var(--whois-border-radius); + border-bottom: 1px solid var(--border-color); +} +.whois-table tr:last-child td:first-child { + border-bottom-left-radius: var(--whois-border-radius); + border-right: 1px solid var(--border-color); +} +.whois-table tr:last-child td:last-child { + border-bottom-right-radius: var(--whois-border-radius); +} +.whois-table .ow-info-icon { + width: 1rem; + height: 1rem; + mask-repeat: no-repeat; + mask-position: center; + mask-size: contain; + display: inline-block; + flex-shrink: 0; + background: #777; + margin-left: 1em; +} +.whois-table .ow-info-icon:hover { + background: #007cba; +} + +.whois { + background: var(--whois-bg-light); +} +.whois summary { + background: white; + border-radius: var(--whois-border-radius); + list-style: none; + display: flex; + align-items: center; + color: var(--whois-text-muted) !important; + font-weight: 600; + cursor: pointer; +} +details.whois[open] summary { + padding-bottom: 0 !important; +} +.whois summary > div { + display: flex; + width: 100%; + justify-content: space-between; + margin-left: 1.4em; +} +.whois summary::-webkit-details-marker { + display: none; +} +.whois summary .mg-arrow { + transform: scale(1.75); + display: inline; +} +.whois > div { + padding: var(--whois-padding); + font-size: 1em; +} +.whois .additional-text { + display: block; +} +.whois .additional-text + .additional-text { + margin-top: 5px; +} +.whois-globe { + flex-shrink: 0; + mask: url(../images/whois_globe.svg) no-repeat center; + -webkit-mask: url(../images/whois_globe.svg) no-repeat center; + width: 1.1rem; + height: 1.1rem; + background-color: var(--whois-text-muted); +} diff --git a/openwisp_controller/config/static/whois/images/whois_globe.svg b/openwisp_controller/config/static/whois/images/whois_globe.svg new file mode 100644 index 000000000..74116f1a6 --- /dev/null +++ b/openwisp_controller/config/static/whois/images/whois_globe.svg @@ -0,0 +1,3 @@ + + + diff --git a/openwisp_controller/config/static/whois/js/whois.js b/openwisp_controller/config/static/whois/js/whois.js new file mode 100644 index 000000000..20f4339f5 --- /dev/null +++ b/openwisp_controller/config/static/whois/js/whois.js @@ -0,0 +1,56 @@ +"use strict"; + +if (typeof gettext === "undefined") { + var gettext = function (word) { + return word; + }; +} + +django.jQuery(function ($) { + const $addForm = $(".add-form"); + const $deviceForm = $("#device_form"); + + if ( + $addForm.length || + !$deviceForm.length || + typeof deviceWHOISDetails === "undefined" + ) { + return; + } + const $parentDiv = $("#overview-group .field-last_ip div:last"); + const tooltipText = gettext( + "This is the Organization associated with registered ASN", + ); + + $parentDiv.after( + ` + + + + + + + + +
+
+ ${gettext("ISP")} + +
+ 		${gettext("Country")}
${deviceWHOISDetails.isp}${deviceWHOISDetails.address.country}
+
+ + +
+ ${gettext("Additional Details")} +
+ +
+ ${gettext("ASN")}: ${deviceWHOISDetails.asn} + ${gettext("Timezone")}: ${deviceWHOISDetails.timezone} + ${gettext("Address")}: ${deviceWHOISDetails.formatted_address} + ${gettext("CIDR")}: ${deviceWHOISDetails.cidr} +
+
`, + ); +}); diff --git a/openwisp_controller/config/templates/admin/config/change_form.html b/openwisp_controller/config/templates/admin/config/change_form.html index 5e0c993a0..02536c00f 100644 --- a/openwisp_controller/config/templates/admin/config/change_form.html +++ b/openwisp_controller/config/templates/admin/config/change_form.html @@ -19,6 +19,9 @@ const owControllerApiHost = window.location; {% endif %} const owCommandApiEndpoint = '{{ commands_api_endpoint | safe }}'; + {% if device_whois_details %} + const deviceWHOISDetails = {{ device_whois_details | safe }}; + {% endif %} {{ block.super}} {% endblock extrahead %} diff --git a/openwisp_controller/config/tests/test_api.py b/openwisp_controller/config/tests/test_api.py index 068528610..fe12fe2cf 100644 --- a/openwisp_controller/config/tests/test_api.py +++ b/openwisp_controller/config/tests/test_api.py @@ -250,7 +250,9 @@ def test_device_create_with_devicegroup(self): def test_device_list_api(self): device = self._create_device() path = reverse("config_api:device_list") - with self.assertNumQueries(3): + with patch.object( + app_settings, "WHOIS_CONFIGURED", False + ), self.assertNumQueries(3): r = self.client.get(path) self.assertEqual(r.status_code, 200) with self.subTest("device list should show most recent first"): @@ -396,7 +398,9 @@ def test_device_filter_templates(self): def test_device_detail_api(self): d1 = self._create_device() path = reverse("config_api:device_detail", args=[d1.pk]) - with self.assertNumQueries(2): + with patch.object( + app_settings, "WHOIS_CONFIGURED", False + ), self.assertNumQueries(2): r = self.client.get(path) self.assertEqual(r.status_code, 200) self.assertEqual(r.data["config"], None) @@ -406,7 +410,9 @@ def test_device_detail_config_api(self): d1 = self._create_device() self._create_config(device=d1) path = reverse("config_api:device_detail", args=[d1.pk]) - with self.assertNumQueries(3): + with patch.object( + app_settings, "WHOIS_CONFIGURED", False + ), self.assertNumQueries(3): r = self.client.get(path) self.assertEqual(r.status_code, 200) self.assertNotEqual(r.data["config"], None) diff --git a/openwisp_controller/config/whois/mixins.py b/openwisp_controller/config/whois/mixins.py new file mode 100644 index 000000000..4e2ee2ea7 --- /dev/null +++ b/openwisp_controller/config/whois/mixins.py @@ -0,0 +1,24 @@ +from .. import settings as app_settings +from .serializers import BriefWHOISSerializer, WHOISSerializer + + +class WHOISMixin: + """Mixin to add WHOIS information to the device representation.""" + + serializer_class = WHOISSerializer + + def to_representation(self, obj): + data = super().to_representation(obj) + if app_settings.WHOIS_CONFIGURED and obj.whois_service.is_whois_enabled: + data["whois_info"] = self.get_whois_info(obj) + return data + + def get_whois_info(self, obj): + whois_obj = obj.whois_service.get_device_whois_info() + if not whois_obj: + return None + return self.serializer_class(whois_obj).data + + +class BriefWHOISMixin(WHOISMixin): + serializer_class = BriefWHOISSerializer diff --git a/openwisp_controller/config/whois/serializers.py b/openwisp_controller/config/whois/serializers.py new file mode 100644 index 000000000..fa859b8e6 --- /dev/null +++ b/openwisp_controller/config/whois/serializers.py @@ -0,0 +1,28 @@ +from rest_framework import serializers +from swapper import load_model + +WHOISInfo = load_model("config", "WHOISInfo") + + +class BriefWHOISSerializer(serializers.ModelSerializer): + """ + Serializer for brief representation of WHOIS model. + """ + + country = serializers.CharField(source="address.country", read_only=True) + + class Meta: + model = WHOISInfo + fields = ("isp", "country", "ip_address") + + +class WHOISSerializer(serializers.ModelSerializer): + """ + Serializer for detailed representation of WHOIS model. + """ + + address = serializers.JSONField() + + class Meta: + model = WHOISInfo + fields = "__all__" diff --git a/openwisp_controller/config/whois/test_whois.py b/openwisp_controller/config/whois/test_whois.py index f5c93b6d3..d98fb8336 100644 --- a/openwisp_controller/config/whois/test_whois.py +++ b/openwisp_controller/config/whois/test_whois.py @@ -1,17 +1,21 @@ import importlib from unittest import mock +from django.contrib.staticfiles.testing import StaticLiveServerTestCase from django.core.exceptions import ImproperlyConfigured, ValidationError from django.db.models.signals import post_delete, post_save -from django.test import TestCase, TransactionTestCase, override_settings +from django.test import TestCase, TransactionTestCase, override_settings, tag from django.urls import reverse from geoip2 import errors +from selenium.webdriver.common.by import By from swapper import load_model +from openwisp_utils.tests import SeleniumTestMixin + from ...tests.utils import TestAdminMixin from .. import settings as app_settings from .handlers import connect_whois_handlers -from .utils import CreateWHOISMixin +from .tests_utils import CreateWHOISMixin Device = load_model("config", "Device") WHOISInfo = load_model("config", "WHOISInfo") @@ -183,6 +187,83 @@ def test_whois_enabled(self): app_settings.WHOIS_ENABLED, ) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_whois_details_device_api(self): + """ + Test the WHOIS details API endpoint. + """ + whois_obj = self._create_whois_info() + device = self._create_device(last_ip=whois_obj.ip_address) + self._login() + + with self.subTest( + "Device List API has whois_info when WHOIS_CONFIGURED is True" + ): + response = self.client.get(reverse("config_api:device_list")) + self.assertEqual(response.status_code, 200) + self.assertIn("whois_info", response.data["results"][0]) + self.assertDictEqual( + response.data["results"][0]["whois_info"], + { + "isp": whois_obj.isp, + "country": whois_obj.address["country"], + "ip_address": whois_obj.ip_address, + }, + ) + + with self.subTest( + "Device Detail API has whois_info when WHOIS_CONFIGURED is True" + ): + + response = self.client.get( + reverse("config_api:device_detail", args=[device.pk]) + ) + self.assertEqual(response.status_code, 200) + self.assertIn("whois_info", response.data) + api_whois_info = response.data["whois_info"] + self.assertEqual(api_whois_info["isp"], whois_obj.isp) + self.assertEqual(api_whois_info["cidr"], whois_obj.cidr) + self.assertEqual(api_whois_info["asn"], whois_obj.asn) + self.assertEqual(api_whois_info["timezone"], whois_obj.timezone) + self.assertEqual(api_whois_info["address"], whois_obj.address) + + with self.subTest( + "Device List API has whois_info as None when no WHOIS Info exists" + ): + device.last_ip = "172.217.22.24" + device.save() + response = self.client.get(reverse("config_api:device_list")) + self.assertEqual(response.status_code, 200) + self.assertIn("whois_info", response.data["results"][0]) + self.assertIsNone(response.data["results"][0]["whois_info"]) + + with self.subTest( + "Device Detail API has whois_info as None when no WHOIS Info exists" + ): + response = self.client.get( + reverse("config_api:device_detail", args=[device.pk]) + ) + self.assertEqual(response.status_code, 200) + self.assertIn("whois_info", response.data) + self.assertIsNone(response.data["whois_info"]) + + with mock.patch.object(app_settings, "WHOIS_CONFIGURED", False): + with self.subTest( + "Device List API has no whois_info when WHOIS_CONFIGURED is False" + ): + response = self.client.get(reverse("config_api:device_list")) + self.assertEqual(response.status_code, 200) + self.assertNotIn("whois_info", response.data["results"][0]) + + with self.subTest( + "Device Detail API has no whois_info when WHOIS_CONFIGURED is False" + ): + response = self.client.get( + reverse("config_api:device_detail", args=[device.pk]) + ) + self.assertEqual(response.status_code, 200) + self.assertNotIn("whois_info", response.data) + class TestWHOISInfoModel(CreateWHOISMixin, TestCase): def test_whois_model_fields_validation(self): @@ -223,7 +304,7 @@ def test_whois_model_fields_validation(self): class TestWHOISTransaction(CreateWHOISMixin, TransactionTestCase): _WHOIS_GEOIP_CLIENT = ( - "openwisp_controller.config.whois.tasks.geoip2_webservice.Client" + "openwisp_controller.config.whois.tasks.geoip2_webservice.Client.city" ) _WHOIS_TASKS_INFO_LOGGER = "openwisp_controller.config.whois.tasks.logger.info" _WHOIS_TASKS_WARN_LOGGER = "openwisp_controller.config.whois.tasks.logger.warning" @@ -260,7 +341,7 @@ def test_whois_task_called(self, mocked_task): mocked_task.assert_not_called() mocked_task.reset_mock() - with self.subTest("task not called when last_ip has related WHOISInfo"): + with self.subTest("task not called when last_ip has related WHOIS Info"): device.last_ip = "172.217.22.10" self._create_whois_info(ip_address=device.last_ip) device.save() @@ -419,7 +500,7 @@ def _verify_whois_details(instance, ip_address): mock_response.traits.autonomous_system_number = 15169 mock_response.traits.network = "172.217.22.0/24" mock_response.location.time_zone = "America/Los_Angeles" - mock_client.return_value.city.return_value = mock_response + mock_client.return_value = mock_response with self.subTest("Test WHOIS create when device is created"): device = self._create_device(last_ip="172.217.22.14") @@ -499,3 +580,63 @@ def assert_logging_on_exception( assert_logging_on_exception(errors.AddressNotFoundError) assert_logging_on_exception(errors.AuthenticationError) assert_logging_on_exception(errors.PermissionRequiredError) + + +@tag("selenium_tests") +class TestWHOISSelenium(CreateWHOISMixin, SeleniumTestMixin, StaticLiveServerTestCase): + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_whois_device_admin(self): + whois_obj = self._create_whois_info() + device = self._create_device(last_ip=whois_obj.ip_address) + self.login() + + with self.subTest( + "WHOIS details visible in device admin when WHOIS_CONFIGURED is True" + ): + self.open(reverse("admin:config_device_change", args=[device.pk])) + table = self.find_element(By.CSS_SELECTOR, "table.whois-table") + rows = table.find_elements(By.TAG_NAME, "tr") + for row in rows: + if cells := row.find_elements(By.TAG_NAME, "td"): + self.assertEqual(cells[0].text, whois_obj.isp) + self.assertEqual(cells[1].text, whois_obj.address["country"]) + + details = self.find_element(By.CSS_SELECTOR, "details.whois") + self.web_driver.execute_script( + "arguments[0].setAttribute('open','')", details + ) + additional_text = details.find_elements(By.CSS_SELECTOR, ".additional-text") + self.assertIn(whois_obj.asn, additional_text[0].text) + self.assertIn(whois_obj.timezone, additional_text[1].text) + self.assertIn(whois_obj.formatted_address, additional_text[2].text) + self.assertIn(whois_obj.cidr, additional_text[3].text) + + with mock.patch.object(app_settings, "WHOIS_CONFIGURED", False): + with self.subTest( + "WHOIS details not visible in device admin " + + "when WHOIS_CONFIGURED is False" + ): + self.open(reverse("admin:config_device_change", args=[device.pk])) + self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") + self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") + + with self.subTest( + "WHOIS details not visible in device admin when WHOIS is disabled" + ): + org = self._get_org() + org.config_settings.whois_enabled = False + org.config_settings.save(update_fields=["whois_enabled"]) + self.open(reverse("admin:config_device_change", args=[device.pk])) + self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") + self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") + + with self.subTest( + "WHOIS details not visible in device admin when WHOIS Info does not exist" + ): + org = self._get_org() + org.config_settings.whois_enabled = True + org.config_settings.save(update_fields=["whois_enabled"]) + WHOISInfo.objects.all().delete() + self.open(reverse("admin:config_device_change", args=[device.pk])) + self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") + self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") diff --git a/openwisp_controller/config/whois/tests_utils.py b/openwisp_controller/config/whois/tests_utils.py new file mode 100644 index 000000000..fd31ff6cf --- /dev/null +++ b/openwisp_controller/config/whois/tests_utils.py @@ -0,0 +1,36 @@ +from swapper import load_model + +from ..tests.utils import CreateConfigMixin + +Device = load_model("config", "Device") +WHOISInfo = load_model("config", "WHOISInfo") +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + + +class CreateWHOISMixin(CreateConfigMixin): + def _create_whois_info(self, **kwargs): + options = dict( + ip_address="172.217.22.14", + address={ + "city": "Mountain View", + "country": "United States", + "continent": "North America", + "postal": "94043", + }, + asn="15169", + isp="Google LLC", + timezone="America/Los_Angeles", + cidr="172.217.22.0/24", + ) + + options.update(kwargs) + w = WHOISInfo(**options) + w.full_clean() + w.save() + return w + + def setUp(self): + super().setUp() + OrganizationConfigSettings.objects.create( + organization=self._get_org(), whois_enabled=True + ) diff --git a/openwisp_controller/config/whois/utils.py b/openwisp_controller/config/whois/utils.py index fd31ff6cf..51919af08 100644 --- a/openwisp_controller/config/whois/utils.py +++ b/openwisp_controller/config/whois/utils.py @@ -1,36 +1,25 @@ from swapper import load_model -from ..tests.utils import CreateConfigMixin +from .. import settings as app_settings +from .serializers import WHOISSerializer +from .service import WHOISService Device = load_model("config", "Device") -WHOISInfo = load_model("config", "WHOISInfo") -OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") -class CreateWHOISMixin(CreateConfigMixin): - def _create_whois_info(self, **kwargs): - options = dict( - ip_address="172.217.22.14", - address={ - "city": "Mountain View", - "country": "United States", - "continent": "North America", - "postal": "94043", - }, - asn="15169", - isp="Google LLC", - timezone="America/Los_Angeles", - cidr="172.217.22.0/24", - ) - - options.update(kwargs) - w = WHOISInfo(**options) - w.full_clean() - w.save() - return w - - def setUp(self): - super().setUp() - OrganizationConfigSettings.objects.create( - organization=self._get_org(), whois_enabled=True - ) +def get_whois_info(pk): + if not app_settings.WHOIS_CONFIGURED or not pk: + return None + device = ( + Device.objects.select_related("organization__config_settings") + .filter(pk=pk) + .first() + ) + if not device or not device._get_organization__config_settings().whois_enabled: + return None + whois_obj = WHOISService(device).get_device_whois_info() + if not whois_obj: + return None + data = WHOISSerializer(whois_obj).data + data["formatted_address"] = getattr(whois_obj, "formatted_address", None) + return data From 0633b169d05aa61db8982b846dca6f78eb9e4e44 Mon Sep 17 00:00:00 2001 From: JgtAman <41839176+DragnEmperor@users.noreply.github.com> Date: Sat, 20 Sep 2025 21:51:12 +0530 Subject: [PATCH 3/5] [feature] Allow estimating geographic location based on WHOIS data #1034 - Estimated geographic location logic, closes #1034. - Added notification and admin warning, closes #1035. - Set location estimated flag to ``False`` on manual update, closes #1036. - Added admin and API filters for estimated locations, closes #1028. --------- Signed-off-by: DragnEmperor --- docs/index.rst | 1 + docs/user/estimated-location.rst | 80 ++ docs/user/rest-api.rst | 43 + docs/user/settings.rst | 17 + docs/user/whois.rst | 39 +- openwisp_controller/config/admin.py | 2 +- openwisp_controller/config/base/device.py | 12 +- .../config/base/multitenancy.py | 14 + openwisp_controller/config/base/whois.py | 61 +- .../config/controller/views.py | 9 - .../config/management/__init__.py | 0 .../config/management/commands/__init__.py | 0 .../management/commands/clear_last_ip.py | 51 ++ ...s_approximate_location_enabled_and_more.py | 35 + openwisp_controller/config/settings.py | 5 + openwisp_controller/config/whois/service.py | 94 ++- openwisp_controller/config/whois/tasks.py | 48 +- .../config/whois/tests/__init__.py | 0 .../whois/{test_whois.py => tests/tests.py} | 215 +++-- .../config/whois/tests/utils.py | 152 ++++ .../config/whois/tests_utils.py | 36 - openwisp_controller/config/whois/utils.py | 59 +- openwisp_controller/geo/admin.py | 54 +- openwisp_controller/geo/api/filters.py | 15 + openwisp_controller/geo/api/serializers.py | 14 +- openwisp_controller/geo/api/views.py | 8 + openwisp_controller/geo/apps.py | 3 + openwisp_controller/geo/base/models.py | 73 ++ .../geo/estimated_location/__init__.py | 0 .../geo/estimated_location/handlers.py | 41 + .../geo/estimated_location/mixins.py | 30 + .../geo/estimated_location/tasks.py | 170 ++++ .../geo/estimated_location/tests/__init__.py | 0 .../geo/estimated_location/tests/tests.py | 761 ++++++++++++++++++ .../geo/estimated_location/tests/utils.py | 15 + .../geo/estimated_location/utils.py | 6 + .../migrations/0004_location_is_estimated.py | 21 + .../admin/geo/location/change_form.html | 11 + openwisp_controller/geo/tests/test_admin.py | 10 + ...s_approximate_location_enabled_and_more.py | 35 + .../migrations/0004_location_is_estimated.py | 21 + 41 files changed, 2064 insertions(+), 197 deletions(-) create mode 100644 docs/user/estimated-location.rst create mode 100644 openwisp_controller/config/management/__init__.py create mode 100644 openwisp_controller/config/management/commands/__init__.py create mode 100644 openwisp_controller/config/management/commands/clear_last_ip.py create mode 100644 openwisp_controller/config/migrations/0062_organizationconfigsettings_approximate_location_enabled_and_more.py create mode 100644 openwisp_controller/config/whois/tests/__init__.py rename openwisp_controller/config/whois/{test_whois.py => tests/tests.py} (80%) create mode 100644 openwisp_controller/config/whois/tests/utils.py delete mode 100644 openwisp_controller/config/whois/tests_utils.py create mode 100644 openwisp_controller/geo/estimated_location/__init__.py create mode 100644 openwisp_controller/geo/estimated_location/handlers.py create mode 100644 openwisp_controller/geo/estimated_location/mixins.py create mode 100644 openwisp_controller/geo/estimated_location/tasks.py create mode 100644 openwisp_controller/geo/estimated_location/tests/__init__.py create mode 100644 openwisp_controller/geo/estimated_location/tests/tests.py create mode 100644 openwisp_controller/geo/estimated_location/tests/utils.py create mode 100644 openwisp_controller/geo/estimated_location/utils.py create mode 100644 openwisp_controller/geo/migrations/0004_location_is_estimated.py create mode 100644 openwisp_controller/geo/templates/admin/geo/location/change_form.html create mode 100644 tests/openwisp2/sample_config/migrations/0009_organizationconfigsettings_approximate_location_enabled_and_more.py create mode 100644 tests/openwisp2/sample_geo/migrations/0004_location_is_estimated.py diff --git a/docs/index.rst b/docs/index.rst index 6895e1c48..c6b268f55 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -49,6 +49,7 @@ the OpenWISP architecture. user/openvpn.rst user/subnet-division-rules.rst user/whois.rst + user/estimated-location.rst user/rest-api.rst user/settings.rst diff --git a/docs/user/estimated-location.rst b/docs/user/estimated-location.rst new file mode 100644 index 000000000..1f00a7440 --- /dev/null +++ b/docs/user/estimated-location.rst @@ -0,0 +1,80 @@ +Estimated Location +================== + +.. important:: + + The **Estimated Location** feature is **disabled by default**. + + Before enabling it, the :doc:`WHOIS Lookup feature ` must be + enabled. Then set + :ref:`OPENWISP_CONTROLLER_ESTIMATED_LOCATION_ENABLED` to ``True`` + +.. contents:: **Table of contents**: + :depth: 1 + :local: + +Overview +-------- + +The Estimated Location feature automatically creates or updates a device’s +location based on latitude and longitude information retrieved from the +WHOIS Lookup feature. + +Trigger Conditions +------------------ + +Estimated Location is triggered when: + +- A **fresh WHOIS lookup** is performed for a device. +- Or when a WHOIS record already exists for the device’s IP **and**: + + - The device’s last IP address is **public**. + - WHOIS lookup and Estimated Location is **enabled** for the device’s + organization. + +Behavior +-------- + +The system will **attach the already existing matching location** of +another device with same ip to the current device if: + +- Only one device is found with that IP and it has a location. +- The current device **has no location** or that location is + **estimated**. + +If there are multiple devices with location for the same IP, the system +will **not attach any location** to the current device and a notification +will be sent suggesting the user to manually assign/create a location for +the device. + +If there is **no matching location**, a new estimated location is created +or the existing one is updated using coordinates from the WHOIS record, +but only if the existing location is estimated. + +If two devices share the same IP address and are assigned to the same +location, and the last IP of one of the devices is updated, the system +will create a new estimated location for that device. + +Visibility of Estimated Status +------------------------------ + +The estimated status of a location is visible on the location page if the +feature is enabled for the organization. The location admin page also +includes indicators for the estimated status. + +- The name of the location will have suffix **(Estimated Location : + )**. +- A warning on top of the page. +- **Is Estimated** field. + +Changes to the ``coordinates`` and ``geometry`` of the estimated location +will set the ``is_estimated`` field to ``False`` and remove the +"(Estimated Location)" suffix with IP from the location name. + +In REST API, the field will be visible in the :ref:`Device Location +`, :ref:`Location list +`, :ref:`Location Detail +` and :ref:`Location list (GeoJson) +` if the feature is **enabled**. The field can +also be used for filtering in the location list (including geojson) +endpoints and in the :ref:`Device List `. diff --git a/docs/user/rest-api.rst b/docs/user/rest-api.rst index c58d1ce93..fe2efba0e 100644 --- a/docs/user/rest-api.rst +++ b/docs/user/rest-api.rst @@ -76,6 +76,14 @@ If :doc:`WHOIS Lookup feature ` is enabled, each device in the list response will also include a ``whois_info`` field with related brief WHOIS information. +.. _device_list_estimated_filters: + +**Estimated Location Filters** + +if :doc:`Estimated Location feature ` is enabled, +devices can be filtered based on the estimated nature of their location +using the ``geo_is_estimated``. + **Available filters** You can filter a list of devices based on their configuration status using @@ -544,6 +552,13 @@ of certificate's organization as show in the example below: GET /api/v1/controller/cert/{common_name}/group/?org={org1_slug},{org2_slug} +.. |est_loc| replace:: Estimated Location feature + +.. _est_loc: estimated-location.html + +.. |estimated_details| replace:: If |est_loc|_ is enabled, the location + response will also include ``is_estimated`` status field. + Get Device Location ~~~~~~~~~~~~~~~~~~~ @@ -551,6 +566,12 @@ Get Device Location GET /api/v1/controller/device/{id}/location/ +.. _device_location_estimated: + +**Estimated Status** + +|estimated_details| + .. _create_device_location: Create Device Location @@ -787,6 +808,14 @@ List Locations GET /api/v1/controller/location/ +.. _location_list_estimated: + +**Estimated Status** + +|estimated_details| + +Locations can also be filtered using the ``is_estimated``. + **Available filters** You can filter using ``organization_id`` or ``organization_slug`` to get @@ -868,6 +897,12 @@ Get Location Details GET /api/v1/controller/location/{pk}/ +.. _location_detail_estimated: + +**Estimated Status** + +|estimated_details| + Change Location Details ~~~~~~~~~~~~~~~~~~~~~~~ @@ -910,6 +945,14 @@ List Locations with Devices Deployed (in GeoJSON Format) GET /api/v1/controller/location/geojson/ +.. _location_geojson_estimated: + +**Estimated Status** + +|estimated_details| + +Locations can also be filtered using the ``is_estimated``. + **Available filters** You can filter using ``organization_id`` or ``organization_slug`` to get diff --git a/docs/user/settings.rst b/docs/user/settings.rst index de49b9799..9a59bbf22 100644 --- a/docs/user/settings.rst +++ b/docs/user/settings.rst @@ -812,3 +812,20 @@ Maxmind Account ID required for the :doc:`WHOIS Lookup feature `. ============ ======= Maxmind License Key required for the :doc:`WHOIS Lookup feature `. + +.. _openwisp_controller_whois_estimated_location_enabled: + +``OPENWISP_CONTROLLER_WHOIS_ESTIMATED_LOCATION_ENABLED`` +-------------------------------------------------------- + +============ ========= +**type**: ``bool`` +**default**: ``False`` +============ ========= + +Allows enabling the optional :doc:`Estimated Location feature +`. + +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-location-setting.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-location-setting.png + :alt: Estimated Location setting diff --git a/docs/user/whois.rst b/docs/user/whois.rst index 8fc7d0e1c..e9c63274c 100644 --- a/docs/user/whois.rst +++ b/docs/user/whois.rst @@ -28,6 +28,7 @@ associated with the device's public IP address and includes: - CIDR block assigned to the ASN - Physical address registered to the ASN - Timezone of the ASN's registered location +- Coordinates (Latitude and Longitude) Trigger Conditions ------------------ @@ -40,25 +41,23 @@ A WHOIS lookup is triggered automatically when: However, the lookup will only run if **all** the following conditions are met: +- The device is either **newly created** or has a **changed last IP**. - The device's last IP address is **public**. - There is **no existing WHOIS record** for that IP. - WHOIS lookup is **enabled** for the device's organization. -Behavior with Shared IP Addresses ---------------------------------- +Managing WHOIS Records +---------------------- -If multiple devices share the same public IP address and one of them -switches to a different IP, the following occurs: - -- A lookup is triggered for the **new IP**. -- The WHOIS record for the **old IP** is deleted. -- The next time a device still using the old IP fetches its checksum, a - new lookup is triggered, ensuring up-to-date data. +If a device updates its last IP address, lookup is triggered for the **new +IP** and the **WHOIS record for the old IP** is deleted if no active +devices are associated with that IP address. .. note:: When a device with an associated WHOIS record is deleted, its WHOIS - record is automatically removed. + record is automatically removed only if no active devices are + associated with that IP address. .. _controller_setup_whois_lookup: @@ -79,6 +78,26 @@ Setup Instructions - Set :ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT` to **Account ID**. - Set :ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY` to **License Key**. +6. Restart the application/containers if using ansible-openwisp2 or + docker. +7. Run the ``clear_last_ip`` management command to clear the last IP + address of **all active devices across organizations**. + + - If using ansible-openwisp2 (default directory is /opt/openwisp2, + unless changed in Ansible playbook configuration): + + .. code-block:: bash + + source /opt/openwisp2/env/bin/activate + python /opt/openwisp2/src/manage.py clear_last_ip + + - If using docker: + + .. code-block:: bash + + docker exec -it sh + python manage.py clear_last_ip + Viewing WHOIS Lookup Data ------------------------- diff --git a/openwisp_controller/config/admin.py b/openwisp_controller/config/admin.py index 4ec225bb2..af9e20320 100644 --- a/openwisp_controller/config/admin.py +++ b/openwisp_controller/config/admin.py @@ -1393,7 +1393,7 @@ def get_fields(self, request, obj=None): if app_settings.REGISTRATION_ENABLED: fields += ["registration_enabled", "shared_secret"] if app_settings.WHOIS_CONFIGURED: - fields += ["whois_enabled"] + fields += ["whois_enabled", "estimated_location_enabled"] fields += ["context"] return fields diff --git a/openwisp_controller/config/base/device.py b/openwisp_controller/config/base/device.py index 1921d5ad3..525098737 100644 --- a/openwisp_controller/config/base/device.py +++ b/openwisp_controller/config/base/device.py @@ -287,7 +287,7 @@ def save(self, *args, **kwargs): state_adding = self._state.adding super().save(*args, **kwargs) if app_settings.WHOIS_CONFIGURED: - self._check_last_ip() + self._check_last_ip(creating=state_adding) if state_adding and self.group and self.group.templates.exists(): self.create_default_config() # The value of "self._state.adding" will always be "False" @@ -529,9 +529,13 @@ def whois_service(self): """ return WHOISService(self) - def _check_last_ip(self): - """Trigger WHOIS lookup if last_ip is not deferred.""" + def _check_last_ip(self, creating=False): + """ + Process details and location related to last_ip if last_ip has + changed or is being set for the first time. + """ if self._initial_last_ip == models.DEFERRED: return - self.whois_service.trigger_whois_lookup() + if creating or self.last_ip != self._initial_last_ip: + self.whois_service.process_ip_data_and_location() self._initial_last_ip = self.last_ip diff --git a/openwisp_controller/config/base/multitenancy.py b/openwisp_controller/config/base/multitenancy.py index 3095ab779..bceb5de42 100644 --- a/openwisp_controller/config/base/multitenancy.py +++ b/openwisp_controller/config/base/multitenancy.py @@ -39,6 +39,11 @@ class AbstractOrganizationConfigSettings(UUIDModel): fallback=app_settings.WHOIS_ENABLED, verbose_name=_("WHOIS Enabled"), ) + estimated_location_enabled = FallbackBooleanChoiceField( + help_text=_("Whether the estimated location feature is enabled"), + fallback=app_settings.ESTIMATED_LOCATION_ENABLED, + verbose_name=_("Estimated Location Enabled"), + ) context = JSONField( blank=True, default=dict, @@ -71,6 +76,15 @@ def clean(self): ) } ) + if not self.whois_enabled and self.estimated_location_enabled: + raise ValidationError( + { + "estimated_location_enabled": _( + "Estimated Location feature requires " + "WHOIS Lookup feature to be enabled." + ) + } + ) return super().clean() def save( diff --git a/openwisp_controller/config/base/whois.py b/openwisp_controller/config/base/whois.py index 80af038f2..4ae29aebb 100644 --- a/openwisp_controller/config/base/whois.py +++ b/openwisp_controller/config/base/whois.py @@ -1,10 +1,12 @@ from ipaddress import ip_address, ip_network +from django.contrib.gis.db.models import PointField from django.core.cache import cache from django.core.exceptions import ValidationError from django.db import models, transaction from django.utils.translation import gettext_lazy as _ from jsonfield import JSONField +from swapper import load_model from openwisp_utils.base import TimeStampedEditableModel @@ -51,6 +53,12 @@ class AbstractWHOISInfo(TimeStampedEditableModel): blank=True, help_text=_("CIDR"), ) + coordinates = PointField( + null=True, + blank=True, + help_text=_("Coordinates"), + srid=4326, + ) class Meta: abstract = True @@ -74,6 +82,20 @@ def clean(self): raise ValidationError( {"cidr": _("Invalid CIDR format: %(error)s") % {"error": str(e)}} ) + + if self.coordinates: + if not (-90 <= self.coordinates.y <= 90): + raise ValidationError( + {"coordinates": _("Latitude must be between -90 and 90 degrees.")} + ) + if not (-180 <= self.coordinates.x <= 180): + raise ValidationError( + { + "coordinates": _( + "Longitude must be between -180 and 180 degrees." + ) + } + ) return super().clean() @staticmethod @@ -82,8 +104,18 @@ def device_whois_info_delete_handler(instance, **kwargs): Delete WHOIS information for a device when the last IP address is removed or when device is deleted. """ - if instance._get_organization__config_settings().whois_enabled: - transaction.on_commit(lambda: delete_whois_record.delay(instance.last_ip)) + Device = load_model("config", "Device") + + last_ip = instance.last_ip + existing_devices = Device.objects.filter(_is_deactivated=False).filter( + last_ip=last_ip + ) + if ( + last_ip + and instance._get_organization__config_settings().whois_enabled + and not existing_devices.exists() + ): + transaction.on_commit(lambda: delete_whois_record.delay(last_ip)) # this method is kept here instead of in OrganizationConfigSettings because # currently the caching is used only for WHOIS feature @@ -113,3 +145,28 @@ def formatted_address(self): ], ) ) + + @property + def _location_name(self): + """ + Used to get location name based on the address and IP. + """ + address = self.formatted_address + if address: + parts = [part.strip() for part in address.split(",")[:2] if part.strip()] + location = ", ".join(parts) + return _(f"{location} (Estimated Location: {self.ip_address})") + return _(f"Estimated Location: {self.ip_address}") + + def _get_defaults_for_estimated_location(self): + """ + Used to get default values for creating or updating + an estimated location based on the WHOIS information. + """ + return { + "name": self._location_name, + "type": "outdoor", + "is_mobile": False, + "geometry": self.coordinates, + "address": self.formatted_address, + } diff --git a/openwisp_controller/config/controller/views.py b/openwisp_controller/config/controller/views.py index 68023ea6d..3dc714c0c 100644 --- a/openwisp_controller/config/controller/views.py +++ b/openwisp_controller/config/controller/views.py @@ -153,15 +153,6 @@ def get(self, request, pk): # updates cache if ip addresses changed if updated: self.update_device_cache(device) - # When update fields are present then save() will run the WHOIS - # lookup. But if there are no update fields, we still want to - # trigger the WHOIS lookup if there is no record for the device's - # last_ip. - elif ( - app_settings.WHOIS_CONFIGURED - and not device.whois_service.get_device_whois_info() - ): - device.whois_service.trigger_whois_lookup() checksum_requested.send( sender=device.__class__, instance=device, request=request ) diff --git a/openwisp_controller/config/management/__init__.py b/openwisp_controller/config/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/config/management/commands/__init__.py b/openwisp_controller/config/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/config/management/commands/clear_last_ip.py b/openwisp_controller/config/management/commands/clear_last_ip.py new file mode 100644 index 000000000..4a5efd4c7 --- /dev/null +++ b/openwisp_controller/config/management/commands/clear_last_ip.py @@ -0,0 +1,51 @@ +from django.core.management.base import BaseCommand, CommandError +from django.db.models import OuterRef, Subquery +from swapper import load_model + + +class Command(BaseCommand): + help = "Clear the last IP address, if set, of active devices of all organizations." + + def add_arguments(self, parser): + parser.add_argument( + "--noinput", + "--no-input", + action="store_false", + dest="interactive", + help="Do NOT prompt the user for input of any kind.", + ) + return super().add_arguments(parser) + + def handle(self, *args, **options): + Device = load_model("config", "Device") + WHOISInfo = load_model("config", "WHOISInfo") + + if options["interactive"]: + message = ["\n"] + message.append( + "This will clear last IP of all active devices across organizations!\n" + ) + message.append( + "Are you sure you want to do this?\n\n" + "Type 'yes' to continue, or 'no' to cancel: " + ) + if input("".join(message)) != "yes": + raise CommandError("Operation cancelled by user.") + + devices = Device.objects.filter(_is_deactivated=False).only("last_ip") + # Filter devices that have no WHOIS information for their last IP + devices = devices.exclude(last_ip=None).exclude( + last_ip__in=Subquery( + WHOISInfo.objects.filter(ip_address=OuterRef("last_ip")).values( + "ip_address" + ) + ), + ) + + updated_devices = devices.update(last_ip=None) + if updated_devices: + self.stdout.write( + f"Cleared last IP addresses for {updated_devices} active device(s)." + ) + else: + self.stdout.write("No active devices with last IP to clear.") diff --git a/openwisp_controller/config/migrations/0062_organizationconfigsettings_approximate_location_enabled_and_more.py b/openwisp_controller/config/migrations/0062_organizationconfigsettings_approximate_location_enabled_and_more.py new file mode 100644 index 000000000..8fe4d4c2c --- /dev/null +++ b/openwisp_controller/config/migrations/0062_organizationconfigsettings_approximate_location_enabled_and_more.py @@ -0,0 +1,35 @@ +# Generated by Django 5.2.1 on 2025-07-10 18:09 + +import django.contrib.gis.db.models.fields +from django.db import migrations + +import openwisp_utils.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("config", "0061_whoisinfo"), + ] + + operations = [ + migrations.AddField( + model_name="organizationconfigsettings", + name="estimated_location_enabled", + field=openwisp_utils.fields.FallbackBooleanChoiceField( + blank=True, + default=None, + fallback=False, + help_text="Whether the estimated location feature is enabled", + null=True, + verbose_name="Estimated Location Enabled", + ), + ), + migrations.AddField( + model_name="whoisinfo", + name="coordinates", + field=django.contrib.gis.db.models.fields.PointField( + blank=True, help_text="Coordinates", null=True, srid=4326 + ), + ), + ] diff --git a/openwisp_controller/config/settings.py b/openwisp_controller/config/settings.py index a94617b34..fa3efc750 100644 --- a/openwisp_controller/config/settings.py +++ b/openwisp_controller/config/settings.py @@ -77,3 +77,8 @@ def get_setting(option, default): "WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY must be set " + "when WHOIS_ENABLED is True." ) +ESTIMATED_LOCATION_ENABLED = get_setting("ESTIMATED_LOCATION_ENABLED", False) +if ESTIMATED_LOCATION_ENABLED and not WHOIS_ENABLED: + raise ImproperlyConfigured( + "WHOIS must be enabled before enabling ESTIMATED_LOCATION globally" + ) diff --git a/openwisp_controller/config/whois/service.py b/openwisp_controller/config/whois/service.py index 81275a8e3..3e9dad6f1 100644 --- a/openwisp_controller/config/whois/service.py +++ b/openwisp_controller/config/whois/service.py @@ -1,4 +1,4 @@ -from ipaddress import ip_address +from ipaddress import ip_address as ip_addr from django.core.cache import cache from django.db import transaction @@ -6,7 +6,7 @@ from openwisp_controller.config import settings as app_settings -from .tasks import fetch_whois_details +from .tasks import fetch_whois_details, manage_estimated_locations class WHOISService: @@ -30,7 +30,7 @@ def is_valid_public_ip_address(ip): Check if given IP address is a valid public IP address. """ try: - return ip and ip_address(ip).is_global + return ip and ip_addr(ip).is_global except ValueError: return False @@ -43,20 +43,24 @@ def _get_whois_info_from_db(ip_address): return WHOISInfo.objects.filter(ip_address=ip_address) - @property - def is_whois_enabled(self): + @staticmethod + def get_org_config_settings(org_id): """ - Check if the WHOIS lookup feature is enabled. - The OrganizationConfigSettings are cached as these settings - are not expected to change frequently. The timeout for the cache - is set to the same as the checksum cache timeout for consistency - with DeviceChecksumView. + Retrieve and cache organization-specific configuration settings. + + Returns a "read-only" OrganizationConfigSettings instance for the + given organization. + If no settings exist for the organization, returns an empty instance to allow + fallback to global defaults. + + OrganizationConfigSettings are cached for performance, using the same timeout + as DeviceChecksumView for consistency. """ OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") Config = load_model("config", "Config") - org_id = self.device.organization.pk - org_settings = cache.get(self.get_cache_key(org_id=org_id)) + cache_key = WHOISService.get_cache_key(org_id=org_id) + org_settings = cache.get(cache_key) if org_settings is None: try: org_settings = OrganizationConfigSettings.objects.get( @@ -64,13 +68,38 @@ def is_whois_enabled(self): ) except OrganizationConfigSettings.DoesNotExist: # If organization settings do not exist, fall back to global setting - return app_settings.WHOIS_ENABLED + org_settings = OrganizationConfigSettings() cache.set( - self.get_cache_key(org_id=org_id), + cache_key, org_settings, timeout=Config._CHECKSUM_CACHE_TIMEOUT, ) - return getattr(org_settings, "whois_enabled", app_settings.WHOIS_ENABLED) + return org_settings + + @staticmethod + def check_estimate_location_configured(org_id): + if not org_id: + return False + if not app_settings.WHOIS_CONFIGURED: + return False + org_settings = WHOISService.get_org_config_settings(org_id=org_id) + return org_settings.estimated_location_enabled + + @property + def is_whois_enabled(self): + """ + Check if the WHOIS lookup feature is enabled. + """ + org_settings = self.get_org_config_settings(org_id=self.device.organization.pk) + return org_settings.whois_enabled + + @property + def is_estimated_location_enabled(self): + """ + Check if the Estimated location feature is enabled. + """ + org_settings = self.get_org_config_settings(org_id=self.device.organization.pk) + return org_settings.estimated_location_enabled def _need_whois_lookup(self, new_ip): """ @@ -92,6 +121,19 @@ def _need_whois_lookup(self, new_ip): return self.is_whois_enabled + def _need_estimated_location_management(self, new_ip): + """ + Used to determine if Estimated locations need to be created/updated + or not during WHOIS lookup. + """ + if not self.is_valid_public_ip_address(new_ip): + return False + + if not self.is_whois_enabled: + return False + + return self.is_estimated_location_enabled + def get_device_whois_info(self): """ If the WHOIS lookup feature is enabled and the device ``last_ip`` @@ -103,16 +145,28 @@ def get_device_whois_info(self): return self._get_whois_info_from_db(ip_address=ip_address).first() - def trigger_whois_lookup(self): + def process_ip_data_and_location(self): """ - Trigger WHOIS lookup based on the conditions of `_need_whois_lookup`. - Task is triggered on commit to ensure redundant data is not created. + Trigger WHOIS lookup based on the conditions of `_need_whois_lookup` + and also manage estimated locations based on the conditions of + `_need_estimated_location_management`. + Tasks are triggered on commit to ensure redundant data is not created. """ - if self._need_whois_lookup(self.device.last_ip): + new_ip = self.device.last_ip + if self._need_whois_lookup(new_ip): transaction.on_commit( lambda: fetch_whois_details.delay( device_pk=self.device.pk, initial_ip_address=self.device._initial_last_ip, - new_ip_address=self.device.last_ip, + new_ip_address=new_ip, + ) + ) + # To handle the case when WHOIS already exists as in that case + # WHOIS lookup is not triggered but we still need to + # manage estimated locations. + elif self._need_estimated_location_management(new_ip): + transaction.on_commit( + lambda: manage_estimated_locations.delay( + device_pk=self.device.pk, ip_address=new_ip ) ) diff --git a/openwisp_controller/config/whois/tasks.py b/openwisp_controller/config/whois/tasks.py index 2ae434cd7..bafca7b78 100644 --- a/openwisp_controller/config/whois/tasks.py +++ b/openwisp_controller/config/whois/tasks.py @@ -2,15 +2,17 @@ import requests from celery import shared_task +from django.contrib.gis.geos import Point from django.utils.translation import gettext as _ from geoip2 import errors from geoip2 import webservice as geoip2_webservice -from openwisp_notifications.signals import notify from swapper import load_model +from openwisp_controller.geo.estimated_location.tasks import manage_estimated_locations from openwisp_utils.tasks import OpenwispCeleryTask from .. import settings as app_settings +from .utils import send_whois_task_notification logger = logging.getLogger(__name__) @@ -43,25 +45,9 @@ class WHOISCeleryRetryTask(OpenwispCeleryTask): def on_failure(self, exc, task_id, args, kwargs, einfo): """Notify the user about the failure of the WHOIS task.""" - Device = load_model("config", "Device") - device_pk = kwargs.get("device_pk") - new_ip_address = kwargs.get("new_ip_address") - device = Device.objects.get(pk=device_pk) - - notify.send( - sender=device, - type="generic_message", - target=device, - action_object=device, - level="error", - message=_( - "Failed to fetch WHOIS details for device" - " [{notification.target}]({notification.target_link})" - ), - description=_( - f"WHOIS details could not be fetched for ip: {new_ip_address}." - ), + send_whois_task_notification( + device_pk=device_pk, notify_type="whois_device_error" ) logger.error(f"WHOIS lookup failed. Details: {exc}") return super().on_failure(exc, task_id, args, kwargs, einfo) @@ -78,6 +64,7 @@ def fetch_whois_details(self, device_pk, initial_ip_address, new_ip_address): Fetches the WHOIS details of the given IP address and creates/updates the WHOIS record. """ + Device = load_model("config", "Device") WHOISInfo = load_model("config", "WHOISInfo") # The task can be triggered for same ip address multiple times @@ -85,6 +72,7 @@ def fetch_whois_details(self, device_pk, initial_ip_address, new_ip_address): if WHOISInfo.objects.filter(ip_address=new_ip_address).exists(): return + device = Device.objects.get(pk=device_pk) # Host is based on the db that is used to fetch the details. # As we are using GeoLite2, 'geolite.info' host is used. # Refer: https://geoip2.readthedocs.io/en/latest/#sync-web-service-example @@ -123,7 +111,7 @@ def fetch_whois_details(self, device_pk, initial_ip_address, new_ip_address): "continent": data.continent.name or "", "postal": str(data.postal.code or ""), } - + coordinates = Point(data.location.longitude, data.location.latitude, srid=4326) whois_obj = WHOISInfo( isp=data.traits.autonomous_system_organization, asn=data.traits.autonomous_system_number, @@ -131,19 +119,23 @@ def fetch_whois_details(self, device_pk, initial_ip_address, new_ip_address): address=address, cidr=data.traits.network, ip_address=new_ip_address, + coordinates=coordinates, ) whois_obj.full_clean() whois_obj.save() logger.info(f"Successfully fetched WHOIS details for {new_ip_address}.") - # the following check ensures that for a case when device last_ip - # is not changed and there is no related WHOIS record, we do not - # delete the newly created record as both `initial_ip_address` and - # `new_ip_address` would be same for such case. - if initial_ip_address != new_ip_address: - # If any active devices are linked to the following record, - # then they will trigger this task and new record gets created - # with latest data. + if device._get_organization__config_settings().estimated_location_enabled: + manage_estimated_locations.delay( + device_pk=device_pk, ip_address=new_ip_address + ) + + # delete WHOIS record for initial IP if no devices are linked to it + if ( + not Device.objects.filter(_is_deactivated=False) + .filter(last_ip=initial_ip_address) + .exists() + ): delete_whois_record(ip_address=initial_ip_address) diff --git a/openwisp_controller/config/whois/tests/__init__.py b/openwisp_controller/config/whois/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/config/whois/test_whois.py b/openwisp_controller/config/whois/tests/tests.py similarity index 80% rename from openwisp_controller/config/whois/test_whois.py rename to openwisp_controller/config/whois/tests/tests.py index d98fb8336..82ad9e83b 100644 --- a/openwisp_controller/config/whois/test_whois.py +++ b/openwisp_controller/config/whois/tests/tests.py @@ -1,8 +1,11 @@ import importlib +from io import StringIO from unittest import mock +from django.contrib.gis.geos import Point from django.contrib.staticfiles.testing import StaticLiveServerTestCase from django.core.exceptions import ImproperlyConfigured, ValidationError +from django.core.management import call_command from django.db.models.signals import post_delete, post_save from django.test import TestCase, TransactionTestCase, override_settings, tag from django.urls import reverse @@ -12,15 +15,15 @@ from openwisp_utils.tests import SeleniumTestMixin -from ...tests.utils import TestAdminMixin -from .. import settings as app_settings -from .handlers import connect_whois_handlers -from .tests_utils import CreateWHOISMixin +from ....tests.utils import TestAdminMixin +from ... import settings as app_settings +from ..handlers import connect_whois_handlers +from .utils import CreateWHOISMixin, WHOISTransactionMixin Device = load_model("config", "Device") WHOISInfo = load_model("config", "WHOISInfo") -OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") Notification = load_model("openwisp_notifications", "Notification") +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") notification_qs = Notification.objects.all() @@ -214,7 +217,6 @@ def test_whois_details_device_api(self): with self.subTest( "Device Detail API has whois_info when WHOIS_CONFIGURED is True" ): - response = self.client.get( reverse("config_api:device_detail", args=[device.pk]) ) @@ -264,6 +266,20 @@ def test_whois_details_device_api(self): self.assertEqual(response.status_code, 200) self.assertNotIn("whois_info", response.data) + def test_last_ip_management_command(self): + out = StringIO() + device = self._create_device(last_ip="172.217.22.11") + args = ["--noinput"] + call_command("clear_last_ip", *args, stdout=out, stderr=StringIO()) + self.assertIn( + "Cleared last IP addresses for 1 active device(s).", out.getvalue() + ) + device.refresh_from_db() + self.assertIsNone(device.last_ip) + + call_command("clear_last_ip", *args, stdout=out, stderr=StringIO()) + self.assertIn("No active devices with last IP to clear.", out.getvalue()) + class TestWHOISInfoModel(CreateWHOISMixin, TestCase): def test_whois_model_fields_validation(self): @@ -301,10 +317,31 @@ def test_whois_model_fields_validation(self): with self.assertRaises(ValidationError): self._create_whois_info(asn="InvalidASN") + # Common validation checks for longitude and latitude + coordinates_cases = [ + (150.0, 100.0, "Latitude must be between -90 and 90 degrees."), + (150.0, -100.0, "Latitude must be between -90 and 90 degrees."), + (200.0, 80.0, "Longitude must be between -180 and 180 degrees."), + (-200.0, -80.0, "Longitude must be between -180 and 180 degrees."), + ] + for longitude, latitude, expected_msg in coordinates_cases: + with self.assertRaises(ValidationError) as context_manager: + point = Point(longitude, latitude, srid=4326) + self._create_whois_info(coordinates=point) + try: + self.assertEqual( + context_manager.exception.message_dict["coordinates"][0], + expected_msg, + ) + except AssertionError: + self.fail("ValidationError message not equal to expected message.") + -class TestWHOISTransaction(CreateWHOISMixin, TransactionTestCase): +class TestWHOISTransaction( + CreateWHOISMixin, WHOISTransactionMixin, TransactionTestCase +): _WHOIS_GEOIP_CLIENT = ( - "openwisp_controller.config.whois.tasks.geoip2_webservice.Client.city" + "openwisp_controller.config.whois.tasks.geoip2_webservice.Client" ) _WHOIS_TASKS_INFO_LOGGER = "openwisp_controller.config.whois.tasks.logger.info" _WHOIS_TASKS_WARN_LOGGER = "openwisp_controller.config.whois.tasks.logger.warning" @@ -316,76 +353,38 @@ def setUp(self): @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) @mock.patch("openwisp_controller.config.whois.tasks.fetch_whois_details.delay") - def test_whois_task_called(self, mocked_task): - org = self._get_org() + def test_whois_task_called(self, mocked_lookup_task): connect_whois_handlers() + self._task_called(mocked_lookup_task) - with self.subTest("task called when last_ip is public"): - with mock.patch("django.core.cache.cache.set") as mocked_set: - device = self._create_device(last_ip="172.217.22.14") - mocked_task.assert_called() - mocked_set.assert_called_once() - mocked_task.reset_mock() - - with self.subTest("task called when last_ip is changed and is public"): - with mock.patch("django.core.cache.cache.get") as mocked_get: - device.last_ip = "172.217.22.10" - device.save() - mocked_task.assert_called() - mocked_get.assert_called_once() - mocked_task.reset_mock() - - with self.subTest("task not called when last_ip is private"): - device.last_ip = "10.0.0.1" - device.save() - mocked_task.assert_not_called() - mocked_task.reset_mock() - - with self.subTest("task not called when last_ip has related WHOIS Info"): - device.last_ip = "172.217.22.10" + Device.objects.all().delete() # Clear existing devices + device = self._create_device() + with self.subTest( + "WHOIS lookup task not called when last_ip has related WhoIsInfo" + ): + device.organization.config_settings.whois_enabled = True + device.organization.config_settings.save() + device.last_ip = "172.217.22.14" self._create_whois_info(ip_address=device.last_ip) device.save() - mocked_task.assert_not_called() - mocked_task.reset_mock() - - with self.subTest("task not called when WHOIS is disabled"): - Device.objects.all().delete() - org.config_settings.whois_enabled = False - # Invalidates old org config settings cache - org.config_settings.save(update_fields=["whois_enabled"]) - device = self._create_device(last_ip="172.217.22.14") - mocked_task.assert_not_called() - mocked_task.reset_mock() - - with self.subTest("task called via DeviceChecksumView when WHOIS is enabled"): - org.config_settings.whois_enabled = True - # Invalidates old org config settings cache - org.config_settings.save(update_fields=["whois_enabled"]) - # config is required for checksum view to work - self._create_config(device=device) - # setting remote address field to a public IP to trigger WHOIS task - # since the view uses this header for tracking the device's IP - response = self.client.get( - reverse("controller:device_checksum", args=[device.pk]), - {"key": device.key}, - REMOTE_ADDR="172.217.22.10", - ) - self.assertEqual(response.status_code, 200) - mocked_task.assert_called() - mocked_task.reset_mock() + mocked_lookup_task.assert_not_called() + mocked_lookup_task.reset_mock() with self.subTest( - "task called via DeviceChecksumView when a device has no WHOIS record" + "WHOIS lookup task not called via DeviceChecksumView when " + "last_ip has related WhoIsInfo" ): WHOISInfo.objects.all().delete() + self._create_whois_info(ip_address=device.last_ip) + self._create_config(device=device) response = self.client.get( reverse("controller:device_checksum", args=[device.pk]), {"key": device.key}, REMOTE_ADDR=device.last_ip, ) self.assertEqual(response.status_code, 200) - mocked_task.assert_called() - mocked_task.reset_mock() + mocked_lookup_task.assert_not_called() + mocked_lookup_task.reset_mock() @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) @mock.patch("openwisp_controller.config.whois.tasks.fetch_whois_details.delay") @@ -429,6 +428,7 @@ def test_whois_multiple_orgs(self, mocked_task): {"key": device1.key}, REMOTE_ADDR="172.217.22.20", ) + device1.refresh_from_db() self.assertEqual(response.status_code, 200) mocked_task.assert_called() mocked_task.reset_mock() @@ -437,12 +437,13 @@ def test_whois_multiple_orgs(self, mocked_task): {"key": device2.key}, REMOTE_ADDR="172.217.22.30", ) + device2.refresh_from_db() self.assertEqual(response.status_code, 200) mocked_task.assert_not_called() mocked_task.reset_mock() with self.subTest( - "task called via DeviceChecksumView when a device has no WHOIS record" + "Task not called via DeviceChecksumView when a device has no WHOIS record" ): WHOISInfo.objects.all().delete() response = self.client.get( @@ -451,7 +452,7 @@ def test_whois_multiple_orgs(self, mocked_task): REMOTE_ADDR=device1.last_ip, ) self.assertEqual(response.status_code, 200) - mocked_task.assert_called() + mocked_task.assert_not_called() mocked_task.reset_mock() response = self.client.get( reverse("controller:device_checksum", args=[device2.pk]), @@ -489,18 +490,11 @@ def _verify_whois_details(instance, ip_address): instance.formatted_address, "Mountain View, United States, North America, 94043", ) + self.assertEqual(instance.coordinates.x, 150.0) + self.assertEqual(instance.coordinates.y, 50.0) # mocking the response from the geoip2 client - mock_response = mock.MagicMock() - mock_response.city.name = "Mountain View" - mock_response.country.name = "United States" - mock_response.continent.name = "North America" - mock_response.postal.code = "94043" - mock_response.traits.autonomous_system_organization = "Google LLC" - mock_response.traits.autonomous_system_number = 15169 - mock_response.traits.network = "172.217.22.0/24" - mock_response.location.time_zone = "America/Los_Angeles" - mock_client.return_value = mock_response + mock_client.return_value.city.return_value = self._mocked_client_response() with self.subTest("Test WHOIS create when device is created"): device = self._create_device(last_ip="172.217.22.14") @@ -514,6 +508,7 @@ def _verify_whois_details(instance, ip_address): with self.subTest( "Test WHOIS create & deletion of old record when last ip is updated" + " when no other devices are linked to the old ip address" ): old_ip_address = device.last_ip device.last_ip = "172.217.22.10" @@ -531,12 +526,61 @@ def _verify_whois_details(instance, ip_address): WHOISInfo.objects.filter(ip_address=old_ip_address).count(), 0 ) - with self.subTest("Test WHOIS delete when device is deleted"): + with self.subTest( + "Test WHOIS create & deletion of old record when last ip is updated" + " when other devices are linked to the old ip address" + ): + old_ip_address = device.last_ip + self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.11", + ) + device.last_ip = "172.217.22.11" + device.save() + self.assertEqual(mock_info.call_count, 1) + mock_info.reset_mock() + device.refresh_from_db() + + _verify_whois_details( + device.whois_service.get_device_whois_info(), device.last_ip + ) + + # details related to old ip address should be not be deleted + self.assertEqual( + WHOISInfo.objects.filter(ip_address=old_ip_address).count(), 1 + ) + + with self.subTest( + "Test WHOIS not deleted when device is deleted and" + " other active devices are linked to the last_ip" + ): ip_address = device.last_ip device.delete(check_deactivated=False) self.assertEqual(mock_info.call_count, 0) mock_info.reset_mock() + # WHOIS related to the device's last_ip should be deleted + self.assertEqual(WHOISInfo.objects.filter(ip_address=ip_address).count(), 1) + + Device.objects.all().delete() + with self.subTest( + "Test WHOIS deleted when device is deleted and" + " no other active devices are linked to the last_ip" + ): + device1 = self._create_device(last_ip="172.217.22.11") + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.11", + ) + device2.deactivate() + mock_info.reset_mock() + ip_address = device1.last_ip + device1.delete(check_deactivated=False) + self.assertEqual(mock_info.call_count, 0) + mock_info.reset_mock() + # WHOIS related to the device's last_ip should be deleted self.assertEqual(WHOISInfo.objects.filter(ip_address=ip_address).count(), 0) @@ -563,12 +607,13 @@ def assert_logging_on_exception( notification = notification_qs.first() self.assertEqual(notification.actor, device) self.assertEqual(notification.target, device) + self.assertEqual(notification.level, "error") self.assertEqual(notification.type, "generic_message") self.assertIn( "Failed to fetch WHOIS details for device", notification.message, ) - self.assertIn(device.last_ip, notification.description) + self.assertIn(device.last_ip, notification.rendered_description) mock_info.reset_mock() mock_warn.reset_mock() @@ -586,6 +631,16 @@ def assert_logging_on_exception( class TestWHOISSelenium(CreateWHOISMixin, SeleniumTestMixin, StaticLiveServerTestCase): @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) def test_whois_device_admin(self): + def _assert_no_js_errors(): + browser_logs = [] + for log in self.get_browser_logs(): + if self.browser == "chrome" and log["source"] != "console-api": + continue + elif log["message"] in ["wrong event specified: touchleave"]: + continue + browser_logs.append(log) + self.assertEqual(browser_logs, []) + whois_obj = self._create_whois_info() device = self._create_device(last_ip=whois_obj.ip_address) self.login() @@ -610,6 +665,7 @@ def test_whois_device_admin(self): self.assertIn(whois_obj.timezone, additional_text[1].text) self.assertIn(whois_obj.formatted_address, additional_text[2].text) self.assertIn(whois_obj.cidr, additional_text[3].text) + _assert_no_js_errors() with mock.patch.object(app_settings, "WHOIS_CONFIGURED", False): with self.subTest( @@ -619,6 +675,7 @@ def test_whois_device_admin(self): self.open(reverse("admin:config_device_change", args=[device.pk])) self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") + _assert_no_js_errors() with self.subTest( "WHOIS details not visible in device admin when WHOIS is disabled" @@ -629,6 +686,7 @@ def test_whois_device_admin(self): self.open(reverse("admin:config_device_change", args=[device.pk])) self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") + _assert_no_js_errors() with self.subTest( "WHOIS details not visible in device admin when WHOIS Info does not exist" @@ -640,3 +698,4 @@ def test_whois_device_admin(self): self.open(reverse("admin:config_device_change", args=[device.pk])) self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") + _assert_no_js_errors() diff --git a/openwisp_controller/config/whois/tests/utils.py b/openwisp_controller/config/whois/tests/utils.py new file mode 100644 index 000000000..7139ae883 --- /dev/null +++ b/openwisp_controller/config/whois/tests/utils.py @@ -0,0 +1,152 @@ +from unittest import mock + +from django.contrib.gis.geos import Point +from django.urls import reverse +from swapper import load_model + +from ...tests.utils import CreateConfigMixin + +Device = load_model("config", "Device") +WHOISInfo = load_model("config", "WHOISInfo") +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + + +class CreateWHOISMixin(CreateConfigMixin): + def _create_whois_info(self, **kwargs): + options = dict( + ip_address="172.217.22.14", + address={ + "city": "Mountain View", + "country": "United States", + "continent": "North America", + "postal": "94043", + }, + asn="15169", + isp="Google LLC", + timezone="America/Los_Angeles", + cidr="172.217.22.0/24", + coordinates=Point(150, 50, srid=4326), + ) + + options.update(kwargs) + w = WHOISInfo(**options) + w.full_clean() + w.save() + return w + + def setUp(self): + super().setUp() + OrganizationConfigSettings.objects.create( + organization=self._get_org(), whois_enabled=True + ) + + +class WHOISTransactionMixin: + @staticmethod + def _mocked_client_response(): + mock_response = mock.MagicMock() + mock_response.city.name = "Mountain View" + mock_response.country.name = "United States" + mock_response.continent.name = "North America" + mock_response.postal.code = "94043" + mock_response.traits.autonomous_system_organization = "Google LLC" + mock_response.traits.autonomous_system_number = 15169 + mock_response.traits.network = "172.217.22.0/24" + mock_response.location.time_zone = "America/Los_Angeles" + mock_response.location.latitude = 50 + mock_response.location.longitude = 150 + return mock_response + + def _task_called(self, mocked_task, task_name="WHOIS lookup"): + org = self._get_org() + + with self.subTest(f"{task_name} task called when last_ip is public"): + with mock.patch("django.core.cache.cache.set") as mocked_set: + device = self._create_device(last_ip="172.217.22.14") + mocked_task.assert_called() + mocked_set.assert_called_once() + mocked_task.reset_mock() + + with self.subTest( + f"{task_name} task called when last_ip is changed and is public" + ): + with mock.patch("django.core.cache.cache.get") as mocked_get, mock.patch( + "django.core.cache.cache.set" + ) as mocked_set: + device.last_ip = "172.217.22.10" + device.save() + mocked_task.assert_called() + mocked_set.assert_not_called() + mocked_get.assert_called_once() + mocked_task.reset_mock() + + with self.subTest(f"{task_name} task not called when last_ip not updated"): + device.name = "default.test.Device2" + device.save() + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest(f"{task_name} task not called when last_ip is private"): + device.last_ip = "10.0.0.1" + device.save() + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest(f"{task_name} task not called when WHOIS is disabled"): + Device.objects.all().delete() + org.config_settings.whois_enabled = False + # Invalidates old org config settings cache + org.config_settings.save(update_fields=["whois_enabled"]) + device = self._create_device(last_ip="172.217.22.14") + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest( + f"{task_name} task called via DeviceChecksumView when WHOIS is enabled" + ): + org.config_settings.whois_enabled = True + # Invalidates old org config settings cache + org.config_settings.save(update_fields=["whois_enabled"]) + # config is required for checksum view to work + device.refresh_from_db() + self._create_config(device=device) + # setting remote address field to a public IP to trigger WHOIS task + # since the view uses this header for tracking the device's IP + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR="172.217.22.10", + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_called() + mocked_task.reset_mock() + + with self.subTest( + f"{task_name} task called via DeviceChecksumView for no WHOIS record" + ): + WHOISInfo.objects.all().delete() + device.refresh_from_db() + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest( + f"{task_name} task not called via DeviceChecksumView when WHOIS is disabled" + ): + WHOISInfo.objects.all().delete() + device.refresh_from_db() + org.config_settings.whois_enabled = False + org.config_settings.save(update_fields=["whois_enabled"]) + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_not_called() + mocked_task.reset_mock() diff --git a/openwisp_controller/config/whois/tests_utils.py b/openwisp_controller/config/whois/tests_utils.py deleted file mode 100644 index fd31ff6cf..000000000 --- a/openwisp_controller/config/whois/tests_utils.py +++ /dev/null @@ -1,36 +0,0 @@ -from swapper import load_model - -from ..tests.utils import CreateConfigMixin - -Device = load_model("config", "Device") -WHOISInfo = load_model("config", "WHOISInfo") -OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") - - -class CreateWHOISMixin(CreateConfigMixin): - def _create_whois_info(self, **kwargs): - options = dict( - ip_address="172.217.22.14", - address={ - "city": "Mountain View", - "country": "United States", - "continent": "North America", - "postal": "94043", - }, - asn="15169", - isp="Google LLC", - timezone="America/Los_Angeles", - cidr="172.217.22.0/24", - ) - - options.update(kwargs) - w = WHOISInfo(**options) - w.full_clean() - w.save() - return w - - def setUp(self): - super().setUp() - OrganizationConfigSettings.objects.create( - organization=self._get_org(), whois_enabled=True - ) diff --git a/openwisp_controller/config/whois/utils.py b/openwisp_controller/config/whois/utils.py index 51919af08..4a955810e 100644 --- a/openwisp_controller/config/whois/utils.py +++ b/openwisp_controller/config/whois/utils.py @@ -1,13 +1,66 @@ +from django.utils.translation import gettext_lazy as _ +from openwisp_notifications.signals import notify from swapper import load_model from .. import settings as app_settings -from .serializers import WHOISSerializer -from .service import WHOISService -Device = load_model("config", "Device") +MESSAGE_MAP = { + "whois_device_error": { + "type": "generic_message", + "level": "error", + "message": _( + "Failed to fetch WHOIS details for device" + " [{notification.target}]({notification.target_link})" + ), + "description": _("WHOIS details could not be fetched for ip: {ip_address}."), + }, + "estimated_location_error": { + "level": "error", + "type": "estimated_location_info", + "message": _( + "Unable to create estimated location for device " + "[{notification.target}]({notification.target_link}). " + "Please assign/create a location manually." + ), + "description": _("Multiple devices found for IP: {ip_address}"), + }, + "estimated_location_created": { + "type": "estimated_location_info", + "description": _("Estimated Location {notification.verb} for IP: {ip_address}"), + }, + "estimated_location_updated": { + "type": "estimated_location_info", + "message": _( + "Estimated location [{notification.actor}]({notification.actor_link})" + " for device" + " [{notification.target}]({notification.target_link})" + " updated successfully." + ), + "description": _("Estimated Location updated for IP: {ip_address}"), + }, +} + + +def send_whois_task_notification(device_pk, notify_type, actor=None): + Device = load_model("config", "Device") + + device = Device.objects.get(pk=device_pk) + notify_details = MESSAGE_MAP[notify_type] + notify.send( + sender=actor or device, + target=device, + action_object=device, + ip_address=device.last_ip, + **notify_details, + ) def get_whois_info(pk): + from .serializers import WHOISSerializer + from .service import WHOISService + + Device = load_model("config", "Device") + if not app_settings.WHOIS_CONFIGURED or not pk: return None device = ( diff --git a/openwisp_controller/geo/admin.py b/openwisp_controller/geo/admin.py index 9064d657a..ffab0c99b 100644 --- a/openwisp_controller/geo/admin.py +++ b/openwisp_controller/geo/admin.py @@ -12,6 +12,8 @@ ) from swapper import load_model +from openwisp_controller.config import settings as config_app_settings +from openwisp_controller.config.whois.service import WHOISService from openwisp_users.multitenancy import MultitenantOrgFilter from ..admin import MultitenantAdminMixin @@ -98,6 +100,29 @@ class LocationAdmin(MultitenantAdminMixin, AbstractLocationAdmin): form = LocationForm inlines = [FloorPlanInline] list_select_related = ("organization",) + change_form_template = "admin/geo/location/change_form.html" + + def get_fields(self, request, obj=None): + fields = super().get_fields(request, obj) + org_id = obj.organization_id if obj else None + if not WHOISService.check_estimate_location_configured(org_id): + if "is_estimated" in fields: + fields.remove("is_estimated") + return fields + + def get_readonly_fields(self, request, obj=None): + fields = super().get_readonly_fields(request, obj) + org_id = obj.organization_id if obj else None + if obj and WHOISService.check_estimate_location_configured(org_id): + fields = fields + ("is_estimated",) + return fields + + def change_view(self, request, object_id, form_url="", extra_context=None): + obj = self.get_object(request, object_id) + org_id = obj.organization_id if obj else None + estimated_configured = WHOISService.check_estimate_location_configured(org_id) + extra_context = {"estimated_configured": estimated_configured} + return super().change_view(request, object_id, form_url, extra_context) LocationAdmin.list_display.insert(1, "organization") @@ -122,16 +147,39 @@ class DeviceLocationFilter(admin.SimpleListFilter): title = _("has geographic position set?") parameter_name = "with_geo" + def __init__(self, request, params, model, model_admin): + super().__init__(request, params, model, model_admin) + if config_app_settings.WHOIS_CONFIGURED: + self.title = _("geographic position") + def lookups(self, request, model_admin): + if config_app_settings.WHOIS_CONFIGURED: + return ( + ("outdoor", _("Outdoor")), + ("indoor", _("Indoor")), + ("estimated", _("Estimated")), + ("false", _("No Location")), + ) return ( ("true", _("Yes")), ("false", _("No")), ) def queryset(self, request, queryset): - if self.value(): - return queryset.filter(devicelocation__isnull=self.value() == "false") - return queryset + value = self.value() + if not value: + return queryset + if config_app_settings.WHOIS_CONFIGURED: + if value == "estimated": + return queryset.filter(devicelocation__location__is_estimated=True) + elif value in ("indoor", "outdoor"): + # estimated locations are outdoor by default + # so we need to exclude them from the result + return queryset.filter( + devicelocation__location__type=value, + devicelocation__location__is_estimated=False, + ) + return queryset.filter(devicelocation__isnull=self.value() == "false") # Prepend DeviceLocationInline to config.DeviceAdminExportable diff --git a/openwisp_controller/geo/api/filters.py b/openwisp_controller/geo/api/filters.py index 93a7c366e..2be821e58 100644 --- a/openwisp_controller/geo/api/filters.py +++ b/openwisp_controller/geo/api/filters.py @@ -1,6 +1,7 @@ from django.utils.translation import gettext_lazy as _ from django_filters import rest_framework as filters +from openwisp_controller.config import settings as config_app_settings from openwisp_controller.config.api.filters import ( DeviceListFilter as BaseDeviceListFilter, ) @@ -30,6 +31,20 @@ def filter_devicelocation(self, queryset, name, value): # Returns list of device that have devicelocation objects return queryset.exclude(devicelocation__isnull=value) + def filter_is_estimated(self, queryset, name, value): + return queryset.filter(devicelocation__location__is_estimated=value) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if config_app_settings.WHOIS_CONFIGURED: + self.filters["geo_is_estimated"] = filters.BooleanFilter( + field_name="devicelocation__location__is_estimated", + method=self.filter_is_estimated, + ) + self.filters["geo_is_estimated"].label = _( + "Is geographic location estimated?" + ) + class Meta: model = BaseDeviceListFilter.Meta.model geo_fields = [ diff --git a/openwisp_controller/geo/api/serializers.py b/openwisp_controller/geo/api/serializers.py index e4a800d90..0626efdab 100644 --- a/openwisp_controller/geo/api/serializers.py +++ b/openwisp_controller/geo/api/serializers.py @@ -11,6 +11,10 @@ from openwisp_utils.api.serializers import ValidatedModelSerializer from ...serializers import BaseSerializer +from ..estimated_location.mixins import ( + EstimatedLocationGeoJsonSerializer, + EstimatedLocationMixin, +) Device = load_model("config", "Device") Location = load_model("geo", "Location") @@ -31,7 +35,9 @@ class Meta: fields = "__all__" -class GeoJsonLocationSerializer(gis_serializers.GeoFeatureModelSerializer): +class GeoJsonLocationSerializer( + EstimatedLocationGeoJsonSerializer, gis_serializers.GeoFeatureModelSerializer +): device_count = IntegerField() class Meta: @@ -126,7 +132,7 @@ class Meta: read_only_fields = ("name",) -class LocationSerializer(BaseSerializer): +class LocationSerializer(EstimatedLocationMixin, BaseSerializer): floorplan = FloorPlanLocationSerializer(required=False, allow_null=True) class Meta: @@ -225,7 +231,9 @@ def update(self, instance, validated_data): return super().update(instance, validated_data) -class NestedtLocationSerializer(gis_serializers.GeoFeatureModelSerializer): +class NestedtLocationSerializer( + EstimatedLocationGeoJsonSerializer, gis_serializers.GeoFeatureModelSerializer +): class Meta: model = Location geo_field = "geometry" diff --git a/openwisp_controller/geo/api/views.py b/openwisp_controller/geo/api/views.py index b521c3d27..9b64ca084 100644 --- a/openwisp_controller/geo/api/views.py +++ b/openwisp_controller/geo/api/views.py @@ -11,6 +11,7 @@ from rest_framework_gis.pagination import GeoJsonPagination from swapper import load_model +from openwisp_controller.config import settings as config_app_settings from openwisp_controller.config.api.views import DeviceListCreateView from openwisp_users.api.filters import OrganizationManagedFilter from openwisp_users.api.mixins import FilterByOrganizationManaged, FilterByParentManaged @@ -51,6 +52,13 @@ class Meta(OrganizationManagedFilter.Meta): model = Location fields = OrganizationManagedFilter.Meta.fields + ["is_mobile", "type"] + def __init__(self, data=None, queryset=None, *, request=None, prefix=None): + super().__init__(data, queryset, request=request, prefix=prefix) + if config_app_settings.WHOIS_CONFIGURED: + self.filters["is_estimated"] = filters.BooleanFilter( + field_name="is_estimated" + ) + class FloorPlanOrganizationFilter(OrganizationManagedFilter): class Meta(OrganizationManagedFilter.Meta): diff --git a/openwisp_controller/geo/apps.py b/openwisp_controller/geo/apps.py index 90a626f8e..0e2cc6264 100644 --- a/openwisp_controller/geo/apps.py +++ b/openwisp_controller/geo/apps.py @@ -14,6 +14,8 @@ from openwisp_utils.admin_theme import register_dashboard_chart from openwisp_utils.admin_theme.menu import register_menu_group +from .estimated_location.handlers import register_estimated_location_notification_types + class GeoConfig(LociConfig): name = "openwisp_controller.geo" @@ -27,6 +29,7 @@ def ready(self): super().ready() self.register_dashboard_charts() self.register_menu_groups() + register_estimated_location_notification_types() if getattr(settings, "TESTING", False): self._add_params_to_test_config() diff --git a/openwisp_controller/geo/base/models.py b/openwisp_controller/geo/base/models.py index c7bc59798..c1225f838 100644 --- a/openwisp_controller/geo/base/models.py +++ b/openwisp_controller/geo/base/models.py @@ -1,4 +1,9 @@ +import re + from django.contrib.gis.db import models +from django.core.exceptions import ValidationError +from django.utils.translation import gettext +from django.utils.translation import gettext_lazy as _ from django_loci.base.models import ( AbstractFloorPlan, AbstractLocation, @@ -6,13 +11,81 @@ ) from swapper import get_model_name +from openwisp_controller.config.whois.service import WHOISService from openwisp_users.mixins import OrgMixin, ValidateOrgMixin class BaseLocation(OrgMixin, AbstractLocation): + _changed_checked_fields = ["is_estimated", "address", "geometry"] + + is_estimated = models.BooleanField( + default=False, + help_text=_("Whether the location's coordinates are estimated."), + ) + class Meta(AbstractLocation.Meta): abstract = True + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._set_initial_values_for_changed_checked_fields() + + def _set_initial_values_for_changed_checked_fields(self): + deferred_fields = self.get_deferred_fields() + for field in self._changed_checked_fields: + if field in deferred_fields: + setattr(self, f"_initial_{field}", models.DEFERRED) + else: + setattr(self, f"_initial_{field}", getattr(self, field)) + + def clean(self): + # Raise validation error if `is_estimated` is True but estimated feature is + # disabled. + if ( + (self._state.adding or self._initial_is_estimated != self.is_estimated) + and self.is_estimated + and not WHOISService.check_estimate_location_configured( + self.organization_id + ) + ): + raise ValidationError( + { + "is_estimated": _( + "Estimated Location feature required to be configured." + ) + } + ) + return super().clean() + + def save(self, *args, _set_estimated=False, **kwargs): + """ + Save the location object with special handling for estimated locations. + + Parameters: + _set_estimated: Boolean flag to indicate if this save is being performed + by the estimated location system. When False (default), + manual edits will clear the estimated status. + *args, **kwargs: Arguments passed to the parent save method. + + Returns: + The result of the parent save method. + """ + if WHOISService.check_estimate_location_configured(self.organization_id): + if not _set_estimated and ( + self._initial_address != self.address + or self._initial_geometry != self.geometry + ): + self.is_estimated = False + estimated_string = gettext("Estimated Location") + if self.name and estimated_string in self.name: + # remove string starting with "(Estimated Location" + self.name = re.sub( + rf"\s\({estimated_string}.*", "", self.name, flags=re.IGNORECASE + ) + else: + self.is_estimated = self._initial_is_estimated + return super().save(*args, **kwargs) + class BaseFloorPlan(OrgMixin, AbstractFloorPlan): location = models.ForeignKey(get_model_name("geo", "Location"), models.CASCADE) diff --git a/openwisp_controller/geo/estimated_location/__init__.py b/openwisp_controller/geo/estimated_location/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/geo/estimated_location/handlers.py b/openwisp_controller/geo/estimated_location/handlers.py new file mode 100644 index 000000000..60ea6f44a --- /dev/null +++ b/openwisp_controller/geo/estimated_location/handlers.py @@ -0,0 +1,41 @@ +from django.utils.translation import gettext_lazy as _ +from openwisp_notifications.types import register_notification_type +from swapper import load_model + +from openwisp_controller.config import settings as config_app_settings + + +def register_estimated_location_notification_types(): + """ + Register the notification types used by the Estimated Location module. + This is necessary to ensure that the notifications are properly configured. + """ + if not config_app_settings.WHOIS_CONFIGURED: + return + + Device = load_model("config", "Device") + Location = load_model("geo", "Location") + + register_notification_type( + "estimated_location_info", + { + "verbose_name": _("Estimated Location INFO"), + "verb": _("created"), + "level": "info", + "email_subject": _( + "Estimated Location: Created for device {notification.target}" + ), + "message": _( + "Estimated location [{notification.actor}]({notification.actor_link})" + " for device" + " [{notification.target}]({notification.target_link})" + " {notification.verb} successfully." + ), + "target_link": ( + "openwisp_controller.geo.estimated_location.utils" + ".get_device_location_notification_target_url" + ), + "email_notification": False, + }, + models=[Device, Location], + ) diff --git a/openwisp_controller/geo/estimated_location/mixins.py b/openwisp_controller/geo/estimated_location/mixins.py new file mode 100644 index 000000000..f4d2fc93b --- /dev/null +++ b/openwisp_controller/geo/estimated_location/mixins.py @@ -0,0 +1,30 @@ +from openwisp_controller.config.whois.service import WHOISService + + +class EstimatedLocationMixin: + """ + Serializer mixin to add estimated location field to the serialized data + if the estimated location feature is configured and enabled for the organization. + """ + + def to_representation(self, obj): + data = super().to_representation(obj) + if WHOISService.check_estimate_location_configured(obj.organization_id): + data["is_estimated"] = obj.is_estimated + else: + data.pop("is_estimated", None) + return data + + +class EstimatedLocationGeoJsonSerializer(EstimatedLocationMixin): + """ + Extension of EstimatedLocationMixin for GeoJSON serialization. + """ + + def to_representation(self, obj): + data = super(EstimatedLocationMixin, self).to_representation(obj) + if WHOISService.check_estimate_location_configured(obj.organization_id): + data["properties"]["is_estimated"] = obj.is_estimated + else: + data["properties"].pop("is_estimated", None) + return data diff --git a/openwisp_controller/geo/estimated_location/tasks.py b/openwisp_controller/geo/estimated_location/tasks.py new file mode 100644 index 000000000..8ec080509 --- /dev/null +++ b/openwisp_controller/geo/estimated_location/tasks.py @@ -0,0 +1,170 @@ +import logging + +from celery import shared_task +from django.db import transaction +from swapper import load_model + +from openwisp_controller.config.whois.utils import send_whois_task_notification + +logger = logging.getLogger(__name__) + + +@shared_task +def manage_estimated_locations(device_pk, ip_address): + """ + Creates/updates estimated location for a device based on the latitude and + longitude or attaches an existing location. + Existing location here means a location of another device whose last_ip matches + the given ip_address. + Does not alters the existing location if it is not estimated. + + - If the current device has no location or location is estimate, either update + to an existing location; if it exists, else + + - A new location is created if current device has no location, or + if it does; it is updated using coords from WHOIS record if it is estimated. + + In case of multiple devices with same last_ip, the task will send a notification + to the user to resolve the conflict manually. + """ + Device = load_model("config", "Device") + Location = load_model("geo", "Location") + WHOISInfo = load_model("config", "WHOISInfo") + DeviceLocation = load_model("geo", "DeviceLocation") + + def _create_estimated_location(device_location, location_defaults): + with transaction.atomic(): + location = Location(**location_defaults, is_estimated=True) + location.full_clean() + location.save(_set_estimated=True) + device_location.location = location + device_location.full_clean() + device_location.save() + logger.info( + f"Estimated location saved successfully for {device_pk}" + f" for IP: {ip_address}" + ) + send_whois_task_notification( + device_pk=device_pk, + notify_type="estimated_location_created", + actor=location, + ) + + def _update_or_create_estimated_location( + device_location, whois_obj, attached_devices_exists=False + ): + # Used to update an existing location if it is estimated + # or create a new one if it doesn't exist + if whois_obj and whois_obj.coordinates: + location_defaults = { + **whois_obj._get_defaults_for_estimated_location(), + "organization_id": device.organization_id, + } + if current_location and current_location.is_estimated: + if attached_devices_exists: + # If there are other devices attached to the current location, + # we do not update it, but create a new one. + _create_estimated_location(device_location, location_defaults) + return + update_fields = [] + for attr, value in location_defaults.items(): + if getattr(current_location, attr) != value: + setattr(current_location, attr, value) + update_fields.append(attr) + if update_fields: + current_location.save( + update_fields=update_fields, _set_estimated=True + ) + logger.info( + f"Estimated location saved successfully for {device_pk}" + f" for IP: {ip_address}" + ) + send_whois_task_notification( + device_pk=device_pk, + notify_type="estimated_location_updated", + actor=current_location, + ) + elif not current_location: + # If there is no current location, we create a new one. + _create_estimated_location(device_location, location_defaults) + else: + logger.warning( + f"Coordinates not available for {device_pk} for IP: {ip_address}." + " Estimated location cannot be determined." + ) + return + + def _handle_attach_existing_location( + device, device_location, whois_obj, attached_devices_exists=False + ): + # For handling the case when WHOIS already exists for device's new last_ip + # then we attach the location of the device with same last_ip if it exists. + devices_with_location = ( + Device.objects.select_related("devicelocation") + .filter(organization_id=device.organization_id) + .filter(last_ip=ip_address, devicelocation__location__isnull=False) + .exclude(pk=device_pk) + ) + # If there are multiple devices with same last_ip then we need to inform + # the user to resolve the conflict manually. + if devices_with_location.count() > 1: + send_whois_task_notification( + device_pk=device_pk, notify_type="estimated_location_error" + ) + logger.error( + "Multiple devices with locations found with same " + f"last_ip {ip_address}. Please resolve the conflict manually." + ) + return + first_device = devices_with_location.first() + # If existing devices with same last_ip do not have any location + # then we create a new location based on WHOIS data. + if not first_device: + _update_or_create_estimated_location( + device_location, whois_obj, attached_devices_exists + ) + return + existing_location = first_device.devicelocation.location + # We need to remove any existing estimated location of the device + if current_location and not attached_devices_exists: + current_location.delete() + device_location.location = existing_location + device_location.full_clean() + device_location.save() + logger.info( + f"Estimated location saved successfully for {device_pk}" + f" for IP: {ip_address}" + ) + send_whois_task_notification( + device_pk=device_pk, + notify_type="estimated_location_updated", + actor=existing_location, + ) + + whois_obj = WHOISInfo.objects.filter(ip_address=ip_address).first() + device = ( + Device.objects.select_related("devicelocation__location", "organization") + .only("organization_id", "devicelocation") + .get(pk=device_pk) + ) + + if not (device_location := getattr(device, "devicelocation", None)): + device_location = DeviceLocation(content_object=device) + + attached_devices_exists = False + if current_location := device_location.location: + attached_devices_exists = ( + Device.objects.filter(devicelocation__location_id=current_location.pk) + .exclude(pk=device_pk) + .exists() + ) + + if not current_location or current_location.is_estimated: + _handle_attach_existing_location( + device, device_location, whois_obj, attached_devices_exists + ) + else: + logger.info( + f"Non Estimated location already set for {device_pk}. Update" + f" location manually as per IP: {ip_address}" + ) diff --git a/openwisp_controller/geo/estimated_location/tests/__init__.py b/openwisp_controller/geo/estimated_location/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/geo/estimated_location/tests/tests.py b/openwisp_controller/geo/estimated_location/tests/tests.py new file mode 100644 index 000000000..7b8eb1751 --- /dev/null +++ b/openwisp_controller/geo/estimated_location/tests/tests.py @@ -0,0 +1,761 @@ +import contextlib +import importlib +from unittest import mock + +from django.contrib.gis.geos import GEOSGeometry +from django.core.exceptions import ImproperlyConfigured, ValidationError +from django.test import TestCase, TransactionTestCase, override_settings +from django.urls import reverse +from openwisp_notifications.types import unregister_notification_type +from swapper import load_model + +from openwisp_controller.config import settings as config_app_settings +from openwisp_controller.config.whois.handlers import connect_whois_handlers +from openwisp_controller.config.whois.tests.utils import WHOISTransactionMixin + +from ....tests.utils import TestAdminMixin +from ...tests.utils import TestGeoMixin +from ..handlers import register_estimated_location_notification_types +from .utils import TestEstimatedLocationMixin + +Device = load_model("config", "Device") +Location = load_model("geo", "Location") +DeviceLocation = load_model("geo", "DeviceLocation") +WHOISInfo = load_model("config", "WHOISInfo") +Notification = load_model("openwisp_notifications", "Notification") +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + +notification_qs = Notification.objects.all() + + +class TestEstimatedLocation(TestAdminMixin, TestCase): + @override_settings( + OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT="test_account", + OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY="test_key", + ) + def test_estimated_location_configuration_setting(self): + # reload app_settings to apply the overridden settings + importlib.reload(config_app_settings) + with self.subTest( + "ImproperlyConfigured raised when ESTIMATED_LOCATION_ENABLED is True " + "and WHOIS_ENABLED is False globally" + ): + with override_settings( + OPENWISP_CONTROLLER_WHOIS_ENABLED=False, + OPENWISP_CONTROLLER_ESTIMATED_LOCATION_ENABLED=True, + ): + with self.assertRaises(ImproperlyConfigured): + # reload app_settings to apply the overridden settings + importlib.reload(config_app_settings) + + with self.subTest( + "Test WHOIS not enabled does not allow enabling Estimated Location" + ): + org_settings_obj = OrganizationConfigSettings(organization=self._get_org()) + with self.assertRaises(ValidationError) as context_manager: + org_settings_obj.whois_enabled = False + org_settings_obj.estimated_location_enabled = True + org_settings_obj.full_clean() + try: + self.assertEqual( + context_manager.exception.message_dict[ + "estimated_location_enabled" + ][0], + "Estimated Location feature requires " + + "WHOIS Lookup feature to be enabled.", + ) + except AssertionError: + self.fail("ValidationError message not equal to expected message.") + + with self.subTest( + "Test Estimated Location field visible on admin when " + "WHOIS_CONFIGURED is True" + ): + self._login() + org = self._get_org() + url = reverse( + "admin:openwisp_users_organization_change", + args=[org.pk], + ) + response = self.client.get(url) + self.assertContains( + response, 'name="config_settings-0-estimated_location_enabled"' + ) + + with override_settings( + OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT=None, + OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY=None, + ): + importlib.reload(config_app_settings) + with self.subTest( + "Test Estimated Location field hidden on admin when " + "WHOIS_CONFIGURED is False" + ): + self._login() + org = self._get_org() + url = reverse( + "admin:openwisp_users_organization_change", + args=[org.pk], + ) + response = self.client.get(url) + self.assertNotContains( + response, 'name="config_settings-0-estimated_location_enabled"' + ) + + +class TestEstimatedLocationField(TestEstimatedLocationMixin, TestGeoMixin, TestCase): + location_model = Location + + def test_estimated_location_field(self): + org = self._get_org() + org.config_settings.estimated_location_enabled = False + org.config_settings.save() + org.refresh_from_db() + with self.assertRaises(ValidationError) as context_manager: + self._create_location(organization=org, is_estimated=True) + try: + self.assertEqual( + context_manager.exception.message_dict["is_estimated"][0], + "Estimated Location feature required to be configured.", + ) + except AssertionError: + self.fail("ValidationError message not equal to expected message.") + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + def test_estimated_location_admin(self): + connect_whois_handlers() + admin = self._create_admin() + self.client.force_login(admin) + org = self._get_org() + location = self._create_location(organization=org, is_estimated=True) + path = reverse("admin:geo_location_change", args=[location.pk]) + response = self.client.get(path) + self.assertContains(response, "field-is_estimated") + self.assertContains( + response, "Whether the location's coordinates are estimated." + ) + org.config_settings.estimated_location_enabled = False + org.config_settings.save() + response = self.client.get(path) + self.assertNotContains(response, "field-is_estimated") + self.assertNotContains( + response, "Whether the location's coordinates are estimated." + ) + + +class TestEstimatedLocationTransaction( + TestEstimatedLocationMixin, WHOISTransactionMixin, TransactionTestCase +): + _WHOIS_GEOIP_CLIENT = ( + "openwisp_controller.config.whois.tasks.geoip2_webservice.Client" + ) + _ESTIMATED_LOCATION_INFO_LOGGER = ( + "openwisp_controller.geo.estimated_location.tasks.logger.info" + ) + _ESTIMATED_LOCATION_ERROR_LOGGER = ( + "openwisp_controller.geo.estimated_location.tasks.logger.error" + ) + + def setUp(self): + super().setUp() + self.admin = self._get_admin() + # Unregister the notification type if it was previously registered + with contextlib.suppress(ImproperlyConfigured): + unregister_notification_type("estimated_location_info") + with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True): + register_estimated_location_notification_types() + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch( + "openwisp_controller.geo.estimated_location.tasks.manage_estimated_locations.delay" # noqa + ) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_estimated_location_task_called( + self, mocked_client, mocked_estimated_location_task + ): + connect_whois_handlers() + mocked_client.return_value.city.return_value = self._mocked_client_response() + + self._task_called( + mocked_estimated_location_task, task_name="Estimated location" + ) + + Device.objects.all().delete() + device = self._create_device() + with self.subTest( + "Estimated location task called when last_ip has related WhoIsInfo" + ): + with mock.patch("django.core.cache.cache.get") as mocked_get, mock.patch( + "django.core.cache.cache.set" + ) as mocked_set: + device.organization.config_settings.whois_enabled = True + device.organization.config_settings.estimated_location_enabled = True + device.organization.config_settings.save() + device.last_ip = "172.217.22.14" + self._create_whois_info(ip_address=device.last_ip) + device.save() + mocked_set.assert_not_called() + # The cache `get` is called twice, once for `whois_enabled` and + # once for `estimated_location_enabled` + mocked_get.assert_called() + mocked_estimated_location_task.assert_called() + mocked_estimated_location_task.reset_mock() + + with self.subTest( + "Estimated location task not called via DeviceChecksumView when " + "last_ip has no related WhoIsInfo" + ): + WHOISInfo.objects.all().delete() + self._create_whois_info(ip_address=device.last_ip) + self._create_config(device=device) + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_estimated_location_task.assert_not_called() + mocked_estimated_location_task.reset_mock() + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_ESTIMATED_LOCATION_INFO_LOGGER) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_estimated_location_creation_and_update(self, mock_client, mock_info): + connect_whois_handlers() + + def _verify_location_details(device, mocked_response): + location = device.devicelocation.location + mocked_location = mocked_response.location + address = ", ".join( + [ + mocked_response.city.name, + mocked_response.country.name, + mocked_response.continent.name, + mocked_response.postal.code, + ] + ) + ip_address = mocked_response.ip_address or device.last_ip + location_name = ( + ",".join(address.split(",")[:2]) + + f" (Estimated Location: {ip_address})" + ) + self.assertEqual(location.name, location_name) + self.assertEqual(location.address, address) + self.assertEqual( + location.geometry, + GEOSGeometry( + f"POINT({mocked_location.longitude} {mocked_location.latitude})", + srid=4326, + ), + ) + + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + + with self.subTest("Test Estimated location created when device is created"): + device = self._create_device(last_ip="172.217.22.14") + + location = device.devicelocation.location + mocked_response.ip_address = device.last_ip + self.assertEqual(location.is_estimated, True) + self.assertEqual(location.is_mobile, False) + self.assertEqual(location.type, "outdoor") + _verify_location_details(device, mocked_response) + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device.pk}" + f" for IP: {device.last_ip}" + ) + mock_info.reset_mock() + + with self.subTest("Test Estimated location updated when last ip is updated"): + device.last_ip = "172.217.22.10" + mocked_response.location.latitude = 50 + mocked_response.location.longitude = 150 + mocked_response.city.name = "New City" + mock_client.return_value.city.return_value = mocked_response + device.save() + device.refresh_from_db() + + location = device.devicelocation.location + mocked_response.ip_address = device.last_ip + self.assertEqual(location.is_estimated, True) + self.assertEqual(location.is_mobile, False) + self.assertEqual(location.type, "outdoor") + _verify_location_details(device, mocked_response) + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device.pk}" + f" for IP: {device.last_ip}" + ) + mock_info.reset_mock() + + with self.subTest( + "Test Non Estimated Location not updated when last ip is updated" + ): + mocked_response.ip_address = device.last_ip + device.last_ip = "172.217.22.11" + device.devicelocation.location.is_estimated = False + mock_client.return_value.city.return_value = self._mocked_client_response() + device.devicelocation.location.save(_set_estimated=True) + device.save() + device.refresh_from_db() + + location = device.devicelocation.location + self.assertEqual(location.is_estimated, False) + self.assertEqual(location.is_mobile, False) + self.assertEqual(location.type, "outdoor") + _verify_location_details(device, mocked_response) + mock_info.assert_called_once_with( + f"Non Estimated location already set for {device.pk}. Update" + f" location manually as per IP: {device.last_ip}" + ) + mock_info.reset_mock() + + with self.subTest( + "Test location shared for same IP when new device's location does not exist" + ): + Device.objects.all().delete() + device1 = self._create_device(last_ip="172.217.22.10") + mock_info.reset_mock() + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.10", + ) + + self.assertEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device2.pk}" + f" for IP: {device2.last_ip}" + ) + mock_info.reset_mock() + + with self.subTest( + "Test location shared for same IP when new device's location is estimated" + ): + Device.objects.all().delete() + device1 = self._create_device(last_ip="172.217.22.10") + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.11", + ) + mock_info.reset_mock() + old_location = device2.devicelocation.location + device2.last_ip = "172.217.22.10" + device2.save() + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device2.pk}" + f" for IP: {device2.last_ip}" + ) + device2.refresh_from_db() + + self.assertEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + self.assertEqual(Location.objects.filter(pk=old_location.pk).count(), 0) + mock_info.reset_mock() + + with self.subTest( + "Test location not shared for same IP when new " + "device's location is not estimated" + ): + Device.objects.all().delete() + device1 = self._create_device(last_ip="172.217.22.10") + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.11", + ) + mock_info.reset_mock() + old_location = device2.devicelocation.location + old_location.is_estimated = False + old_location.save() + device2.last_ip = "172.217.22.10" + device2.save() + mock_info.assert_called_once_with( + f"Non Estimated location already set for {device2.pk}. Update" + f" location manually as per IP: {device2.last_ip}" + ) + device2.refresh_from_db() + + self.assertNotEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + self.assertEqual(Location.objects.filter(pk=old_location.pk).count(), 1) + mock_info.reset_mock() + + with self.subTest( + "Shared location not updated when either device's last_ip changes. " + "New location created for device with updated last_ip" + ): + Device.objects.all().delete() + device1 = self._create_device(last_ip="172.217.22.10") + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.10", + ) + mock_info.reset_mock() + self.assertEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + device2.last_ip = "172.217.22.11" + device2.save() + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device2.pk}" + f" for IP: {device2.last_ip}" + ) + device2.refresh_from_db() + self.assertNotEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + mock_info.reset_mock() + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_ESTIMATED_LOCATION_INFO_LOGGER) + @mock.patch(_ESTIMATED_LOCATION_ERROR_LOGGER) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_estimated_location_notification(self, mock_client, mock_error, mock_info): + def _verify_notification(device, messages, notify_level="info"): + self.assertEqual(notification_qs.count(), 1) + notification = notification_qs.first() + device_location = getattr(device, "devicelocation", None) + actor = device + if device_location: + actor = device_location.location + self.assertEqual(notification.actor, actor) + self.assertEqual(notification.target, device) + self.assertEqual(notification.type, "estimated_location_info") + self.assertEqual(notification.level, notify_level) + for message in messages: + self.assertIn(message, notification.message) + self.assertIn(device.last_ip, notification.rendered_description) + self.assertIn("#devicelocation-group", notification.target_url) + + with self.subTest("Test Notification for location create"): + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + device1 = self._create_device(last_ip="172.217.22.10") + messages = ["Estimated location", "created successfully"] + _verify_notification(device1, messages) + + with self.subTest("Test Notification for location update"): + notification_qs.delete() + # will have same location as first device + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.10", + ) + messages = ["Estimated location", "updated successfully"] + _verify_notification(device2, messages) + + with self.subTest("Test Error Notification for conflicting locations"): + device2.last_ip = device1.last_ip + device2.save() + notification_qs.delete() + mock_info.reset_mock() + mock_error.reset_mock() + device3 = self._create_device( + name="11:22:33:44:55:77", + mac_address="11:22:33:44:55:77", + last_ip=device2.last_ip, + ) + mock_info.assert_not_called() + mock_error.assert_called_once_with( + f"Multiple devices with locations found with same " + f"last_ip {device3.last_ip}. Please resolve the conflict manually." + ) + messages = ["Unable to create estimated location for device"] + _verify_notification(device3, messages, "error") + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_estimate_location_status_remove(self, mock_client): + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + device = self._create_device(last_ip="172.217.22.10") + location = device.devicelocation.location + self.assertTrue(location.is_estimated) + org = self._get_org() + + with self.subTest( + "Test Estimated Status unchanged if Estimated feature is disabled" + ): + org.config_settings.estimated_location_enabled = False + org.config_settings.save() + location.geometry = GEOSGeometry("POINT(12.512124 41.898903)", srid=4326) + location.save() + self.assertTrue(location.is_estimated) + self.assertIn(f"(Estimated Location: {device.last_ip})", location.name) + + with self.subTest( + "Test Estimated Status unchanged if Estimated feature is enabled" + " and desired fields not changed" + ): + org.config_settings.estimated_location_enabled = True + org.config_settings.save() + location._set_initial_values_for_changed_checked_fields() + location.type = "outdoor" + location.is_mobile = True + location.save() + self.assertTrue(location.is_estimated) + + with self.subTest( + "Test Estimated Status changed if Estimated feature is enabled" + " and desired fields changed" + ): + location.geometry = GEOSGeometry("POINT(15.512124 45.898903)", srid=4326) + location.save() + self.assertFalse(location.is_estimated) + self.assertNotIn(f"(Estimated Location: {device.last_ip})", location.name) + + +class TestEstimatedLocationFieldFilters( + TestEstimatedLocationMixin, TestGeoMixin, TestCase +): + location_model = Location + object_location_model = DeviceLocation + + def setUp(self): + super().setUp() + admin = self._create_admin() + self.client.force_login(admin) + + def _create_device_location(self, **kwargs): + options = dict() + options.update(kwargs) + device_location = self.object_location_model(**options) + device_location.full_clean() + device_location.save() + return device_location + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + def test_estimated_location_api_status_configured(self): + org1 = self._get_org() + org2 = self._create_org(name="org2") + OrganizationConfigSettings.objects.create( + organization=org2, + whois_enabled=False, + estimated_location_enabled=False, + ) + org1_location = self._create_location( + name="org1-location", organization=org1, is_estimated=True + ) + org2_location = self._create_location(name="org2-location", organization=org2) + org1_device = self._create_device(organization=org1) + org2_device = self._create_device(organization=org2) + self._create_device_location(content_object=org1_device, location=org1_location) + self._create_device_location(content_object=org2_device, location=org2_location) + + with self.subTest("Test Estimated Location in Locations List"): + path = reverse("geo_api:list_location") + with self.assertNumQueries(5): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 2) + self.assertContains(response, org1_location.id) + self.assertContains(response, org2_location.id) + location1 = response.data["results"][1] + location2 = response.data["results"][0] + self.assertIn("is_estimated", location1) + self.assertNotIn("is_estimated", location2) + + with self.subTest("Test Estimated Location in Device Locations List"): + path = reverse("geo_api:device_location", args=[org1_device.pk]) + with self.assertNumQueries(4): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertIn("is_estimated", response.data["location"]["properties"]) + + path = reverse("geo_api:device_location", args=[org2_device.pk]) + with self.assertNumQueries(4): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertNotIn("is_estimated", response.data["location"]["properties"]) + + with self.subTest("Test Estimated Location in GeoJSON List"): + path = reverse("geo_api:location_geojson") + with self.assertNumQueries(3): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 2) + for i in response.data["features"]: + if i["id"] == org1_location.id: + self.assertIn("is_estimated", i["properties"]) + self.assertTrue(i["properties"]["is_estimated"]) + elif i["id"] == org2_location.id: + self.assertNotIn("is_estimated", i["properties"]) + self.assertFalse(i["properties"]["is_estimated"]) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False) + def test_estimated_location_api_status_not_configured(self): + org = self._get_org() + location = self._create_location(name="org1-location", organization=org) + device = self._create_device(organization=org) + self._create_device_location(content_object=device, location=location) + + with self.subTest("Test Estimated status not in Locations List"): + path = reverse("geo_api:list_location") + with self.assertNumQueries(4): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 1) + self.assertContains(response, location.id) + api_location = response.data["results"][0] + self.assertNotIn("is_estimated", api_location) + + with self.subTest("Test Estimated status not in Device Locations List"): + path = reverse("geo_api:device_location", args=[device.pk]) + with self.assertNumQueries(4): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertNotIn("is_estimated", response.data["location"]["properties"]) + + with self.subTest("Test Estimated status not in GeoJSON Location List"): + path = reverse("geo_api:location_geojson") + with self.assertNumQueries(3): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 1) + location_features = response.data["features"][0] + self.assertNotIn("is_estimated", location_features["properties"]) + self.assertFalse(location_features["properties"].get("is_estimated", False)) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + def test_estimated_location_filter_list_api(self): + org = self._get_org() + location1 = self._create_location( + name="location1", is_estimated=True, organization=org + ) + location2 = self._create_location( + name="location2", is_estimated=False, organization=org + ) + device1 = self._create_device() + device2 = self._create_device( + name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66" + ) + self._create_device_location(content_object=device1, location=location1) + self._create_device_location(content_object=device2, location=location2) + + path = reverse("geo_api:list_location") + + with self.subTest( + "Test Estimated Location filter available in location list " + "when WHOIS is configured" + ): + with self.assertNumQueries(4): + response = self.client.get(path, {"is_estimated": True}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 1) + self.assertContains(response, location1.id) + self.assertNotContains(response, location2.id) + + with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False): + with self.subTest( + "Test Estimated Location filter not available in location list " + "when WHOIS not configured" + ): + with self.assertNumQueries(5): + response = self.client.get(path, {"is_estimated": True}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 2) + self.assertContains(response, location1.id) + self.assertContains(response, location2.id) + + path = reverse("config_api:device_list") + + with self.subTest( + "Test Estimated Location filter available in device list " + "when WHOIS is configured" + ): + with self.assertNumQueries(3): + response = self.client.get(path, {"geo_is_estimated": True}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 1) + self.assertContains(response, device1.id) + self.assertNotContains(response, device2.id) + + with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False): + with self.subTest( + "Test Estimated Location filter not available in device list " + "when WHOIS not configured" + ): + with self.assertNumQueries(3): + response = self.client.get(path, {"geo_is_estimated": True}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 2) + self.assertContains(response, device1.id) + self.assertContains(response, device2.id) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + def test_estimated_location_filter_admin(self): + org = self._get_org() + estimated_location = self._create_location( + name="location1", is_estimated=True, organization=org + ) + outdoor_location = self._create_location(name="location2", organization=org) + indoor_location = self._create_location( + name="location3", organization=org, type="indoor" + ) + + estimated_device = self._create_device() + outdoor_device = self._create_device( + name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66" + ) + indoor_device = self._create_device( + name="11:22:33:44:55:77", mac_address="11:22:33:44:55:77" + ) + self._create_device_location( + content_object=estimated_device, location=estimated_location + ) + self._create_device_location( + content_object=outdoor_device, location=outdoor_location + ) + self._create_device_location( + content_object=indoor_device, location=indoor_location + ) + + path = reverse("admin:config_device_changelist") + with self.subTest("Test All Locations Filter"): + response = self.client.get(path) + self.assertContains(response, estimated_device.id) + self.assertContains(response, outdoor_device.id) + self.assertContains(response, indoor_device.id) + + with self.subTest("Test Estimated Location Filter"): + response = self.client.get(path, {"with_geo": "estimated"}) + self.assertContains(response, estimated_device.id) + self.assertNotContains(response, outdoor_device.id) + self.assertNotContains(response, indoor_device.id) + + with self.subTest("Test Outdoor Location Filter"): + response = self.client.get(path, {"with_geo": "outdoor"}) + self.assertContains(response, outdoor_device.id) + self.assertNotContains(response, estimated_device.id) + self.assertNotContains(response, indoor_device.id) + + with self.subTest("Test Indoor Location Filter"): + response = self.client.get(path, {"with_geo": "indoor"}) + self.assertContains(response, indoor_device.id) + self.assertNotContains(response, outdoor_device.id) + self.assertNotContains(response, estimated_device.id) + + with self.subTest("Test Indoor Location Filter"): + response = self.client.get(path, {"with_geo": "false"}) + self.assertNotContains(response, indoor_device.id) + self.assertNotContains(response, outdoor_device.id) + self.assertNotContains(response, estimated_device.id) + + with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False): + with self.subTest( + "Test Estimated Location Admin specific filters not available" + " when WHOIS not configured" + ): + for i in ["estimated", "outdoor", "indoor"]: + response = self.client.get(path, {"with_geo": i}) + self.assertContains(response, estimated_device.id) + self.assertContains(response, outdoor_device.id) + self.assertContains(response, indoor_device.id) diff --git a/openwisp_controller/geo/estimated_location/tests/utils.py b/openwisp_controller/geo/estimated_location/tests/utils.py new file mode 100644 index 000000000..77bb618ba --- /dev/null +++ b/openwisp_controller/geo/estimated_location/tests/utils.py @@ -0,0 +1,15 @@ +from swapper import load_model + +from openwisp_controller.config.whois.tests.utils import CreateWHOISMixin + +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + + +class TestEstimatedLocationMixin(CreateWHOISMixin): + def setUp(self): + super(CreateWHOISMixin, self).setUp() + OrganizationConfigSettings.objects.create( + organization=self._get_org(), + whois_enabled=True, + estimated_location_enabled=True, + ) diff --git a/openwisp_controller/geo/estimated_location/utils.py b/openwisp_controller/geo/estimated_location/utils.py new file mode 100644 index 000000000..ddd43e837 --- /dev/null +++ b/openwisp_controller/geo/estimated_location/utils.py @@ -0,0 +1,6 @@ +from openwisp_notifications.utils import _get_object_link + + +def get_device_location_notification_target_url(obj, field, absolute_url=True): + url = _get_object_link(obj._related_object(field), absolute_url) + return f"{url}#devicelocation-group" diff --git a/openwisp_controller/geo/migrations/0004_location_is_estimated.py b/openwisp_controller/geo/migrations/0004_location_is_estimated.py new file mode 100644 index 000000000..1fcdfca68 --- /dev/null +++ b/openwisp_controller/geo/migrations/0004_location_is_estimated.py @@ -0,0 +1,21 @@ +# Generated by Django 5.2.1 on 2025-06-25 19:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("geo", "0003_alter_devicelocation_floorplan_location"), + ] + + operations = [ + migrations.AddField( + model_name="location", + name="is_estimated", + field=models.BooleanField( + default=False, + help_text=("Whether the location's coordinates are estimated."), + ), + ), + ] diff --git a/openwisp_controller/geo/templates/admin/geo/location/change_form.html b/openwisp_controller/geo/templates/admin/geo/location/change_form.html new file mode 100644 index 000000000..96b4392e9 --- /dev/null +++ b/openwisp_controller/geo/templates/admin/geo/location/change_form.html @@ -0,0 +1,11 @@ +{% extends "admin/change_form.html" %} +{% load i18n admin_urls %} + +{% block messages %} + {{ block.super }} + {% if original and estimated_configured and original.is_estimated %} +
    +
  • {% trans "This location is estimated based on the device's last IP address. Please refine it for greater accuracy." %}
    • +
+ {% endif %} +{% endblock messages %} diff --git a/openwisp_controller/geo/tests/test_admin.py b/openwisp_controller/geo/tests/test_admin.py index 8d914dc8d..06a498cfe 100644 --- a/openwisp_controller/geo/tests/test_admin.py +++ b/openwisp_controller/geo/tests/test_admin.py @@ -137,6 +137,16 @@ def test_admin_menu_groups(self): html=True, ) + def test_location_readonly_fields(self): + location = self._create_location( + name="location1org", type="indoor", organization=self._get_org() + ) + self._create_admin() + self._login() + url = reverse(f"admin:{self.app_label}_location_change", args=[location.pk]) + response = self.client.get(url) + self.assertNotContains(response, ' Date: Wed, 10 Dec 2025 12:58:41 +0530 Subject: [PATCH 4/5] [feature] Updating stale WHOIS records #1058 Added required checks for ensuring WHOIS records older than a certain configurable threshold will be updated along with the coordinates. Closes #1058 Signed-off-by: DragnEmperor --- docs/user/estimated-location.rst | 15 +- docs/user/settings.rst | 13 ++ docs/user/whois.rst | 21 +- openwisp_controller/config/base/device.py | 2 + .../config/controller/views.py | 3 + .../management/commands/clear_last_ip.py | 9 +- .../migrations/0061_config_checksum_db.py | 3 +- .../{0061_whoisinfo.py => 0062_whoisinfo.py} | 2 +- ..._approximate_location_enabled_and_more.py} | 2 +- openwisp_controller/config/settings.py | 1 + openwisp_controller/config/whois/service.py | 197 +++++++++++++++- openwisp_controller/config/whois/tasks.py | 102 ++------- .../config/whois/tests/tests.py | 54 ++++- .../config/whois/tests/utils.py | 9 +- openwisp_controller/config/whois/utils.py | 6 +- .../geo/estimated_location/tasks.py | 212 ++++++++---------- .../geo/estimated_location/tests/tests.py | 107 ++++++++- .../admin/geo/location/change_form.html | 2 +- 18 files changed, 520 insertions(+), 240 deletions(-) rename openwisp_controller/config/migrations/{0061_whoisinfo.py => 0062_whoisinfo.py} (97%) rename openwisp_controller/config/migrations/{0062_organizationconfigsettings_approximate_location_enabled_and_more.py => 0063_organizationconfigsettings_approximate_location_enabled_and_more.py} (96%) diff --git a/docs/user/estimated-location.rst b/docs/user/estimated-location.rst index 1f00a7440..48b3c100f 100644 --- a/docs/user/estimated-location.rst +++ b/docs/user/estimated-location.rst @@ -16,14 +16,14 @@ Estimated Location Overview -------- -The Estimated Location feature automatically creates or updates a device’s -location based on latitude and longitude information retrieved from the -WHOIS Lookup feature. +This feature automatically creates or updates a device’s location based on +latitude and longitude information retrieved from the WHOIS Lookup +feature. Trigger Conditions ------------------ -Estimated Location is triggered when: +This feature is triggered when: - A **fresh WHOIS lookup** is performed for a device. - Or when a WHOIS record already exists for the device’s IP **and**: @@ -78,3 +78,10 @@ In REST API, the field will be visible in the :ref:`Device Location ` if the feature is **enabled**. The field can also be used for filtering in the location list (including geojson) endpoints and in the :ref:`Device List `. + +Managing Older Estimated Locations +---------------------------------- + +Whenever location related fields in WHOIS records are updated as per +:ref:`Managing WHOIS Older Records `; the location +will also be updated automatically. diff --git a/docs/user/settings.rst b/docs/user/settings.rst index 9a59bbf22..cf0d4a4de 100644 --- a/docs/user/settings.rst +++ b/docs/user/settings.rst @@ -829,3 +829,16 @@ Allows enabling the optional :doc:`Estimated Location feature .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-location-setting.png :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-location-setting.png :alt: Estimated Location setting + +.. _openwisp_controller_whois_refresh_threshold_days: + +``OPENWISP_CONTROLLER_WHOIS_REFRESH_THRESHOLD_DAYS`` +---------------------------------------------------- + +============ ======= +**type**: ``int`` +**default**: ``14`` +============ ======= + +Specifies the number of days after which the WHOIS information for a +device is considered stale and eligible for refresh. diff --git a/docs/user/whois.rst b/docs/user/whois.rst index e9c63274c..578bce597 100644 --- a/docs/user/whois.rst +++ b/docs/user/whois.rst @@ -15,10 +15,10 @@ WHOIS Lookup Overview -------- -The WHOIS Lookup feature displays information about the public IP address -used by devices to communicate with OpenWISP (via the ``last_ip`` field). -It helps identify the geographic location and ISP associated with the IP -address, which can be useful for troubleshooting network issues. +This feature displays information about the public IP address used by +devices to communicate with OpenWISP (via the ``last_ip`` field). It helps +identify the geographic location and ISP associated with the IP address, +which can be useful for troubleshooting network issues. The retrieved information pertains to the Autonomous System (ASN) associated with the device's public IP address and includes: @@ -36,6 +36,7 @@ Trigger Conditions A WHOIS lookup is triggered automatically when: - A new device is registered. +- An existing device's last IP address is changed. - A device fetches its checksum. However, the lookup will only run if **all** the following conditions are @@ -114,3 +115,15 @@ retrieved details can be viewed in the following locations: - **Device REST API**: See WHOIS details in the :ref:`Device List ` and :ref:`Device Detail ` responses. + +.. _whois_older_records: + +Managing Older WHOIS Records +---------------------------- + +If a record is older than :ref:`Threshold +`, it will be refreshed +automatically. + +The update mechanism will be triggered whenever a device is registered or +its last IP changes or fetches its checksum. diff --git a/openwisp_controller/config/base/device.py b/openwisp_controller/config/base/device.py index 525098737..2a2d88fd0 100644 --- a/openwisp_controller/config/base/device.py +++ b/openwisp_controller/config/base/device.py @@ -538,4 +538,6 @@ def _check_last_ip(self, creating=False): return if creating or self.last_ip != self._initial_last_ip: self.whois_service.process_ip_data_and_location() + else: + self.whois_service.update_whois_info() self._initial_last_ip = self.last_ip diff --git a/openwisp_controller/config/controller/views.py b/openwisp_controller/config/controller/views.py index 3dc714c0c..5e1c9fa10 100644 --- a/openwisp_controller/config/controller/views.py +++ b/openwisp_controller/config/controller/views.py @@ -153,6 +153,9 @@ def get(self, request, pk): # updates cache if ip addresses changed if updated: self.update_device_cache(device) + # check if WHOIS Info of device requires update + else: + device.whois_service.update_whois_info() checksum_requested.send( sender=device.__class__, instance=device, request=request ) diff --git a/openwisp_controller/config/management/commands/clear_last_ip.py b/openwisp_controller/config/management/commands/clear_last_ip.py index 4a5efd4c7..7c68a3d4b 100644 --- a/openwisp_controller/config/management/commands/clear_last_ip.py +++ b/openwisp_controller/config/management/commands/clear_last_ip.py @@ -4,7 +4,10 @@ class Command(BaseCommand): - help = "Clear the last IP address, if set, of active devices of all organizations." + help = ( + "Clears the last IP address (if set) for every active device" + " across all organizations." + ) def add_arguments(self, parser): parser.add_argument( @@ -29,11 +32,11 @@ def handle(self, *args, **options): "Are you sure you want to do this?\n\n" "Type 'yes' to continue, or 'no' to cancel: " ) - if input("".join(message)) != "yes": + if input("".join(message)).lower() != "yes": raise CommandError("Operation cancelled by user.") devices = Device.objects.filter(_is_deactivated=False).only("last_ip") - # Filter devices that have no WHOIS information for their last IP + # Filter out devices that have WHOIS information for their last IP devices = devices.exclude(last_ip=None).exclude( last_ip__in=Subquery( WHOISInfo.objects.filter(ip_address=OuterRef("last_ip")).values( diff --git a/openwisp_controller/config/migrations/0061_config_checksum_db.py b/openwisp_controller/config/migrations/0061_config_checksum_db.py index a572951cf..34319cae3 100644 --- a/openwisp_controller/config/migrations/0061_config_checksum_db.py +++ b/openwisp_controller/config/migrations/0061_config_checksum_db.py @@ -1,7 +1,6 @@ # Generated by Django for issue #1113 optimization from django.db import migrations, models -from swapper import load_model def populate_checksum_db(apps, schema_editor): @@ -13,7 +12,7 @@ def populate_checksum_db(apps, schema_editor): hence we use Config.objects.bulk_update() instead of Config.update_status_if_checksum_changed(). """ - Config = load_model("config", "Config") + Config = apps.get_model("config", "Config") chunk_size = 100 updated_configs = [] qs = ( diff --git a/openwisp_controller/config/migrations/0061_whoisinfo.py b/openwisp_controller/config/migrations/0062_whoisinfo.py similarity index 97% rename from openwisp_controller/config/migrations/0061_whoisinfo.py rename to openwisp_controller/config/migrations/0062_whoisinfo.py index d8148756b..4755b8485 100644 --- a/openwisp_controller/config/migrations/0061_whoisinfo.py +++ b/openwisp_controller/config/migrations/0062_whoisinfo.py @@ -10,7 +10,7 @@ class Migration(migrations.Migration): dependencies = [ - ("config", "0060_cleanup_api_task_notification_types"), + ("config", "0061_config_checksum_db"), ] operations = [ diff --git a/openwisp_controller/config/migrations/0062_organizationconfigsettings_approximate_location_enabled_and_more.py b/openwisp_controller/config/migrations/0063_organizationconfigsettings_approximate_location_enabled_and_more.py similarity index 96% rename from openwisp_controller/config/migrations/0062_organizationconfigsettings_approximate_location_enabled_and_more.py rename to openwisp_controller/config/migrations/0063_organizationconfigsettings_approximate_location_enabled_and_more.py index 8fe4d4c2c..bdb3dab64 100644 --- a/openwisp_controller/config/migrations/0062_organizationconfigsettings_approximate_location_enabled_and_more.py +++ b/openwisp_controller/config/migrations/0063_organizationconfigsettings_approximate_location_enabled_and_more.py @@ -9,7 +9,7 @@ class Migration(migrations.Migration): dependencies = [ - ("config", "0061_whoisinfo"), + ("config", "0062_whoisinfo"), ] operations = [ diff --git a/openwisp_controller/config/settings.py b/openwisp_controller/config/settings.py index fa3efc750..8bcda862d 100644 --- a/openwisp_controller/config/settings.py +++ b/openwisp_controller/config/settings.py @@ -68,6 +68,7 @@ def get_setting(option, default): "API_TASK_RETRY_OPTIONS", dict(max_retries=5, retry_backoff=True, retry_backoff_max=600, retry_jitter=True), ) +WHOIS_REFRESH_THRESHOLD_DAYS = get_setting("WHOIS_REFRESH_THRESHOLD_DAYS", 14) WHOIS_GEOIP_ACCOUNT = get_setting("WHOIS_GEOIP_ACCOUNT", None) WHOIS_GEOIP_KEY = get_setting("WHOIS_GEOIP_KEY", None) WHOIS_ENABLED = get_setting("WHOIS_ENABLED", False) diff --git a/openwisp_controller/config/whois/service.py b/openwisp_controller/config/whois/service.py index 3e9dad6f1..71884d52d 100644 --- a/openwisp_controller/config/whois/service.py +++ b/openwisp_controller/config/whois/service.py @@ -1,12 +1,37 @@ +from datetime import timedelta from ipaddress import ip_address as ip_addr +import requests +from django.contrib.gis.geos import Point from django.core.cache import cache from django.db import transaction +from django.utils import timezone +from django.utils.translation import gettext as _ +from geoip2 import errors +from geoip2 import webservice as geoip2_webservice from swapper import load_model from openwisp_controller.config import settings as app_settings from .tasks import fetch_whois_details, manage_estimated_locations +from .utils import send_whois_task_notification + +EXCEPTION_MESSAGES = { + errors.AddressNotFoundError: _( + "No WHOIS information found for IP address {ip_address}" + ), + errors.AuthenticationError: _( + "Authentication failed for GeoIP2 service. " + "Check your OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT and " + "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY settings." + ), + errors.OutOfQueriesError: _( + "Your account has run out of queries for the GeoIP2 service." + ), + errors.PermissionRequiredError: _( + "Your account does not have permission to access this service." + ), +} class WHOISService: @@ -17,6 +42,20 @@ class WHOISService: def __init__(self, device): self.device = device + @staticmethod + def get_geoip_client(): + """ + Used to get a GeoIP2 web service client instance. + Host is based on the db that is used to fetch the details. + As we are using GeoLite2, 'geolite.info' host is used. + Refer: https://geoip2.readthedocs.io/en/latest/#sync-web-service-example + """ + return geoip2_webservice.Client( + account_id=app_settings.WHOIS_GEOIP_ACCOUNT, + license_key=app_settings.WHOIS_GEOIP_KEY, + host="geolite.info", + ) + @staticmethod def get_cache_key(org_id): """ @@ -43,6 +82,15 @@ def _get_whois_info_from_db(ip_address): return WHOISInfo.objects.filter(ip_address=ip_address) + @staticmethod + def is_older(datetime): + """ + Check if given datetime is older than the refresh threshold. + """ + return (timezone.now() - datetime) >= timedelta( + days=app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + ) + @staticmethod def get_org_config_settings(org_id): """ @@ -90,6 +138,8 @@ def is_whois_enabled(self): """ Check if the WHOIS lookup feature is enabled. """ + if not app_settings.WHOIS_CONFIGURED: + return False org_settings = self.get_org_config_settings(org_id=self.device.organization.pk) return org_settings.whois_enabled @@ -98,9 +148,57 @@ def is_estimated_location_enabled(self): """ Check if the Estimated location feature is enabled. """ + if not app_settings.WHOIS_CONFIGURED: + return False org_settings = self.get_org_config_settings(org_id=self.device.organization.pk) return org_settings.estimated_location_enabled + def process_whois_details(self, ip_address): + """ + Fetch WHOIS details for a given IP address and return only + the relevant information. + """ + ip_client = self.get_geoip_client() + try: + data = ip_client.city(ip_address=ip_address) + # Catching all possible exceptions raised by the geoip2 client + # and raising them with appropriate messages to be handled by the task + # retry mechanism. + except ( + errors.AddressNotFoundError, + errors.AuthenticationError, + errors.OutOfQueriesError, + errors.PermissionRequiredError, + ) as e: + exc_type = type(e) + message = EXCEPTION_MESSAGES.get(exc_type) + if exc_type is errors.AddressNotFoundError: + message = message.format(ip_address=ip_address) + raise exc_type(message) + except requests.RequestException as e: + raise e + else: + # The attributes are always present in the response, + # but they can be None, so added fallbacks. + address = { + "city": data.city.name or "", + "country": data.country.name or "", + "continent": data.continent.name or "", + "postal": str(data.postal.code or ""), + } + coordinates = Point( + data.location.longitude, data.location.latitude, srid=4326 + ) + return { + "isp": data.traits.autonomous_system_organization, + "asn": data.traits.autonomous_system_number, + "timezone": data.location.time_zone, + "address": address, + "coordinates": coordinates, + "cidr": data.traits.network, + "ip_address": ip_address, + } + def _need_whois_lookup(self, new_ip): """ This is used to determine if the WHOIS lookup should be triggered @@ -108,17 +206,17 @@ def _need_whois_lookup(self, new_ip): The lookup is not triggered if: - The new IP address is None or it is a private IP address. - - The WHOIS information of new ip is already present. + - The WHOIS information of new ip is present and is not older than + X days (defined by "WHOIS_REFRESH_THRESHOLD_DAYS"). - WHOIS is disabled in the organization settings. (query from db) """ # Check cheap conditions first before hitting the database if not self.is_valid_public_ip_address(new_ip): return False - - if self._get_whois_info_from_db(new_ip).exists(): + whois_obj = self._get_whois_info_from_db(ip_address=new_ip).first() + if whois_obj and not self.is_older(whois_obj.modified): return False - return self.is_whois_enabled def _need_estimated_location_management(self, new_ip): @@ -128,10 +226,8 @@ def _need_estimated_location_management(self, new_ip): """ if not self.is_valid_public_ip_address(new_ip): return False - if not self.is_whois_enabled: return False - return self.is_estimated_location_enabled def get_device_whois_info(self): @@ -145,7 +241,7 @@ def get_device_whois_info(self): return self._get_whois_info_from_db(ip_address=ip_address).first() - def process_ip_data_and_location(self): + def process_ip_data_and_location(self, force_lookup=False): """ Trigger WHOIS lookup based on the conditions of `_need_whois_lookup` and also manage estimated locations based on the conditions of @@ -153,12 +249,11 @@ def process_ip_data_and_location(self): Tasks are triggered on commit to ensure redundant data is not created. """ new_ip = self.device.last_ip - if self._need_whois_lookup(new_ip): + if force_lookup or self._need_whois_lookup(new_ip): transaction.on_commit( lambda: fetch_whois_details.delay( device_pk=self.device.pk, initial_ip_address=self.device._initial_last_ip, - new_ip_address=new_ip, ) ) # To handle the case when WHOIS already exists as in that case @@ -170,3 +265,87 @@ def process_ip_data_and_location(self): device_pk=self.device.pk, ip_address=new_ip ) ) + + def update_whois_info(self): + """ + Update the WHOIS information for the device. + """ + ip_address = self.device.last_ip + if not self.is_valid_public_ip_address(ip_address): + return + if not self.is_whois_enabled: + return + whois_obj = WHOISService._get_whois_info_from_db(ip_address=ip_address).first() + if whois_obj and self.is_older(whois_obj.modified): + fetch_whois_details.delay(device_pk=self.device.pk, initial_ip_address=None) + + def _create_or_update_whois(self, whois_details, whois_instance=None): + """ + Used to update an existing WHOIS instance; else, creates a new one. + Returns the updated or created WHOIS instance along with update fields. + """ + WHOISInfo = load_model("config", "WHOISInfo") + + update_fields = [] + if whois_instance: + for attr, value in whois_details.items(): + if getattr(whois_instance, attr) != value: + update_fields.append(attr) + setattr(whois_instance, attr, value) + if update_fields: + whois_instance.save(update_fields=update_fields) + else: + whois_instance = WHOISInfo(**whois_details) + whois_instance.full_clean() + whois_instance.save(force_insert=True) + return whois_instance, update_fields + + def _create_or_update_estimated_location( + self, location_defaults, attached_devices_exists + ): + """ + Create or update estimated location for the device based on the + given location defaults. + """ + Location = load_model("geo", "Location") + DeviceLocation = load_model("geo", "DeviceLocation") + + if not (device_location := getattr(self.device, "devicelocation", None)): + device_location = DeviceLocation(content_object=self.device) + + current_location = device_location.location + + if not current_location or ( + attached_devices_exists and current_location.is_estimated + ): + with transaction.atomic(): + current_location = Location(**location_defaults, is_estimated=True) + current_location.full_clean() + current_location.save(_set_estimated=True) + device_location.location = current_location + device_location.full_clean() + device_location.save() + send_whois_task_notification( + device=self.device, + notify_type="estimated_location_created", + actor=current_location, + ) + elif current_location.is_estimated: + update_fields = [] + for attr, value in location_defaults.items(): + if getattr(current_location, attr) != value: + setattr(current_location, attr, value) + update_fields.append(attr) + if update_fields: + with transaction.atomic(): + current_location.save( + update_fields=update_fields, _set_estimated=True + ) + + send_whois_task_notification( + device=self.device, + notify_type="estimated_location_updated", + actor=current_location, + ) + + return current_location diff --git a/openwisp_controller/config/whois/tasks.py b/openwisp_controller/config/whois/tasks.py index bafca7b78..26b0a5037 100644 --- a/openwisp_controller/config/whois/tasks.py +++ b/openwisp_controller/config/whois/tasks.py @@ -1,11 +1,8 @@ import logging -import requests from celery import shared_task -from django.contrib.gis.geos import Point -from django.utils.translation import gettext as _ +from django.db import transaction from geoip2 import errors -from geoip2 import webservice as geoip2_webservice from swapper import load_model from openwisp_controller.geo.estimated_location.tasks import manage_estimated_locations @@ -16,23 +13,6 @@ logger = logging.getLogger(__name__) -EXCEPTION_MESSAGES = { - errors.AddressNotFoundError: _( - "No WHOIS information found for IP address {ip_address}" - ), - errors.AuthenticationError: _( - "Authentication failed for GeoIP2 service. " - "Check your OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT and " - "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY settings." - ), - errors.OutOfQueriesError: _( - "Your account has run out of queries for the GeoIP2 service." - ), - errors.PermissionRequiredError: _( - "Your account does not have permission to access this service." - ), -} - class WHOISCeleryRetryTask(OpenwispCeleryTask): """ @@ -46,9 +26,7 @@ class WHOISCeleryRetryTask(OpenwispCeleryTask): def on_failure(self, exc, task_id, args, kwargs, einfo): """Notify the user about the failure of the WHOIS task.""" device_pk = kwargs.get("device_pk") - send_whois_task_notification( - device_pk=device_pk, notify_type="whois_device_error" - ) + send_whois_task_notification(device=device_pk, notify_type="whois_device_error") logger.error(f"WHOIS lookup failed. Details: {exc}") return super().on_failure(exc, task_id, args, kwargs, einfo) @@ -59,7 +37,7 @@ def on_failure(self, exc, task_id, args, kwargs, einfo): base=WHOISCeleryRetryTask, **app_settings.API_TASK_RETRY_OPTIONS, ) -def fetch_whois_details(self, device_pk, initial_ip_address, new_ip_address): +def fetch_whois_details(self, device_pk, initial_ip_address): """ Fetches the WHOIS details of the given IP address and creates/updates the WHOIS record. @@ -67,65 +45,29 @@ def fetch_whois_details(self, device_pk, initial_ip_address, new_ip_address): Device = load_model("config", "Device") WHOISInfo = load_model("config", "WHOISInfo") - # The task can be triggered for same ip address multiple times - # so we need to return early if WHOIS is already created. - if WHOISInfo.objects.filter(ip_address=new_ip_address).exists(): - return - - device = Device.objects.get(pk=device_pk) - # Host is based on the db that is used to fetch the details. - # As we are using GeoLite2, 'geolite.info' host is used. - # Refer: https://geoip2.readthedocs.io/en/latest/#sync-web-service-example - ip_client = geoip2_webservice.Client( - account_id=app_settings.WHOIS_GEOIP_ACCOUNT, - license_key=app_settings.WHOIS_GEOIP_KEY, - host="geolite.info", - ) - - try: - data = ip_client.city(ip_address=new_ip_address) - - # Catching all possible exceptions raised by the geoip2 client - # and raising them with appropriate messages to be handled by the task - # retry mechanism. - except ( - errors.AddressNotFoundError, - errors.AuthenticationError, - errors.OutOfQueriesError, - errors.PermissionRequiredError, - ) as e: - exc_type = type(e) - message = EXCEPTION_MESSAGES.get(exc_type) - if exc_type is errors.AddressNotFoundError: - message = message.format(ip_address=new_ip_address) - raise exc_type(message) - except requests.RequestException as e: - raise e - - else: - # The attributes are always present in the response, - # but they can be None, so added fallbacks. - address = { - "city": data.city.name or "", - "country": data.country.name or "", - "continent": data.continent.name or "", - "postal": str(data.postal.code or ""), - } - coordinates = Point(data.location.longitude, data.location.latitude, srid=4326) - whois_obj = WHOISInfo( - isp=data.traits.autonomous_system_organization, - asn=data.traits.autonomous_system_number, - timezone=data.location.time_zone, - address=address, - cidr=data.traits.network, - ip_address=new_ip_address, - coordinates=coordinates, + with transaction.atomic(): + device = Device.objects.get(pk=device_pk) + new_ip_address = device.last_ip + WHOISService = device.whois_service + + # If there is existing WHOIS older record then it needs to be updated + whois_obj = WHOISInfo.objects.filter(ip_address=new_ip_address).first() + if whois_obj and not WHOISService.is_older(whois_obj.modified): + return + + fetched_details = WHOISService.process_whois_details(new_ip_address) + whois_obj, update_fields = WHOISService._create_or_update_whois( + fetched_details, whois_obj ) - whois_obj.full_clean() - whois_obj.save() logger.info(f"Successfully fetched WHOIS details for {new_ip_address}.") if device._get_organization__config_settings().estimated_location_enabled: + # the estimated location task should not run if old record is updated + # and location related fields are not updated + if update_fields and not any( + i in update_fields for i in ["address", "coordinates"] + ): + return manage_estimated_locations.delay( device_pk=device_pk, ip_address=new_ip_address ) diff --git a/openwisp_controller/config/whois/tests/tests.py b/openwisp_controller/config/whois/tests/tests.py index 82ad9e83b..3accc02c6 100644 --- a/openwisp_controller/config/whois/tests/tests.py +++ b/openwisp_controller/config/whois/tests/tests.py @@ -1,4 +1,5 @@ import importlib +from datetime import timedelta from io import StringIO from unittest import mock @@ -9,6 +10,7 @@ from django.db.models.signals import post_delete, post_save from django.test import TestCase, TransactionTestCase, override_settings, tag from django.urls import reverse +from django.utils import timezone from geoip2 import errors from selenium.webdriver.common.by import By from swapper import load_model @@ -341,7 +343,7 @@ class TestWHOISTransaction( CreateWHOISMixin, WHOISTransactionMixin, TransactionTestCase ): _WHOIS_GEOIP_CLIENT = ( - "openwisp_controller.config.whois.tasks.geoip2_webservice.Client" + "openwisp_controller.config.whois.service.geoip2_webservice.Client" ) _WHOIS_TASKS_INFO_LOGGER = "openwisp_controller.config.whois.tasks.logger.info" _WHOIS_TASKS_WARN_LOGGER = "openwisp_controller.config.whois.tasks.logger.warning" @@ -584,6 +586,56 @@ def _verify_whois_details(instance, ip_address): # WHOIS related to the device's last_ip should be deleted self.assertEqual(WHOISInfo.objects.filter(ip_address=ip_address).count(), 0) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_whois_update(self, mock_client): + connect_whois_handlers() + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + threshold = app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + 1 + new_time = timezone.now() - timedelta(days=threshold) + + whois_obj = self._create_whois_info() + WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) + + with self.subTest("Test WHOIS update when older than X days for new device"): + mocked_response.traits.autonomous_system_number = 11111 + mock_client.return_value.city.return_value = mocked_response + device = self._create_device(last_ip=whois_obj.ip_address) + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj.asn, str(11111)) + + with self.subTest( + "Test WHOIS update when older than X days for existing device" + ): + Device.objects.all().delete() + WHOISInfo.objects.all().delete() + device = self._create_device(last_ip="172.217.22.11") + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj.asn, str(11111)) + WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) + mocked_response.traits.autonomous_system_number = 22222 + mock_client.return_value.city.return_value = mocked_response + device.save() + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj.asn, str(22222)) + + with self.subTest( + "Test WHOIS update when older than X days for existing device " + "from DeviceChecksum View" + ): + Device.objects.all().delete() + WHOISInfo.objects.all().delete() + device = self._create_device(last_ip="172.217.22.11") + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj.asn, str(22222)) + WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) + mocked_response.traits.autonomous_system_number = 33333 + mock_client.return_value.city.return_value = mocked_response + device.save() + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj.asn, str(33333)) + # we need to allow the task to propagate exceptions to ensure # `on_failure` method is called and notifications are executed @override_settings(CELERY_TASK_EAGER_PROPAGATES=False) diff --git a/openwisp_controller/config/whois/tests/utils.py b/openwisp_controller/config/whois/tests/utils.py index 7139ae883..03822270a 100644 --- a/openwisp_controller/config/whois/tests/utils.py +++ b/openwisp_controller/config/whois/tests/utils.py @@ -61,10 +61,13 @@ def _task_called(self, mocked_task, task_name="WHOIS lookup"): org = self._get_org() with self.subTest(f"{task_name} task called when last_ip is public"): - with mock.patch("django.core.cache.cache.set") as mocked_set: + with mock.patch( + "django.core.cache.cache.get", side_effect=[None, org.config_settings] + ) as mocked_get, mock.patch("django.core.cache.cache.set") as mocked_set: device = self._create_device(last_ip="172.217.22.14") mocked_task.assert_called() mocked_set.assert_called_once() + mocked_get.assert_called() mocked_task.reset_mock() with self.subTest( @@ -77,7 +80,7 @@ def _task_called(self, mocked_task, task_name="WHOIS lookup"): device.save() mocked_task.assert_called() mocked_set.assert_not_called() - mocked_get.assert_called_once() + mocked_get.assert_called() mocked_task.reset_mock() with self.subTest(f"{task_name} task not called when last_ip not updated"): @@ -122,7 +125,7 @@ def _task_called(self, mocked_task, task_name="WHOIS lookup"): mocked_task.reset_mock() with self.subTest( - f"{task_name} task called via DeviceChecksumView for no WHOIS record" + f"{task_name} task not called via DeviceChecksumView for no WHOIS record" ): WHOISInfo.objects.all().delete() device.refresh_from_db() diff --git a/openwisp_controller/config/whois/utils.py b/openwisp_controller/config/whois/utils.py index 4a955810e..c1b0430e9 100644 --- a/openwisp_controller/config/whois/utils.py +++ b/openwisp_controller/config/whois/utils.py @@ -41,10 +41,10 @@ } -def send_whois_task_notification(device_pk, notify_type, actor=None): +def send_whois_task_notification(device, notify_type, actor=None): Device = load_model("config", "Device") - - device = Device.objects.get(pk=device_pk) + if not isinstance(device, Device): + device = Device.objects.get(pk=device) notify_details = MESSAGE_MAP[notify_type] notify.send( sender=actor or device, diff --git a/openwisp_controller/geo/estimated_location/tasks.py b/openwisp_controller/geo/estimated_location/tasks.py index 8ec080509..c900af12c 100644 --- a/openwisp_controller/geo/estimated_location/tasks.py +++ b/openwisp_controller/geo/estimated_location/tasks.py @@ -1,7 +1,6 @@ import logging from celery import shared_task -from django.db import transaction from swapper import load_model from openwisp_controller.config.whois.utils import send_whois_task_notification @@ -9,6 +8,71 @@ logger = logging.getLogger(__name__) +def _handle_attach_existing_location( + device, device_location, ip_address, existing_device_location +): + """ + Helper function to: + 1. Attach existing device's location (same last_ip) to current device, else + 2. Update current location of device using WHOIS data; if it exists, else + 3. Create a new estimated location for the device using WHOIS data. + """ + Device = load_model("config", "Device") + WHOISInfo = load_model("config", "WHOISInfo") + + current_location = device_location.location + attached_devices_exists = None + if current_location is not None: + attached_devices_exists = ( + Device.objects.filter(devicelocation__location_id=current_location.pk) + .exclude(pk=device.pk) + .exists() + ) + if existing_device_location: + existing_location = existing_device_location.location + device_location.location = existing_location + device_location.full_clean() + device_location.save() + logger.info( + f"Estimated location saved successfully for {device.pk}" + f" for IP: {ip_address}" + ) + # We need to remove existing estimated location of the device + # if it is not shared + if attached_devices_exists is False: + current_location.delete() + send_whois_task_notification( + device=device, + notify_type="estimated_location_updated", + actor=existing_location, + ) + return + # If existing devices with same last_ip do not have any location + # then we create a new location based on WHOIS data. + whois_obj = WHOISInfo.objects.filter(ip_address=ip_address).first() + if not whois_obj or not whois_obj.coordinates: + logger.warning( + f"Coordinates not available for {device.pk} for IP: {ip_address}." + " Estimated location cannot be determined." + ) + return + + location_defaults = { + **whois_obj._get_defaults_for_estimated_location(), + "organization_id": device.organization_id, + } + # create new location if no location exists for device or the estimated location + # of device is shared. + whois_service = device.whois_service + whois_service._create_or_update_estimated_location( + location_defaults, attached_devices_exists + ) + logger.info( + f"Estimated location saved successfully for {device.pk}" + f" for IP: {ip_address}" + ) + + @shared_task def manage_estimated_locations(device_pk, ip_address): """ @@ -28,140 +92,40 @@ def manage_estimated_locations(device_pk, ip_address): to the user to resolve the conflict manually. """ Device = load_model("config", "Device") - Location = load_model("geo", "Location") - WHOISInfo = load_model("config", "WHOISInfo") DeviceLocation = load_model("geo", "DeviceLocation") - def _create_estimated_location(device_location, location_defaults): - with transaction.atomic(): - location = Location(**location_defaults, is_estimated=True) - location.full_clean() - location.save(_set_estimated=True) - device_location.location = location - device_location.full_clean() - device_location.save() - logger.info( - f"Estimated location saved successfully for {device_pk}" - f" for IP: {ip_address}" - ) - send_whois_task_notification( - device_pk=device_pk, - notify_type="estimated_location_created", - actor=location, - ) - - def _update_or_create_estimated_location( - device_location, whois_obj, attached_devices_exists=False - ): - # Used to update an existing location if it is estimated - # or create a new one if it doesn't exist - if whois_obj and whois_obj.coordinates: - location_defaults = { - **whois_obj._get_defaults_for_estimated_location(), - "organization_id": device.organization_id, - } - if current_location and current_location.is_estimated: - if attached_devices_exists: - # If there are other devices attached to the current location, - # we do not update it, but create a new one. - _create_estimated_location(device_location, location_defaults) - return - update_fields = [] - for attr, value in location_defaults.items(): - if getattr(current_location, attr) != value: - setattr(current_location, attr, value) - update_fields.append(attr) - if update_fields: - current_location.save( - update_fields=update_fields, _set_estimated=True - ) - logger.info( - f"Estimated location saved successfully for {device_pk}" - f" for IP: {ip_address}" - ) - send_whois_task_notification( - device_pk=device_pk, - notify_type="estimated_location_updated", - actor=current_location, - ) - elif not current_location: - # If there is no current location, we create a new one. - _create_estimated_location(device_location, location_defaults) - else: - logger.warning( - f"Coordinates not available for {device_pk} for IP: {ip_address}." - " Estimated location cannot be determined." - ) - return - - def _handle_attach_existing_location( - device, device_location, whois_obj, attached_devices_exists=False - ): - # For handling the case when WHOIS already exists for device's new last_ip - # then we attach the location of the device with same last_ip if it exists. - devices_with_location = ( - Device.objects.select_related("devicelocation") - .filter(organization_id=device.organization_id) - .filter(last_ip=ip_address, devicelocation__location__isnull=False) - .exclude(pk=device_pk) - ) - # If there are multiple devices with same last_ip then we need to inform - # the user to resolve the conflict manually. - if devices_with_location.count() > 1: - send_whois_task_notification( - device_pk=device_pk, notify_type="estimated_location_error" - ) - logger.error( - "Multiple devices with locations found with same " - f"last_ip {ip_address}. Please resolve the conflict manually." - ) - return - first_device = devices_with_location.first() - # If existing devices with same last_ip do not have any location - # then we create a new location based on WHOIS data. - if not first_device: - _update_or_create_estimated_location( - device_location, whois_obj, attached_devices_exists - ) - return - existing_location = first_device.devicelocation.location - # We need to remove any existing estimated location of the device - if current_location and not attached_devices_exists: - current_location.delete() - device_location.location = existing_location - device_location.full_clean() - device_location.save() - logger.info( - f"Estimated location saved successfully for {device_pk}" - f" for IP: {ip_address}" - ) - send_whois_task_notification( - device_pk=device_pk, - notify_type="estimated_location_updated", - actor=existing_location, - ) + device = Device.objects.select_related("devicelocation__location").get(pk=device_pk) - whois_obj = WHOISInfo.objects.filter(ip_address=ip_address).first() - device = ( - Device.objects.select_related("devicelocation__location", "organization") - .only("organization_id", "devicelocation") - .get(pk=device_pk) + devices_with_location = ( + Device.objects.only("devicelocation") + .select_related("devicelocation__location") + .filter(organization_id=device.organization_id) + .filter(last_ip=ip_address, devicelocation__location__isnull=False) + .exclude(pk=device.pk) ) + # multiple devices can have same last_ip in cases like usage of proxy + if devices_with_location.count() > 1: + send_whois_task_notification( + device=device, notify_type="estimated_location_error" + ) + logger.error( + "Multiple devices with locations found with same " + f"last_ip {ip_address}. Please resolve the conflict manually." + ) + return + if not (device_location := getattr(device, "devicelocation", None)): device_location = DeviceLocation(content_object=device) - attached_devices_exists = False - if current_location := device_location.location: - attached_devices_exists = ( - Device.objects.filter(devicelocation__location_id=current_location.pk) - .exclude(pk=device_pk) - .exists() - ) + current_location = device_location.location if not current_location or current_location.is_estimated: + existing_device_location = getattr( + devices_with_location.first(), "devicelocation", None + ) _handle_attach_existing_location( - device, device_location, whois_obj, attached_devices_exists + device, device_location, ip_address, existing_device_location ) else: logger.info( diff --git a/openwisp_controller/geo/estimated_location/tests/tests.py b/openwisp_controller/geo/estimated_location/tests/tests.py index 7b8eb1751..8760a7796 100644 --- a/openwisp_controller/geo/estimated_location/tests/tests.py +++ b/openwisp_controller/geo/estimated_location/tests/tests.py @@ -1,11 +1,13 @@ import contextlib import importlib +from datetime import timedelta from unittest import mock from django.contrib.gis.geos import GEOSGeometry from django.core.exceptions import ImproperlyConfigured, ValidationError from django.test import TestCase, TransactionTestCase, override_settings from django.urls import reverse +from django.utils import timezone from openwisp_notifications.types import unregister_notification_type from swapper import load_model @@ -16,6 +18,7 @@ from ....tests.utils import TestAdminMixin from ...tests.utils import TestGeoMixin from ..handlers import register_estimated_location_notification_types +from ..tasks import manage_estimated_locations from .utils import TestEstimatedLocationMixin Device = load_model("config", "Device") @@ -147,7 +150,7 @@ class TestEstimatedLocationTransaction( TestEstimatedLocationMixin, WHOISTransactionMixin, TransactionTestCase ): _WHOIS_GEOIP_CLIENT = ( - "openwisp_controller.config.whois.tasks.geoip2_webservice.Client" + "openwisp_controller.config.whois.service.geoip2_webservice.Client" ) _ESTIMATED_LOCATION_INFO_LOGGER = ( "openwisp_controller.geo.estimated_location.tasks.logger.info" @@ -174,7 +177,9 @@ def test_estimated_location_task_called( self, mocked_client, mocked_estimated_location_task ): connect_whois_handlers() - mocked_client.return_value.city.return_value = self._mocked_client_response() + mocked_response = self._mocked_client_response() + mocked_client.return_value.city.return_value = mocked_response + threshold = config_app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + 1 self._task_called( mocked_estimated_location_task, task_name="Estimated location" @@ -182,6 +187,8 @@ def test_estimated_location_task_called( Device.objects.all().delete() device = self._create_device() + self._create_config(device=device) + with self.subTest( "Estimated location task called when last_ip has related WhoIsInfo" ): @@ -207,7 +214,6 @@ def test_estimated_location_task_called( ): WHOISInfo.objects.all().delete() self._create_whois_info(ip_address=device.last_ip) - self._create_config(device=device) response = self.client.get( reverse("controller:device_checksum", args=[device.pk]), {"key": device.key}, @@ -217,10 +223,80 @@ def test_estimated_location_task_called( mocked_estimated_location_task.assert_not_called() mocked_estimated_location_task.reset_mock() + with self.subTest( + "Estimate location task not called when address/coordinates not updated" + ): + WHOISInfo.objects.all().delete() + whois_obj = self._create_whois_info(ip_address=device.last_ip) + WHOISInfo.objects.filter(pk=whois_obj.pk).update( + modified=timezone.now() - timedelta(days=threshold) + ) + device.save() + mocked_estimated_location_task.assert_not_called() + mocked_estimated_location_task.reset_mock() + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_estimated_location_task.assert_not_called() + mocked_estimated_location_task.reset_mock() + + with self.subTest( + "Estimate location task called when address/coordinates updated" + ): + WHOISInfo.objects.all().delete() + whois_obj = self._create_whois_info(ip_address=device.last_ip) + WHOISInfo.objects.filter(pk=whois_obj.pk).update( + modified=timezone.now() - timedelta(days=threshold) + ) + mocked_response.city.name = "New city" + mocked_client.return_value.city.return_value = mocked_response + device.save() + mocked_estimated_location_task.assert_called() + mocked_estimated_location_task.reset_mock() + mocked_response.city.name = "New city 2" + mocked_client.return_value.city.return_value = mocked_response + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_estimated_location_task.assert_called() + + mocked_response.location.latitude = 60 + mocked_client.return_value.city.return_value = mocked_response + device.save() + mocked_estimated_location_task.assert_called() + mocked_estimated_location_task.reset_mock() + mocked_response.location.longitude = 160 + mocked_client.return_value.city.return_value = mocked_response + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_estimated_location_task.assert_called() + mocked_estimated_location_task.reset_mock() + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch( + "openwisp_controller.config.whois.service.send_whois_task_notification" # noqa + ) + @mock.patch( + "openwisp_controller.geo.estimated_location.tasks.send_whois_task_notification" # noqa + ) + @mock.patch( + "openwisp_controller.geo.estimated_location.tasks.manage_estimated_locations.delay" # noqa + ) @mock.patch(_ESTIMATED_LOCATION_INFO_LOGGER) @mock.patch(_WHOIS_GEOIP_CLIENT) - def test_estimated_location_creation_and_update(self, mock_client, mock_info): + def test_estimated_location_creation_and_update( + self, mock_client, mock_info, _mocked_task, _mocked_notify, _mocked_notify2 + ): connect_whois_handlers() def _verify_location_details(device, mocked_response): @@ -254,6 +330,8 @@ def _verify_location_details(device, mocked_response): with self.subTest("Test Estimated location created when device is created"): device = self._create_device(last_ip="172.217.22.14") + with self.assertNumQueries(14): + manage_estimated_locations(device.pk, device.last_ip) location = device.devicelocation.location mocked_response.ip_address = device.last_ip @@ -274,6 +352,8 @@ def _verify_location_details(device, mocked_response): mocked_response.city.name = "New City" mock_client.return_value.city.return_value = mocked_response device.save() + with self.assertNumQueries(8): + manage_estimated_locations(device.pk, device.last_ip) device.refresh_from_db() location = device.devicelocation.location @@ -297,6 +377,8 @@ def _verify_location_details(device, mocked_response): mock_client.return_value.city.return_value = self._mocked_client_response() device.devicelocation.location.save(_set_estimated=True) device.save() + with self.assertNumQueries(2): + manage_estimated_locations(device.pk, device.last_ip) device.refresh_from_db() location = device.devicelocation.location @@ -315,12 +397,15 @@ def _verify_location_details(device, mocked_response): ): Device.objects.all().delete() device1 = self._create_device(last_ip="172.217.22.10") + manage_estimated_locations(device1.pk, device1.last_ip) mock_info.reset_mock() device2 = self._create_device( name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66", last_ip="172.217.22.10", ) + with self.assertNumQueries(8): + manage_estimated_locations(device2.pk, device2.last_ip) self.assertEqual( device1.devicelocation.location.pk, device2.devicelocation.location.pk @@ -336,15 +421,20 @@ def _verify_location_details(device, mocked_response): ): Device.objects.all().delete() device1 = self._create_device(last_ip="172.217.22.10") + manage_estimated_locations(device1.pk, device1.last_ip) device2 = self._create_device( name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66", last_ip="172.217.22.11", ) + manage_estimated_locations(device2.pk, device2.last_ip) mock_info.reset_mock() old_location = device2.devicelocation.location device2.last_ip = "172.217.22.10" device2.save() + # 3 queries related to notifications cleanup + with self.assertNumQueries(16): + manage_estimated_locations(device2.pk, device2.last_ip) mock_info.assert_called_once_with( f"Estimated location saved successfully for {device2.pk}" f" for IP: {device2.last_ip}" @@ -363,17 +453,21 @@ def _verify_location_details(device, mocked_response): ): Device.objects.all().delete() device1 = self._create_device(last_ip="172.217.22.10") + manage_estimated_locations(device1.pk, device1.last_ip) device2 = self._create_device( name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66", last_ip="172.217.22.11", ) + manage_estimated_locations(device2.pk, device2.last_ip) mock_info.reset_mock() old_location = device2.devicelocation.location old_location.is_estimated = False old_location.save() device2.last_ip = "172.217.22.10" device2.save() + with self.assertNumQueries(2): + manage_estimated_locations(device2.pk, device2.last_ip) mock_info.assert_called_once_with( f"Non Estimated location already set for {device2.pk}. Update" f" location manually as per IP: {device2.last_ip}" @@ -392,17 +486,21 @@ def _verify_location_details(device, mocked_response): ): Device.objects.all().delete() device1 = self._create_device(last_ip="172.217.22.10") + manage_estimated_locations(device1.pk, device1.last_ip) device2 = self._create_device( name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66", last_ip="172.217.22.10", ) + manage_estimated_locations(device2.pk, device2.last_ip) mock_info.reset_mock() self.assertEqual( device1.devicelocation.location.pk, device2.devicelocation.location.pk ) device2.last_ip = "172.217.22.11" device2.save() + with self.assertNumQueries(14): + manage_estimated_locations(device2.pk, device2.last_ip) mock_info.assert_called_once_with( f"Estimated location saved successfully for {device2.pk}" f" for IP: {device2.last_ip}" @@ -480,6 +578,7 @@ def test_estimate_location_status_remove(self, mock_client): location = device.devicelocation.location self.assertTrue(location.is_estimated) org = self._get_org() + connect_whois_handlers() with self.subTest( "Test Estimated Status unchanged if Estimated feature is disabled" diff --git a/openwisp_controller/geo/templates/admin/geo/location/change_form.html b/openwisp_controller/geo/templates/admin/geo/location/change_form.html index 96b4392e9..af2fc9ceb 100644 --- a/openwisp_controller/geo/templates/admin/geo/location/change_form.html +++ b/openwisp_controller/geo/templates/admin/geo/location/change_form.html @@ -1,4 +1,4 @@ -{% extends "admin/change_form.html" %} +{% extends "admin/django_loci/location_change_form.html" %} {% load i18n admin_urls %} {% block messages %} From ea0f1210e62dd8d74f656b847ba1b38e8aae0e6f Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Thu, 26 Feb 2026 16:07:50 -0300 Subject: [PATCH 5/5] [chores] Improved WHOIS & Estimated Location features Fixed multiple issues discovered during testing and review across WHOIS, estimated location, and device IP handling, and consolidated QA and maintenance changes. Key changes: - Ensure clear_last_ip command calls model save() to trigger signals and cache invalidation - Fix WHOIS and estimated location bugs, including shared location recreation and modified timestamp updates - Improve handling of unchanged IPs in DeviceChecksumView - Rename and adjust migrations for estimated location settings - Optimize queries and improve error handling (global HttpError handling) - Improve UI (CSS/JS/templates) and documentation - Improved efficiency of `manage_estimated_locations` - Resolved N+1 query issue in `LocationListCreateView` - Fixed HTTP request inside `transaction.atomic()` - Shifted to Celery `send_task` to retain dependency flow - Removed whois info from device list API to avoid N+1 - Simplified auto-naming of estimated locations - Updated location update description messages - Increased WHOIS refresh threshold to 90 days (~92% reduction in API calls) - WHOIS data refresh now executes on transaction commit - Skips `delete_whois_record` if `initial_ip_address` is None - Fixed late-binding closure issue with `_initial_last_ip` - Error notifications now only sent if `device_pk` is not None - CIDR field length increased to accommodate IPv6 - Added `is_estimated` filter and column to admin - Added `is_estimated` to `LocationSerializer` and API filter - Improved test coverage - Fixed parallel test 401 errors - Several other improvements and fixes for problems found during testing Co-authored-by: DragnEmperor Co-authored-by: Gagan Deep --- docs/index.rst | 2 +- docs/user/estimated-location.rst | 142 ++-- docs/user/intro.rst | 5 + docs/user/rest-api.rst | 8 +- docs/user/settings.rst | 54 +- docs/user/whois.rst | 92 +-- openwisp_controller/config/admin.py | 2 +- openwisp_controller/config/api/serializers.py | 4 +- openwisp_controller/config/base/device.py | 12 +- openwisp_controller/config/base/whois.py | 31 +- .../config/controller/views.py | 3 - .../management/commands/clear_last_ip.py | 44 +- .../config/migrations/0062_whoisinfo.py | 9 +- ...gs_estimated_location_enabled_and_more.py} | 0 openwisp_controller/config/settings.py | 22 +- .../config/static/whois/css/whois.css | 3 - .../config/static/whois/js/whois.js | 24 +- .../templates/admin/config/change_form.html | 7 +- .../config/tests/test_admin.py | 3 + .../config/tests/test_device.py | 10 + openwisp_controller/config/whois/commands.py | 66 ++ openwisp_controller/config/whois/mixins.py | 16 +- .../config/whois/serializers.py | 12 - openwisp_controller/config/whois/service.py | 118 ++-- openwisp_controller/config/whois/tasks.py | 106 ++- .../config/whois/tests/tests.py | 642 ++++++++++++++---- .../config/whois/tests/utils.py | 57 +- openwisp_controller/config/whois/utils.py | 46 +- openwisp_controller/geo/admin.py | 37 +- openwisp_controller/geo/api/serializers.py | 9 +- openwisp_controller/geo/api/views.py | 25 +- openwisp_controller/geo/base/models.py | 56 +- .../geo/estimated_location/handlers.py | 14 +- .../geo/estimated_location/mixins.py | 13 +- .../geo/estimated_location/tasks.py | 70 +- .../geo/estimated_location/tests/tests.py | 522 ++++++++++---- .../geo/estimated_location/tests/utils.py | 19 + .../geo/estimated_location/utils.py | 45 +- .../migrations/0004_location_is_estimated.py | 1 + .../admin/geo/location/change_form.html | 4 +- openwisp_controller/geo/tests/test_api.py | 7 +- requirements.txt | 2 +- ...rganizationconfigsettings_whois_enabled.py | 10 +- .../migrations/0004_location_is_estimated.py | 1 + 44 files changed, 1656 insertions(+), 719 deletions(-) rename openwisp_controller/config/migrations/{0063_organizationconfigsettings_approximate_location_enabled_and_more.py => 0063_organizationconfigsettings_estimated_location_enabled_and_more.py} (100%) create mode 100644 openwisp_controller/config/whois/commands.py diff --git a/docs/index.rst b/docs/index.rst index c6b268f55..8bdb49d88 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -47,9 +47,9 @@ the OpenWISP architecture. user/vxlan-wireguard.rst user/zerotier.rst user/openvpn.rst - user/subnet-division-rules.rst user/whois.rst user/estimated-location.rst + user/subnet-division-rules.rst user/rest-api.rst user/settings.rst diff --git a/docs/user/estimated-location.rst b/docs/user/estimated-location.rst index 48b3c100f..55c9188da 100644 --- a/docs/user/estimated-location.rst +++ b/docs/user/estimated-location.rst @@ -1,13 +1,19 @@ Estimated Location ================== +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/is-estimated-flag.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/is-estimated-flag.png + :alt: Estimated location flag + .. important:: The **Estimated Location** feature is **disabled by default**. - Before enabling it, the :doc:`WHOIS Lookup feature ` must be - enabled. Then set - :ref:`OPENWISP_CONTROLLER_ESTIMATED_LOCATION_ENABLED` to ``True`` + Before enabling it, the :ref:`WHOIS Lookup feature + ` must be enabled. + + Then set :ref:`OPENWISP_CONTROLLER_ESTIMATED_LOCATION_ENABLED` to + ``True``. .. contents:: **Table of contents**: :depth: 1 @@ -16,72 +22,102 @@ Estimated Location Overview -------- -This feature automatically creates or updates a device’s location based on -latitude and longitude information retrieved from the WHOIS Lookup +This feature automatically creates or updates a device's location based on +latitude and longitude information retrieved from the :doc:`whois` feature. -Trigger Conditions ------------------- +It is very useful for those users who have devices scattered across +different geographic regions and would like some help to place the devices +on the map, while being gently reminded to improve the precision of the +location with a direct link for doing so. -This feature is triggered when: +It also significantly reduces the effort required to assign a geographic +location manually when many devices are deployed in large buildings like +schools, offices, hospitals, libraries, etc. Improve the precision of the +estimated location just once and all the other devices sharing the same +public IP will automatically inherit the same location. -- A **fresh WHOIS lookup** is performed for a device. -- Or when a WHOIS record already exists for the device’s IP **and**: +The feature is not useful in the following scenarios: - - The device’s last IP address is **public**. - - WHOIS lookup and Estimated Location is **enabled** for the device’s - organization. +- Most devices are deployed in one single location. +- Most devices are mobile (e.g. moving vehicles). -Behavior --------- +Visibility of Estimated Status +------------------------------ -The system will **attach the already existing matching location** of -another device with same ip to the current device if: +The estimated status of a location is visible in the admin interface in +several ways: -- Only one device is found with that IP and it has a location. -- The current device **has no location** or that location is - **estimated**. +- The location name will mention the *IP address* from which it was + estimated. +- A *warning message* appears at the top of the location list page as in + the image below. -If there are multiple devices with location for the same IP, the system -will **not attach any location** to the current device and a notification -will be sent suggesting the user to manually assign/create a location for -the device. + .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/estimated-warning.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/estimated-warning.png + :alt: Estimated location warning -If there is **no matching location**, a new estimated location is created -or the existing one is updated using coordinates from the WHOIS record, -but only if the existing location is estimated. +- The *Is Estimated?* flag is displayed both in the location list page and + in the location detail page, as in the images below. -If two devices share the same IP address and are assigned to the same -location, and the last IP of one of the devices is updated, the system -will create a new estimated location for that device. + .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/admin-list.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/admin-list.png + :alt: Estimated location admin list -Visibility of Estimated Status ------------------------------- + .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/is-estimated-flag.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/is-estimated-flag.png + :alt: Estimated location flag + +- The device list page also allows filtering devices which are associated + with estimated locations as shown in the image below. -The estimated status of a location is visible on the location page if the -feature is enabled for the organization. The location admin page also -includes indicators for the estimated status. + .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/filter-devices-by-estimated-location.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/filter-devices-by-estimated-location.png + :alt: Filter devices associated to estimated locations -- The name of the location will have suffix **(Estimated Location : - )**. -- A warning on top of the page. -- **Is Estimated** field. +Any change to the geographic coordinates of an estimated location will set +the ``is_estimated`` field to ``False``. -Changes to the ``coordinates`` and ``geometry`` of the estimated location -will set the ``is_estimated`` field to ``False`` and remove the -"(Estimated Location)" suffix with IP from the location name. +When manually increasing the precision of estimated locations, it is +highly recommended to also change the auto-generated location name. -In REST API, the field will be visible in the :ref:`Device Location -`, :ref:`Location list +In the REST API, the ``is_estimated`` field is visible in the :ref:`Device +Location `, :ref:`Location list `, :ref:`Location Detail -` and :ref:`Location list (GeoJson) -` if the feature is **enabled**. The field can -also be used for filtering in the location list (including geojson) -endpoints and in the :ref:`Device List `. +` and :ref:`Location list (GeoJSON) +` endpoints if the feature is enabled. The +field can also be used for filtering in the location list endpoints, +including the GeoJSON endpoint, and in the :ref:`Device List +`. + +Triggers and Record Management +------------------------------ + +The feature is triggered automatically when all the following conditions +are met: + +- A WHOIS lookup is performed. +- The last IP is a public IP address. +- Both WHOIS lookup and Estimated Location features are enabled for the + device's organization. + +If no matching location exists, a new estimated location is created using +coordinates from the WHOIS record. If an estimated location already +exists, it will be updated with the new coordinates. + +If another device with the same IP already has a location, the system will +assign the same location for any device having the same IP and not being +assigned to any other location. + +If two devices share the same IP and are assigned to the same location, +and one of them updates its last IP, the system will create a new +estimated location for that device. -Managing Older Estimated Locations ----------------------------------- +When multiple devices with the same IP already have a location assigned +but the locations differ, the system will send a notification to network +administrators asking to manually resolve the ambiguity. -Whenever location related fields in WHOIS records are updated as per -:ref:`Managing WHOIS Older Records `; the location -will also be updated automatically. +When WHOIS records are updated as described in :ref:`the WHOIS Lookup +section `, any related estimated +location will also be updated, if needed and only if the estimated +location has not been manually modified to increase precision. diff --git a/docs/user/intro.rst b/docs/user/intro.rst index 98c4b0abe..1c4b123ae 100644 --- a/docs/user/intro.rst +++ b/docs/user/intro.rst @@ -35,6 +35,8 @@ following features: - **VPN management**: automatically provision VPN tunnel configurations, including cryptographic keys and IP addresses, e.g.: :doc:`OpenVPN `, :doc:`WireGuard ` +- :doc:`whois`: display information about the public IP address used by + devices to communicate with OpenWISP - :doc:`import-export` It exposes various :doc:`REST API endpoints `. @@ -96,6 +98,9 @@ The geographic app is based on `django-loci geographic coordinates of the devices, as well as their indoor coordinates on floor plan images. +It also provides an :doc:`estimated-location` feature which automatically +creates or updates device locations based on WHOIS data. + It exposes various :doc:`REST API endpoints `. Subnet Division App diff --git a/docs/user/rest-api.rst b/docs/user/rest-api.rst index fe2efba0e..2ec0b48ed 100644 --- a/docs/user/rest-api.rst +++ b/docs/user/rest-api.rst @@ -80,9 +80,9 @@ information. **Estimated Location Filters** -if :doc:`Estimated Location feature ` is enabled, +If :doc:`Estimated Location feature ` is enabled, devices can be filtered based on the estimated nature of their location -using the ``geo_is_estimated``. +using the ``geo_is_estimated`` filter. **Available filters** @@ -814,7 +814,7 @@ List Locations |estimated_details| -Locations can also be filtered using the ``is_estimated``. +Locations can also be filtered using the ``is_estimated`` filter. **Available filters** @@ -951,7 +951,7 @@ List Locations with Devices Deployed (in GeoJSON Format) |estimated_details| -Locations can also be filtered using the ``is_estimated``. +Locations can also be filtered using the ``is_estimated`` filter. **Available filters** diff --git a/docs/user/settings.rst b/docs/user/settings.rst index cf0d4a4de..aa59425ef 100644 --- a/docs/user/settings.rst +++ b/docs/user/settings.rst @@ -774,8 +774,8 @@ documentation regarding automatic retries for known errors. Allows enabling the optional :doc:`WHOIS Lookup feature `. -.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois-admin-setting.png - :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois-admin-setting.png +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-setting.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-setting.png :alt: WHOIS admin setting After enabling this feature, you have to set @@ -796,10 +796,10 @@ After enabling this feature, you have to set ============ ======= **type**: ``str`` -**default**: None +**default**: ``""`` ============ ======= -Maxmind Account ID required for the :doc:`WHOIS Lookup feature `. +MaxMind Account ID required for the :doc:`WHOIS Lookup feature `. .. _openwisp_controller_whois_geoip_key: @@ -808,15 +808,28 @@ Maxmind Account ID required for the :doc:`WHOIS Lookup feature `. ============ ======= **type**: ``str`` -**default**: None +**default**: ``""`` ============ ======= -Maxmind License Key required for the :doc:`WHOIS Lookup feature `. +MaxMind License Key required for the :doc:`WHOIS Lookup feature `. -.. _openwisp_controller_whois_estimated_location_enabled: +.. _openwisp_controller_whois_refresh_threshold_days: + +``OPENWISP_CONTROLLER_WHOIS_REFRESH_THRESHOLD_DAYS`` +---------------------------------------------------- -``OPENWISP_CONTROLLER_WHOIS_ESTIMATED_LOCATION_ENABLED`` --------------------------------------------------------- +============ ======= +**type**: ``int`` +**default**: ``90`` +============ ======= + +Specifies the number of days after which the WHOIS information for a +device is considered stale and eligible for refresh. + +.. _openwisp_controller_estimated_location_enabled: + +``OPENWISP_CONTROLLER_ESTIMATED_LOCATION_ENABLED`` +-------------------------------------------------- ============ ========= **type**: ``bool`` @@ -826,19 +839,14 @@ Maxmind License Key required for the :doc:`WHOIS Lookup feature `. Allows enabling the optional :doc:`Estimated Location feature `. -.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-location-setting.png - :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-location-setting.png - :alt: Estimated Location setting +.. warning:: -.. _openwisp_controller_whois_refresh_threshold_days: + :ref:`OPENWISP_CONTROLLER_WHOIS_ENABLED + ` must be set to ``True`` before + enabling this feature. Enabling estimated locations while + ``WHOIS_ENABLED=False`` will raise ``ImproperlyConfigured`` at startup + time. -``OPENWISP_CONTROLLER_WHOIS_REFRESH_THRESHOLD_DAYS`` ----------------------------------------------------- - -============ ======= -**type**: ``int`` -**default**: ``14`` -============ ======= - -Specifies the number of days after which the WHOIS information for a -device is considered stale and eligible for refresh. +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/admin-setting.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/admin-setting.png + :alt: Estimated Location setting diff --git a/docs/user/whois.rst b/docs/user/whois.rst index 578bce597..264b2b163 100644 --- a/docs/user/whois.rst +++ b/docs/user/whois.rst @@ -1,12 +1,15 @@ WHOIS Lookup ============ +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-details.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-details.png + :alt: WHOIS admin details + .. important:: The **WHOIS Lookup** feature is **disabled by default**. - To enable it, follow the `setup steps - `_ below. + To enable it, follow the :ref:`controller_setup_whois_lookup` below. .. contents:: **Table of contents**: :depth: 1 @@ -20,6 +23,13 @@ devices to communicate with OpenWISP (via the ``last_ip`` field). It helps identify the geographic location and ISP associated with the IP address, which can be useful for troubleshooting network issues. +.. note:: + + Once WHOIS Lookups are enabled, no manual intervention is needed: + everything is handled automatically, including the refresh of old + data. See :ref:`controller_whois_auto_management` for more + information. + The retrieved information pertains to the Autonomous System (ASN) associated with the device's public IP address and includes: @@ -30,35 +40,10 @@ associated with the device's public IP address and includes: - Timezone of the ASN's registered location - Coordinates (Latitude and Longitude) -Trigger Conditions ------------------- - -A WHOIS lookup is triggered automatically when: - -- A new device is registered. -- An existing device's last IP address is changed. -- A device fetches its checksum. - -However, the lookup will only run if **all** the following conditions are -met: - -- The device is either **newly created** or has a **changed last IP**. -- The device's last IP address is **public**. -- There is **no existing WHOIS record** for that IP. -- WHOIS lookup is **enabled** for the device's organization. - -Managing WHOIS Records ----------------------- - -If a device updates its last IP address, lookup is triggered for the **new -IP** and the **WHOIS record for the old IP** is deleted if no active -devices are associated with that IP address. - .. note:: - When a device with an associated WHOIS record is deleted, its WHOIS - record is automatically removed only if no active devices are - associated with that IP address. + This data also serves as a base for the :doc:`Estimated Location + feature <./estimated-location>`. .. _controller_setup_whois_lookup: @@ -82,7 +67,9 @@ Setup Instructions 6. Restart the application/containers if using ansible-openwisp2 or docker. 7. Run the ``clear_last_ip`` management command to clear the last IP - address of **all active devices across organizations**. + address of any active device which doesn't have WHOIS info yet across + all organizations (which will trigger the WHOIS lookup at the next + config checksum check). - If using ansible-openwisp2 (default directory is /opt/openwisp2, unless changed in Ansible playbook configuration): @@ -90,7 +77,7 @@ Setup Instructions .. code-block:: bash source /opt/openwisp2/env/bin/activate - python /opt/openwisp2/src/manage.py clear_last_ip + python /opt/openwisp2/manage.py clear_last_ip - If using docker: @@ -103,27 +90,40 @@ Viewing WHOIS Lookup Data ------------------------- Once the WHOIS Lookup feature is enabled and WHOIS data is available, the -retrieved details can be viewed in the following locations: +retrieved details can be viewed as follows: -- **Device Admin**: On the device's admin page, the WHOIS data is +- **Device Admin**: on the device's admin page, the WHOIS data is displayed alongside the device's last IP address. -.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois-admin-details.png - :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois-admin-details.png +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-details.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-details.png :alt: WHOIS admin details -- **Device REST API**: See WHOIS details in the :ref:`Device List - ` and :ref:`Device Detail ` - responses. +- **Device REST API**: Refer to :ref:`Device List ` and + :ref:`Device Detail `. + +.. _controller_whois_auto_management: -.. _whois_older_records: +Triggers and Record Management +------------------------------ -Managing Older WHOIS Records ----------------------------- +A WHOIS lookup is triggered automatically when: -If a record is older than :ref:`Threshold -`, it will be refreshed -automatically. +- A new device registers and the last IP is a public IP address. +- A device's last IP address changes and is a public IP address. +- A device fetches its checksum **and** either no WHOIS record exists for + the IP or the existing record is older than the :ref:`configured + threshold `. -The update mechanism will be triggered whenever a device is registered or -its last IP changes or fetches its checksum. +The lookup will only run if the device's last IP address is **public** and +WHOIS lookup is **enabled** for the device's organization. + +When a device updates its last IP address, a WHOIS lookup is triggered for +the **new IP** and the **WHOIS record for the old IP** is deleted, unless +any active devices are associated with that IP address. + +.. note:: + + When a device with an associated WHOIS record is deleted, its WHOIS + record is automatically removed, but only if no other active devices + are associated with the same IP address. diff --git a/openwisp_controller/config/admin.py b/openwisp_controller/config/admin.py index af9e20320..7c328a353 100644 --- a/openwisp_controller/config/admin.py +++ b/openwisp_controller/config/admin.py @@ -942,7 +942,7 @@ def get_extra_context(self, pk=None): # passing the whois details to the context to avoid # the need to make an additional request in the js if data := get_whois_info(pk): - ctx["device_whois_details"] = json.dumps(data) + ctx["device_whois_details"] = data return ctx def add_view(self, request, form_url="", extra_context=None): diff --git a/openwisp_controller/config/api/serializers.py b/openwisp_controller/config/api/serializers.py index 9402bbcbf..f07627e60 100644 --- a/openwisp_controller/config/api/serializers.py +++ b/openwisp_controller/config/api/serializers.py @@ -9,7 +9,7 @@ from ...serializers import BaseSerializer from .. import settings as app_settings -from ..whois.mixins import BriefWHOISMixin, WHOISMixin +from ..whois.mixins import WHOISMixin Template = load_model("config", "Template") Vpn = load_model("config", "Vpn") @@ -221,7 +221,7 @@ class DeviceListConfigSerializer(BaseConfigSerializer): templates = FilterTemplatesByOrganization(many=True, write_only=True) -class DeviceListSerializer(BriefWHOISMixin, DeviceConfigSerializer): +class DeviceListSerializer(DeviceConfigSerializer): config = DeviceListConfigSerializer(required=False) class Meta(BaseMeta): diff --git a/openwisp_controller/config/base/device.py b/openwisp_controller/config/base/device.py index 2a2d88fd0..d985ccaab 100644 --- a/openwisp_controller/config/base/device.py +++ b/openwisp_controller/config/base/device.py @@ -119,12 +119,14 @@ class Meta: verbose_name_plural = app_settings.DEVICE_VERBOSE_NAME[1] def __init__(self, *args, **kwargs): + if ( + app_settings.WHOIS_CONFIGURED + and "last_ip" not in self._changed_checked_fields + ): + # Initial value for last_ip is required in WHOIS + # to remove WHOIS info related to that ip address. + self._changed_checked_fields = [*self._changed_checked_fields, "last_ip"] super().__init__(*args, **kwargs) - # Initial value for last_ip is required in WHOIS - # to remove WHOIS info related to that ip address. - if app_settings.WHOIS_CONFIGURED: - self._changed_checked_fields.append("last_ip") - self._set_initial_values_for_changed_checked_fields() def _set_initial_values_for_changed_checked_fields(self): diff --git a/openwisp_controller/config/base/whois.py b/openwisp_controller/config/base/whois.py index 4ae29aebb..6a2abac38 100644 --- a/openwisp_controller/config/base/whois.py +++ b/openwisp_controller/config/base/whois.py @@ -5,8 +5,6 @@ from django.core.exceptions import ValidationError from django.db import models, transaction from django.utils.translation import gettext_lazy as _ -from jsonfield import JSONField -from swapper import load_model from openwisp_utils.base import TimeStampedEditableModel @@ -34,7 +32,7 @@ class AbstractWHOISInfo(TimeStampedEditableModel): help_text=_("Organization for ASN"), ) asn = models.CharField( - max_length=6, + max_length=10, blank=True, help_text=_("ASN"), ) @@ -43,13 +41,13 @@ class AbstractWHOISInfo(TimeStampedEditableModel): blank=True, help_text=_("Time zone"), ) - address = JSONField( + address = models.JSONField( default=dict, help_text=_("Address"), blank=True, ) cidr = models.CharField( - max_length=20, + max_length=49, blank=True, help_text=_("CIDR"), ) @@ -81,8 +79,7 @@ def clean(self): except ValueError as e: raise ValidationError( {"cidr": _("Invalid CIDR format: %(error)s") % {"error": str(e)}} - ) - + ) from e if self.coordinates: if not (-90 <= self.coordinates.y <= 90): raise ValidationError( @@ -104,17 +101,8 @@ def device_whois_info_delete_handler(instance, **kwargs): Delete WHOIS information for a device when the last IP address is removed or when device is deleted. """ - Device = load_model("config", "Device") - last_ip = instance.last_ip - existing_devices = Device.objects.filter(_is_deactivated=False).filter( - last_ip=last_ip - ) - if ( - last_ip - and instance._get_organization__config_settings().whois_enabled - and not existing_devices.exists() - ): + if last_ip and instance._get_organization__config_settings().whois_enabled: transaction.on_commit(lambda: delete_whois_record.delay(last_ip)) # this method is kept here instead of in OrganizationConfigSettings because @@ -155,8 +143,13 @@ def _location_name(self): if address: parts = [part.strip() for part in address.split(",")[:2] if part.strip()] location = ", ".join(parts) - return _(f"{location} (Estimated Location: {self.ip_address})") - return _(f"Estimated Location: {self.ip_address}") + # Use named placeholders so translators receive the template + return _("%(location)s: %(ip)s") % { + "location": location, + "ip": self.ip_address, + } + # Use named placeholder for consistency + return _("Estimated Location: %(ip)s") % {"ip": self.ip_address} def _get_defaults_for_estimated_location(self): """ diff --git a/openwisp_controller/config/controller/views.py b/openwisp_controller/config/controller/views.py index 5e1c9fa10..3dc714c0c 100644 --- a/openwisp_controller/config/controller/views.py +++ b/openwisp_controller/config/controller/views.py @@ -153,9 +153,6 @@ def get(self, request, pk): # updates cache if ip addresses changed if updated: self.update_device_cache(device) - # check if WHOIS Info of device requires update - else: - device.whois_service.update_whois_info() checksum_requested.send( sender=device.__class__, instance=device, request=request ) diff --git a/openwisp_controller/config/management/commands/clear_last_ip.py b/openwisp_controller/config/management/commands/clear_last_ip.py index 7c68a3d4b..55ae9ba20 100644 --- a/openwisp_controller/config/management/commands/clear_last_ip.py +++ b/openwisp_controller/config/management/commands/clear_last_ip.py @@ -1,11 +1,11 @@ -from django.core.management.base import BaseCommand, CommandError -from django.db.models import OuterRef, Subquery -from swapper import load_model +from django.core.management.base import BaseCommand + +from openwisp_controller.config.whois.commands import clear_last_ip_command class Command(BaseCommand): help = ( - "Clears the last IP address (if set) for every active device" + "Clears the last IP address (if set) for active devices without WHOIS records" " across all organizations." ) @@ -17,38 +17,10 @@ def add_arguments(self, parser): dest="interactive", help="Do NOT prompt the user for input of any kind.", ) - return super().add_arguments(parser) def handle(self, *args, **options): - Device = load_model("config", "Device") - WHOISInfo = load_model("config", "WHOISInfo") - - if options["interactive"]: - message = ["\n"] - message.append( - "This will clear last IP of all active devices across organizations!\n" - ) - message.append( - "Are you sure you want to do this?\n\n" - "Type 'yes' to continue, or 'no' to cancel: " - ) - if input("".join(message)).lower() != "yes": - raise CommandError("Operation cancelled by user.") - - devices = Device.objects.filter(_is_deactivated=False).only("last_ip") - # Filter out devices that have WHOIS information for their last IP - devices = devices.exclude(last_ip=None).exclude( - last_ip__in=Subquery( - WHOISInfo.objects.filter(ip_address=OuterRef("last_ip")).values( - "ip_address" - ) - ), + clear_last_ip_command( + stdout=self.stdout, + stderr=self.stderr, + interactive=options["interactive"], ) - - updated_devices = devices.update(last_ip=None) - if updated_devices: - self.stdout.write( - f"Cleared last IP addresses for {updated_devices} active device(s)." - ) - else: - self.stdout.write("No active devices with last IP to clear.") diff --git a/openwisp_controller/config/migrations/0062_whoisinfo.py b/openwisp_controller/config/migrations/0062_whoisinfo.py index 4755b8485..8899f49c3 100644 --- a/openwisp_controller/config/migrations/0062_whoisinfo.py +++ b/openwisp_controller/config/migrations/0062_whoisinfo.py @@ -1,7 +1,6 @@ # Generated by Django 5.2.1 on 2025-06-26 02:13 import django.utils.timezone -import jsonfield.fields import model_utils.fields from django.db import migrations, models @@ -41,7 +40,7 @@ class Migration(migrations.Migration): ), ( "asn", - models.CharField(blank=True, help_text="ASN", max_length=6), + models.CharField(blank=True, help_text="ASN", max_length=10), ), ( "timezone", @@ -49,11 +48,9 @@ class Migration(migrations.Migration): ), ( "address", - jsonfield.fields.JSONField( - blank=True, default=dict, help_text="Address" - ), + models.JSONField(blank=True, default=dict, help_text="Address"), ), - ("cidr", models.CharField(blank=True, help_text="CIDR", max_length=20)), + ("cidr", models.CharField(blank=True, help_text="CIDR", max_length=49)), ( "isp", models.CharField( diff --git a/openwisp_controller/config/migrations/0063_organizationconfigsettings_approximate_location_enabled_and_more.py b/openwisp_controller/config/migrations/0063_organizationconfigsettings_estimated_location_enabled_and_more.py similarity index 100% rename from openwisp_controller/config/migrations/0063_organizationconfigsettings_approximate_location_enabled_and_more.py rename to openwisp_controller/config/migrations/0063_organizationconfigsettings_estimated_location_enabled_and_more.py diff --git a/openwisp_controller/config/settings.py b/openwisp_controller/config/settings.py index 8bcda862d..6c591c489 100644 --- a/openwisp_controller/config/settings.py +++ b/openwisp_controller/config/settings.py @@ -68,18 +68,26 @@ def get_setting(option, default): "API_TASK_RETRY_OPTIONS", dict(max_retries=5, retry_backoff=True, retry_backoff_max=600, retry_jitter=True), ) -WHOIS_REFRESH_THRESHOLD_DAYS = get_setting("WHOIS_REFRESH_THRESHOLD_DAYS", 14) -WHOIS_GEOIP_ACCOUNT = get_setting("WHOIS_GEOIP_ACCOUNT", None) -WHOIS_GEOIP_KEY = get_setting("WHOIS_GEOIP_KEY", None) +WHOIS_GEOIP_ACCOUNT = get_setting("WHOIS_GEOIP_ACCOUNT", "") +WHOIS_GEOIP_KEY = get_setting("WHOIS_GEOIP_KEY", "") WHOIS_ENABLED = get_setting("WHOIS_ENABLED", False) -WHOIS_CONFIGURED = WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY +WHOIS_CONFIGURED = bool(WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY) if WHOIS_ENABLED and not WHOIS_CONFIGURED: raise ImproperlyConfigured( - "WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY must be set " - + "when WHOIS_ENABLED is True." + "OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT and " + "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY must be set " + + "when OPENWISP_CONTROLLER_WHOIS_ENABLED is True." + ) +WHOIS_REFRESH_THRESHOLD_DAYS = get_setting("WHOIS_REFRESH_THRESHOLD_DAYS", 90) +if not ( + isinstance(WHOIS_REFRESH_THRESHOLD_DAYS, int) and WHOIS_REFRESH_THRESHOLD_DAYS > 0 +): + raise ImproperlyConfigured( + "OPENWISP_CONTROLLER_WHOIS_REFRESH_THRESHOLD_DAYS must be a positive integer" ) ESTIMATED_LOCATION_ENABLED = get_setting("ESTIMATED_LOCATION_ENABLED", False) if ESTIMATED_LOCATION_ENABLED and not WHOIS_ENABLED: raise ImproperlyConfigured( - "WHOIS must be enabled before enabling ESTIMATED_LOCATION globally" + "OPENWISP_CONTROLLER_WHOIS_ENABLED must be set to True before " + "setting OPENWISP_CONTROLLER_ESTIMATED_LOCATION_ENABLED to True." ) diff --git a/openwisp_controller/config/static/whois/css/whois.css b/openwisp_controller/config/static/whois/css/whois.css index 44a7b5d1f..2530cf296 100644 --- a/openwisp_controller/config/static/whois/css/whois.css +++ b/openwisp_controller/config/static/whois/css/whois.css @@ -77,9 +77,6 @@ font-weight: 600; cursor: pointer; } -details.whois[open] summary { - padding-bottom: 0 !important; -} .whois summary > div { display: flex; width: 100%; diff --git a/openwisp_controller/config/static/whois/js/whois.js b/openwisp_controller/config/static/whois/js/whois.js index 20f4339f5..7b8c20be7 100644 --- a/openwisp_controller/config/static/whois/js/whois.js +++ b/openwisp_controller/config/static/whois/js/whois.js @@ -1,11 +1,17 @@ "use strict"; -if (typeof gettext === "undefined") { - var gettext = function (word) { +var gettext = + window.gettext || + function (word) { return word; }; +// For XSS prevention +function escapeHtml(text) { + if (text == null) return ""; + const div = document.createElement("div"); + div.textContent = text; + return div.innerHTML; } - django.jQuery(function ($) { const $addForm = $(".add-form"); const $deviceForm = $("#device_form"); @@ -34,8 +40,8 @@ django.jQuery(function ($) { ${gettext("Country")} - ${deviceWHOISDetails.isp} - ${deviceWHOISDetails.address.country} + ${escapeHtml(deviceWHOISDetails.isp)} + ${escapeHtml(deviceWHOISDetails.address.country)}
@@ -46,10 +52,10 @@ django.jQuery(function ($) {
- ${gettext("ASN")}: ${deviceWHOISDetails.asn} - ${gettext("Timezone")}: ${deviceWHOISDetails.timezone} - ${gettext("Address")}: ${deviceWHOISDetails.formatted_address} - ${gettext("CIDR")}: ${deviceWHOISDetails.cidr} + ${gettext("ASN")}: ${escapeHtml(deviceWHOISDetails.asn)} + ${gettext("Timezone")}: ${escapeHtml(deviceWHOISDetails.timezone)} + ${gettext("Address")}: ${escapeHtml(deviceWHOISDetails.formatted_address)} + ${gettext("CIDR")}: ${escapeHtml(deviceWHOISDetails.cidr)}
`, ); diff --git a/openwisp_controller/config/templates/admin/config/change_form.html b/openwisp_controller/config/templates/admin/config/change_form.html index 02536c00f..c4099580e 100644 --- a/openwisp_controller/config/templates/admin/config/change_form.html +++ b/openwisp_controller/config/templates/admin/config/change_form.html @@ -12,6 +12,9 @@ {% endblock %} {% block extrahead %} + {% if device_whois_details %} + {{ device_whois_details|json_script:"device-whois-details" }} + {% endif %} {{ block.super}} diff --git a/openwisp_controller/config/tests/test_admin.py b/openwisp_controller/config/tests/test_admin.py index 3be6f448a..aa9e0ebca 100644 --- a/openwisp_controller/config/tests/test_admin.py +++ b/openwisp_controller/config/tests/test_admin.py @@ -2267,14 +2267,17 @@ def _verify_template_queries(self, config, count): self.assertEqual(response.status_code, 200) # ensuring queries are consistent for different number of templates + @patch.object(app_settings, "WHOIS_CONFIGURED", False) def test_templates_fetch_queries_1(self): config = self._create_config(organization=self._get_org()) self._verify_template_queries(config, 1) + @patch.object(app_settings, "WHOIS_CONFIGURED", False) def test_templates_fetch_queries_5(self): config = self._create_config(organization=self._get_org()) self._verify_template_queries(config, 5) + @patch.object(app_settings, "WHOIS_CONFIGURED", False) def test_templates_fetch_queries_10(self): config = self._create_config(organization=self._get_org()) self._verify_template_queries(config, 10) diff --git a/openwisp_controller/config/tests/test_device.py b/openwisp_controller/config/tests/test_device.py index f40950440..9603606d3 100644 --- a/openwisp_controller/config/tests/test_device.py +++ b/openwisp_controller/config/tests/test_device.py @@ -531,6 +531,16 @@ def test_device_field_changed_checks(self): with self.assertNumQueries(3): device._check_changed_fields() + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_changed_checked_fields_no_duplicates(self): + """Ensure `_changed_checked_fields` contains `last_ip` only once. + This prevents duplicates if __init__ is invoked multiple times. + """ + device = Device() + # Simulate __init__ being called again on the same instance + device.__init__() + self.assertEqual(device._changed_checked_fields.count("last_ip"), 1) + def test_exceed_organization_device_limit(self): org = self._get_org() org.config_limits.device_limit = 1 diff --git a/openwisp_controller/config/whois/commands.py b/openwisp_controller/config/whois/commands.py new file mode 100644 index 000000000..dafbe92e8 --- /dev/null +++ b/openwisp_controller/config/whois/commands.py @@ -0,0 +1,66 @@ +from django.core.management.base import CommandError +from django.db.models import OuterRef, Subquery +from swapper import load_model + +from openwisp_controller.config import settings as app_settings + + +def clear_last_ip_command(stdout, stderr, interactive=True): + """ + Clears the last IP address (if set) for active devices without WHOIS records + across all organizations. + """ + if not app_settings.WHOIS_CONFIGURED: + raise CommandError( + "WHOIS lookup is not configured. Set " + "OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT " + "and OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY to enable this command." + ) + + Device = load_model("config", "Device") + WHOISInfo = load_model("config", "WHOISInfo") + + if interactive: # pragma: no cover + message = ["\n"] + message.append( + "This will clear the last IP of any active device which doesn't " + "have WHOIS info yet!\n" + ) + message.append( + "Are you sure you want to do this?\n\n" + "Type 'yes' to continue, or 'no' to cancel: " + ) + if input("".join(message)).lower() != "yes": + raise CommandError("Operation cancelled by user.") + + devices = ( + Device.objects.filter(_is_deactivated=False).select_related( + "organization__config_settings" + ) + # include the FK field 'organization' in .only() so the related + # `organization__config_settings` traversal is not deferred + .only("last_ip", "organization", "key") + ) + # Filter out devices that have WHOIS information for their last IP + devices = devices.exclude(last_ip=None).exclude( + last_ip__in=Subquery( + WHOISInfo.objects.filter(ip_address=OuterRef("last_ip")).values( + "ip_address" + ) + ), + ) + # We cannot use a queryset-level update here because it bypasses model save() + # and signals, which are required to properly invalidate related caches + # (e.g. DeviceChecksumView.get_device). To ensure correct behavior and + # future compatibility, each device is saved individually. + updated_devices = 0 + for device in devices.iterator(): + device.last_ip = None + device.save(update_fields=["last_ip"]) + updated_devices += 1 + if updated_devices: + stdout.write( + f"Cleared the last IP addresses for {updated_devices} active device(s)." + ) + else: + stdout.write("No active devices with last IP to clear.") diff --git a/openwisp_controller/config/whois/mixins.py b/openwisp_controller/config/whois/mixins.py index 4e2ee2ea7..3cd70949b 100644 --- a/openwisp_controller/config/whois/mixins.py +++ b/openwisp_controller/config/whois/mixins.py @@ -1,5 +1,5 @@ from .. import settings as app_settings -from .serializers import BriefWHOISSerializer, WHOISSerializer +from .serializers import WHOISSerializer class WHOISMixin: @@ -7,18 +7,14 @@ class WHOISMixin: serializer_class = WHOISSerializer - def to_representation(self, obj): - data = super().to_representation(obj) - if app_settings.WHOIS_CONFIGURED and obj.whois_service.is_whois_enabled: - data["whois_info"] = self.get_whois_info(obj) - return data - def get_whois_info(self, obj): whois_obj = obj.whois_service.get_device_whois_info() if not whois_obj: return None return self.serializer_class(whois_obj).data - -class BriefWHOISMixin(WHOISMixin): - serializer_class = BriefWHOISSerializer + def to_representation(self, obj): + data = super().to_representation(obj) + if app_settings.WHOIS_CONFIGURED and obj.whois_service.is_whois_enabled: + data["whois_info"] = self.get_whois_info(obj) + return data diff --git a/openwisp_controller/config/whois/serializers.py b/openwisp_controller/config/whois/serializers.py index fa859b8e6..028fc9426 100644 --- a/openwisp_controller/config/whois/serializers.py +++ b/openwisp_controller/config/whois/serializers.py @@ -4,18 +4,6 @@ WHOISInfo = load_model("config", "WHOISInfo") -class BriefWHOISSerializer(serializers.ModelSerializer): - """ - Serializer for brief representation of WHOIS model. - """ - - country = serializers.CharField(source="address.country", read_only=True) - - class Meta: - model = WHOISInfo - fields = ("isp", "country", "ip_address") - - class WHOISSerializer(serializers.ModelSerializer): """ Serializer for detailed representation of WHOIS model. diff --git a/openwisp_controller/config/whois/service.py b/openwisp_controller/config/whois/service.py index 71884d52d..d6f72d016 100644 --- a/openwisp_controller/config/whois/service.py +++ b/openwisp_controller/config/whois/service.py @@ -2,36 +2,19 @@ from ipaddress import ip_address as ip_addr import requests +from celery import current_app from django.contrib.gis.geos import Point from django.core.cache import cache from django.db import transaction from django.utils import timezone -from django.utils.translation import gettext as _ from geoip2 import errors from geoip2 import webservice as geoip2_webservice from swapper import load_model from openwisp_controller.config import settings as app_settings -from .tasks import fetch_whois_details, manage_estimated_locations -from .utils import send_whois_task_notification - -EXCEPTION_MESSAGES = { - errors.AddressNotFoundError: _( - "No WHOIS information found for IP address {ip_address}" - ), - errors.AuthenticationError: _( - "Authentication failed for GeoIP2 service. " - "Check your OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT and " - "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY settings." - ), - errors.OutOfQueriesError: _( - "Your account has run out of queries for the GeoIP2 service." - ), - errors.PermissionRequiredError: _( - "Your account does not have permission to access this service." - ), -} +from .tasks import fetch_whois_details +from .utils import EXCEPTION_MESSAGES, send_whois_task_notification class WHOISService: @@ -71,6 +54,7 @@ def is_valid_public_ip_address(ip): try: return ip and ip_addr(ip).is_global except ValueError: + # ip_address() from the stdlib raises ValueError for malformed strings return False @staticmethod @@ -83,11 +67,14 @@ def _get_whois_info_from_db(ip_address): return WHOISInfo.objects.filter(ip_address=ip_address) @staticmethod - def is_older(datetime): + def is_older(dt): """ Check if given datetime is older than the refresh threshold. + Raises TypeError if datetime is naive (not timezone-aware). """ - return (timezone.now() - datetime) >= timedelta( + if not timezone.is_aware(dt): + raise TypeError("datetime must be timezone-aware") + return (timezone.now() - dt) >= timedelta( days=app_settings.WHOIS_REFRESH_THRESHOLD_DAYS ) @@ -125,7 +112,7 @@ def get_org_config_settings(org_id): return org_settings @staticmethod - def check_estimate_location_configured(org_id): + def check_estimated_location_enabled(org_id): if not org_id: return False if not app_settings.WHOIS_CONFIGURED: @@ -174,9 +161,9 @@ def process_whois_details(self, ip_address): message = EXCEPTION_MESSAGES.get(exc_type) if exc_type is errors.AddressNotFoundError: message = message.format(ip_address=ip_address) - raise exc_type(message) - except requests.RequestException as e: - raise e + raise exc_type(message) from e + except requests.RequestException: + raise else: # The attributes are always present in the response, # but they can be None, so added fallbacks. @@ -186,16 +173,23 @@ def process_whois_details(self, ip_address): "continent": data.continent.name or "", "postal": str(data.postal.code or ""), } - coordinates = Point( - data.location.longitude, data.location.latitude, srid=4326 - ) + # Coordinates may be None in WHOIS response + # WHOISInfo.timezone is a non-nullable CharField, so store empty + # string when missing to avoid IntegrityError on save. + time_zone, coordinates = "", None + if location := data.location: + if location.latitude is not None and location.longitude is not None: + coordinates = Point( + location.longitude, location.latitude, srid=4326 + ) + time_zone = location.time_zone or "" return { - "isp": data.traits.autonomous_system_organization, - "asn": data.traits.autonomous_system_number, - "timezone": data.location.time_zone, + "isp": str(data.traits.autonomous_system_organization or ""), + "asn": str(data.traits.autonomous_system_number or ""), + "timezone": time_zone, "address": address, "coordinates": coordinates, - "cidr": data.traits.network, + "cidr": str(data.traits.network or ""), "ip_address": ip_address, } @@ -210,25 +204,35 @@ def _need_whois_lookup(self, new_ip): X days (defined by "WHOIS_REFRESH_THRESHOLD_DAYS"). - WHOIS is disabled in the organization settings. (query from db) """ - # Check cheap conditions first before hitting the database + if not self.is_whois_enabled: + return False if not self.is_valid_public_ip_address(new_ip): return False whois_obj = self._get_whois_info_from_db(ip_address=new_ip).first() if whois_obj and not self.is_older(whois_obj.modified): return False - return self.is_whois_enabled + return True def _need_estimated_location_management(self, new_ip): """ Used to determine if Estimated locations need to be created/updated or not during WHOIS lookup. """ + if not self.is_whois_enabled: + return False if not self.is_valid_public_ip_address(new_ip): return False - if not self.is_whois_enabled: + if not self.is_estimated_location_enabled: return False - return self.is_estimated_location_enabled + return True + + def trigger_estimated_location_task(self, ip_address): + """Helper method to trigger the estimated location task.""" + current_app.send_task( + "whois_estimated_location_task", + kwargs={"device_pk": self.device.pk, "ip_address": ip_address}, + ) def get_device_whois_info(self): """ @@ -238,7 +242,6 @@ def get_device_whois_info(self): ip_address = self.device.last_ip if not (self.is_valid_public_ip_address(ip_address) and self.is_whois_enabled): return None - return self._get_whois_info_from_db(ip_address=ip_address).first() def process_ip_data_and_location(self, force_lookup=False): @@ -249,11 +252,12 @@ def process_ip_data_and_location(self, force_lookup=False): Tasks are triggered on commit to ensure redundant data is not created. """ new_ip = self.device.last_ip + initial_ip = self.device._initial_last_ip if force_lookup or self._need_whois_lookup(new_ip): transaction.on_commit( lambda: fetch_whois_details.delay( device_pk=self.device.pk, - initial_ip_address=self.device._initial_last_ip, + initial_ip_address=initial_ip, ) ) # To handle the case when WHOIS already exists as in that case @@ -261,14 +265,16 @@ def process_ip_data_and_location(self, force_lookup=False): # manage estimated locations. elif self._need_estimated_location_management(new_ip): transaction.on_commit( - lambda: manage_estimated_locations.delay( - device_pk=self.device.pk, ip_address=new_ip + lambda: self.trigger_estimated_location_task( + ip_address=new_ip, ) ) def update_whois_info(self): """ - Update the WHOIS information for the device. + Update existing WHOIS data for the device + when the data is older than + ``OPENWISP_CONTROLLER_WHOIS_REFRESH_THRESHOLD_DAYS``. """ ip_address = self.device.last_ip if not self.is_valid_public_ip_address(ip_address): @@ -277,7 +283,12 @@ def update_whois_info(self): return whois_obj = WHOISService._get_whois_info_from_db(ip_address=ip_address).first() if whois_obj and self.is_older(whois_obj.modified): - fetch_whois_details.delay(device_pk=self.device.pk, initial_ip_address=None) + transaction.on_commit( + lambda: fetch_whois_details.delay( + device_pk=self.device.pk, + initial_ip_address=None, + ) + ) def _create_or_update_whois(self, whois_details, whois_instance=None): """ @@ -285,15 +296,18 @@ def _create_or_update_whois(self, whois_details, whois_instance=None): Returns the updated or created WHOIS instance along with update fields. """ WHOISInfo = load_model("config", "WHOISInfo") - update_fields = [] if whois_instance: for attr, value in whois_details.items(): + # whois_details already coerce to string most values if getattr(whois_instance, attr) != value: update_fields.append(attr) setattr(whois_instance, attr, value) - if update_fields: - whois_instance.save(update_fields=update_fields) + # bump modified time so staleness check + # doesn't re-trigger the lookup + update_fields.append("modified") + whois_instance.modified = timezone.now() + whois_instance.save(update_fields=update_fields) else: whois_instance = WHOISInfo(**whois_details) whois_instance.full_clean() @@ -325,11 +339,11 @@ def _create_or_update_estimated_location( device_location.location = current_location device_location.full_clean() device_location.save() - send_whois_task_notification( - device=self.device, - notify_type="estimated_location_created", - actor=current_location, - ) + send_whois_task_notification( + device=self.device, + notify_type="estimated_location_created", + actor=current_location, + ) elif current_location.is_estimated: update_fields = [] for attr, value in location_defaults.items(): @@ -341,11 +355,9 @@ def _create_or_update_estimated_location( current_location.save( update_fields=update_fields, _set_estimated=True ) - send_whois_task_notification( device=self.device, notify_type="estimated_location_updated", actor=current_location, ) - return current_location diff --git a/openwisp_controller/config/whois/tasks.py b/openwisp_controller/config/whois/tasks.py index 26b0a5037..78955600e 100644 --- a/openwisp_controller/config/whois/tasks.py +++ b/openwisp_controller/config/whois/tasks.py @@ -1,11 +1,11 @@ import logging from celery import shared_task +from django.core.cache import cache from django.db import transaction from geoip2 import errors from swapper import load_model -from openwisp_controller.geo.estimated_location.tasks import manage_estimated_locations from openwisp_utils.tasks import OpenwispCeleryTask from .. import settings as app_settings @@ -23,11 +23,32 @@ class WHOISCeleryRetryTask(OpenwispCeleryTask): # that should trigger a retry of the task. autoretry_for = (errors.HTTPError,) + def on_success(self, retval, task_id, args, kwargs): + """Mark the task as successfully completed.""" + task_key = f"{self.name}_last_operation" + cache.set(task_key, "success", None) + return super().on_success(retval, task_id, args, kwargs) + def on_failure(self, exc, task_id, args, kwargs, einfo): - """Notify the user about the failure of the WHOIS task.""" - device_pk = kwargs.get("device_pk") - send_whois_task_notification(device=device_pk, notify_type="whois_device_error") - logger.error(f"WHOIS lookup failed. Details: {exc}") + """ + Notify the user about the failure of the WHOIS task. + + Notifications are sent only once when task fails for the first time. + Subsequent failures do not trigger notifications until a successful run occurs. + """ + device_pk = kwargs.get("device_pk") or (args[0] if args else None) + if device_pk is not None: + # All exceptions are treated globally to prevent notification spam. + # The cache key is global (not per-device) to avoid spamming admins + # with multiple notifications for the same recurring issue. + task_key = f"{self.name}_last_operation" + last_operation = cache.get(task_key) + if last_operation != "errored": + cache.set(task_key, "errored", None) + send_whois_task_notification( + device=device_pk, notify_type="whois_device_error" + ) + logger.error(f"WHOIS lookup failed. Details: {exc}") return super().on_failure(exc, task_id, args, kwargs, einfo) @@ -45,50 +66,65 @@ def fetch_whois_details(self, device_pk, initial_ip_address): Device = load_model("config", "Device") WHOISInfo = load_model("config", "WHOISInfo") - with transaction.atomic(): - device = Device.objects.get(pk=device_pk) - new_ip_address = device.last_ip - WHOISService = device.whois_service + try: + device = Device.objects.select_related("devicelocation").get(pk=device_pk) + except Device.DoesNotExist: + logger.warning(f"Device {device_pk} not found, skipping WHOIS lookup") + return + new_ip_address = device.last_ip + whois_service = device.whois_service + # If there is existing WHOIS older record then it needs to be updated + whois_obj = WHOISInfo.objects.filter(ip_address=new_ip_address).first() + if whois_obj and not whois_service.is_older(whois_obj.modified): + return + # WARNING: execute HTTP requests before transaction lock is acquired + fetched_details = whois_service.process_whois_details(new_ip_address) - # If there is existing WHOIS older record then it needs to be updated - whois_obj = WHOISInfo.objects.filter(ip_address=new_ip_address).first() - if whois_obj and not WHOISService.is_older(whois_obj.modified): - return - - fetched_details = WHOISService.process_whois_details(new_ip_address) - whois_obj, update_fields = WHOISService._create_or_update_whois( + with transaction.atomic(): + whois_obj, update_fields = whois_service._create_or_update_whois( fetched_details, whois_obj ) logger.info(f"Successfully fetched WHOIS details for {new_ip_address}.") - - if device._get_organization__config_settings().estimated_location_enabled: - # the estimated location task should not run if old record is updated - # and location related fields are not updated - if update_fields and not any( - i in update_fields for i in ["address", "coordinates"] - ): - return - manage_estimated_locations.delay( - device_pk=device_pk, ip_address=new_ip_address + if initial_ip_address: + transaction.on_commit( + # execute synchronously as we're already in a background task + lambda: delete_whois_record(ip_address=initial_ip_address) ) - - # delete WHOIS record for initial IP if no devices are linked to it + if not device._get_organization__config_settings().estimated_location_enabled: + return + # the estimated location task should not run if old record is updated + # and location related fields are not updated + device_location = getattr(device, "devicelocation", None) if ( - not Device.objects.filter(_is_deactivated=False) - .filter(last_ip=initial_ip_address) - .exists() + device_location + and device_location.location + and update_fields + and not any(i in update_fields for i in ["address", "coordinates"]) ): - delete_whois_record(ip_address=initial_ip_address) + return + transaction.on_commit( + lambda: whois_service.trigger_estimated_location_task( + ip_address=new_ip_address, + ) + ) @shared_task -def delete_whois_record(ip_address): +def delete_whois_record(ip_address, force=False): """ Deletes the WHOIS record for the device's last IP address. This is used when the device is deleted or its last IP address is changed. + 'force' parameter is used to delete the record without checking for linked devices. """ + Device = load_model("config", "Device") WHOISInfo = load_model("config", "WHOISInfo") - queryset = WHOISInfo.objects.filter(ip_address=ip_address) - if queryset.exists(): + if force: queryset.delete() + else: + if ( + not Device.objects.filter(_is_deactivated=False) + .filter(last_ip=ip_address) + .exists() + ): + queryset.delete() diff --git a/openwisp_controller/config/whois/tests/tests.py b/openwisp_controller/config/whois/tests/tests.py index 3accc02c6..c3622ec96 100644 --- a/openwisp_controller/config/whois/tests/tests.py +++ b/openwisp_controller/config/whois/tests/tests.py @@ -1,17 +1,23 @@ +import copy import importlib from datetime import timedelta from io import StringIO from unittest import mock +from uuid import uuid4 +from django.conf import settings from django.contrib.gis.geos import Point from django.contrib.staticfiles.testing import StaticLiveServerTestCase +from django.core.cache import cache from django.core.exceptions import ImproperlyConfigured, ValidationError -from django.core.management import call_command +from django.core.management import CommandError, call_command from django.db.models.signals import post_delete, post_save from django.test import TestCase, TransactionTestCase, override_settings, tag from django.urls import reverse from django.utils import timezone from geoip2 import errors +from requests.exceptions import RequestException +from selenium.common.exceptions import UnexpectedAlertPresentException from selenium.webdriver.common.by import By from swapper import load_model @@ -20,16 +26,28 @@ from ....tests.utils import TestAdminMixin from ... import settings as app_settings from ..handlers import connect_whois_handlers +from ..service import WHOISService +from ..tasks import delete_whois_record, fetch_whois_details +from ..utils import get_whois_info, send_whois_task_notification from .utils import CreateWHOISMixin, WHOISTransactionMixin +Config = load_model("config", "Config") Device = load_model("config", "Device") WHOISInfo = load_model("config", "WHOISInfo") Notification = load_model("openwisp_notifications", "Notification") OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") -notification_qs = Notification.objects.all() +MODIFIED_CACHE = copy.deepcopy(settings.CACHES) +# add key_prefix to avoid conflicts in parallel tests +MODIFIED_CACHE["default"]["KEY_PREFIX"] = "whois_failure" +def _notification_qs(): + return Notification.objects.all() + + +# SESSION_ENGINE set to DB to avoid conflicts in parallel tests +@override_settings(SESSION_ENGINE="django.contrib.sessions.backends.db") class TestWHOIS(CreateWHOISMixin, TestAdminMixin, TestCase): # Signals are connected when apps are loaded, # and if WHOIS is Configured all related WHOIS @@ -59,6 +77,8 @@ def _disconnect_signals(self): ) def test_whois_configuration_setting(self): self._disconnect_signals() + self.addCleanup(importlib.reload, app_settings) + # ensure organization exists for admin page checks org = self._get_org() # reload app_settings to apply the overridden settings importlib.reload(app_settings) @@ -66,7 +86,6 @@ def test_whois_configuration_setting(self): with self.subTest("Test Signals not connected when WHOIS_CONFIGURED is False"): # should not connect any handlers since WHOIS_CONFIGURED is False connect_whois_handlers() - assert not any( "device.delete_whois_info" in str(r[0]) for r in post_delete.receivers ) @@ -135,16 +154,23 @@ def test_whois_configuration_setting(self): response = self.client.get(url) self.assertContains(response, 'name="config_settings-0-whois_enabled"') + def test_is_older_requires_timezone_aware(self): + """Verify is_older raises TypeError for naive datetimes.""" + naive_dt = timezone.now().replace(tzinfo=None) + with self.assertRaises(TypeError): + WHOISService.is_older(naive_dt) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) def test_whois_enabled(self): OrganizationConfigSettings.objects.all().delete() device = self._create_device() + with self.subTest( "Test WHOIS fallback when Organization settings do not exist" ): self.assertEqual( device.whois_service.is_whois_enabled, app_settings.WHOIS_ENABLED ) - org_settings_obj = OrganizationConfigSettings( organization=self._get_org(), whois_enabled=True ) @@ -154,30 +180,25 @@ def test_whois_enabled(self): app_settings, "WHOIS_CONFIGURED", False ), self.assertRaises(ValidationError) as context_manager: org_settings_obj.full_clean() - try: - self.assertEqual( - context_manager.exception.message_dict["whois_enabled"][0], - "WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY must be set " - + "before enabling WHOIS feature.", - ) - except AssertionError: - self.fail("ValidationError message not equal to expected message.") - - with mock.patch.object(app_settings, "WHOIS_CONFIGURED", True): - org_settings_obj.full_clean() - org_settings_obj.save() + self.assertEqual( + context_manager.exception.message_dict["whois_enabled"][0], + "WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY must be set " + + "before enabling WHOIS feature.", + ) with self.subTest("Test setting WHOIS enabled to True"): org_settings_obj.whois_enabled = True + org_settings_obj.full_clean() org_settings_obj.save(update_fields=["whois_enabled"]) org_settings_obj.refresh_from_db(fields=["whois_enabled"]) - self.assertEqual(getattr(org_settings_obj, "whois_enabled"), True) + self.assertTrue(org_settings_obj.whois_enabled) with self.subTest("Test setting WHOIS enabled to False"): org_settings_obj.whois_enabled = False + org_settings_obj.full_clean() org_settings_obj.save(update_fields=["whois_enabled"]) org_settings_obj.refresh_from_db(fields=["whois_enabled"]) - self.assertEqual(getattr(org_settings_obj, "whois_enabled"), False) + self.assertFalse(org_settings_obj.whois_enabled) with self.subTest( "Test setting WHOIS enabled to None fallbacks to global setting" @@ -185,10 +206,11 @@ def test_whois_enabled(self): # reload app_settings to ensure latest settings are applied importlib.reload(app_settings) org_settings_obj.whois_enabled = None + org_settings_obj.full_clean() org_settings_obj.save(update_fields=["whois_enabled"]) org_settings_obj.refresh_from_db(fields=["whois_enabled"]) self.assertEqual( - getattr(org_settings_obj, "whois_enabled"), + org_settings_obj.whois_enabled, app_settings.WHOIS_ENABLED, ) @@ -202,19 +224,11 @@ def test_whois_details_device_api(self): self._login() with self.subTest( - "Device List API has whois_info when WHOIS_CONFIGURED is True" + "Device List API does not have whois_info (removed to avoid N+1)" ): response = self.client.get(reverse("config_api:device_list")) self.assertEqual(response.status_code, 200) - self.assertIn("whois_info", response.data["results"][0]) - self.assertDictEqual( - response.data["results"][0]["whois_info"], - { - "isp": whois_obj.isp, - "country": whois_obj.address["country"], - "ip_address": whois_obj.ip_address, - }, - ) + self.assertNotIn("whois_info", response.data["results"][0]) with self.subTest( "Device Detail API has whois_info when WHOIS_CONFIGURED is True" @@ -232,14 +246,14 @@ def test_whois_details_device_api(self): self.assertEqual(api_whois_info["address"], whois_obj.address) with self.subTest( - "Device List API has whois_info as None when no WHOIS Info exists" + "Device List API does not have whois_info when no WHOIS Info exists" ): device.last_ip = "172.217.22.24" device.save() + device.refresh_from_db() response = self.client.get(reverse("config_api:device_list")) self.assertEqual(response.status_code, 200) - self.assertIn("whois_info", response.data["results"][0]) - self.assertIsNone(response.data["results"][0]["whois_info"]) + self.assertNotIn("whois_info", response.data["results"][0]) with self.subTest( "Device Detail API has whois_info as None when no WHOIS Info exists" @@ -258,7 +272,6 @@ def test_whois_details_device_api(self): response = self.client.get(reverse("config_api:device_list")) self.assertEqual(response.status_code, 200) self.assertNotIn("whois_info", response.data["results"][0]) - with self.subTest( "Device Detail API has no whois_info when WHOIS_CONFIGURED is False" ): @@ -268,20 +281,102 @@ def test_whois_details_device_api(self): self.assertEqual(response.status_code, 200) self.assertNotIn("whois_info", response.data) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_device_list_api_whois_no_nplus1(self): + """ + Test that WHOIS info doesn't cause N+1 queries in device list API. + Should use constant queries regardless of device count. + """ + self._login() + path = reverse("config_api:device_list") + # Create 3 devices with WHOIS info + for i in range(3): + device = self._create_device( + name=f"device{i}", + mac_address=f"00:11:22:33:44:{i:02x}", + last_ip=f"172.217.22.{i + 1}", + ) + WHOISInfo.objects.create( + ip_address=device.last_ip, + isp="Test ISP", + asn="12345", + address={"city": "Test City", "country": "Test Country"}, + cidr=f"172.217.22.{i + 1}/32", + ) + with self.subTest(f"Device List API with WHOIS info: {i}"): + with self.assertNumQueries(4): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data["results"]), i + 1) + for result in response.data["results"]: + self.assertNotIn("whois_info", result) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) def test_last_ip_management_command(self): out = StringIO() device = self._create_device(last_ip="172.217.22.11") args = ["--noinput"] call_command("clear_last_ip", *args, stdout=out, stderr=StringIO()) self.assertIn( - "Cleared last IP addresses for 1 active device(s).", out.getvalue() + "Cleared the last IP addresses for 1 active device(s).", out.getvalue() ) device.refresh_from_db() self.assertIsNone(device.last_ip) - + out.seek(0) + out.truncate(0) call_command("clear_last_ip", *args, stdout=out, stderr=StringIO()) self.assertIn("No active devices with last IP to clear.", out.getvalue()) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_last_ip_management_command_queries(self): + out = StringIO() + self._create_device(last_ip="172.217.22.11") + self._create_device( + name="default.test.device2", + last_ip="172.217.22.12", + mac_address="11:22:33:44:55:66", + ) + self._create_device( + name="default.test.device3", + last_ip="172.217.22.13", + mac_address="22:33:44:55:66:77", + ) + self._create_device( + name="default.test.device4", mac_address="66:33:44:55:66:77" + ) + args = ["--noinput"] + # 4 queries (3 for each device's last_ip update) and 1 for fetching devices + with self.assertNumQueries(4): + call_command("clear_last_ip", *args, stdout=out, stderr=StringIO()) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_last_ip_management_command_invalidates_cache(self): + device = self._create_device(last_ip="172.217.22.11") + self._create_config(device=device) + call_command("clear_last_ip", "--noinput", stdout=StringIO()) + device.refresh_from_db() + self.assertEqual(device.last_ip, None) + # We will use the DeviceChecksumView to set the last_ip again to + # the same value to verify that the command invalidates the cache + # and the view is able to send the same IP again. + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR="172.217.22.11", + ) + self.assertEqual(response.status_code, 200) + device.refresh_from_db() + self.assertEqual(device.last_ip, "172.217.22.11") + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", False) + def test_clear_last_ip_command_not_enabled(self): + """Test that clear_last_ip command raises error when WHOIS is not configured.""" + out = StringIO() + err = StringIO() + with self.assertRaises(CommandError) as context: + call_command("clear_last_ip", "--noinput", stdout=out, stderr=err) + self.assertIn("WHOIS lookup is not configured", str(context.exception)) + class TestWHOISInfoModel(CreateWHOISMixin, TestCase): def test_whois_model_fields_validation(self): @@ -290,35 +385,24 @@ def test_whois_model_fields_validation(self): """ with self.assertRaises(ValidationError): self._create_whois_info(isp="a" * 101) - with self.assertRaises(ValidationError) as context_manager: self._create_whois_info(ip_address="127.0.0.1") - try: - self.assertEqual( - context_manager.exception.message_dict["ip_address"][0], - "WHOIS information cannot be created for private IP addresses.", - ) - except AssertionError: - self.fail("ValidationError message not equal to expected message.") - + self.assertEqual( + context_manager.exception.message_dict["ip_address"][0], + "WHOIS information cannot be created for private IP addresses.", + ) with self.assertRaises(ValidationError): self._create_whois_info(timezone="a" * 36) - with self.assertRaises(ValidationError) as context_manager: self._create_whois_info(cidr="InvalidCIDR") - try: - # Not using assertEqual here because we are adding error message raised by - # ipaddress module to the ValidationError message. - self.assertIn( - "Invalid CIDR format: 'InvalidCIDR'", - context_manager.exception.message_dict["cidr"][0], - ) - except AssertionError: - self.fail("ValidationError message not equal to expected message.") - + # Not using assertEqual here because we are adding error message raised by + # ipaddress module to the ValidationError message. + self.assertIn( + "Invalid CIDR format: 'InvalidCIDR'", + context_manager.exception.message_dict["cidr"][0], + ) with self.assertRaises(ValidationError): - self._create_whois_info(asn="InvalidASN") - + self._create_whois_info(asn="InvalidASNNumber") # Common validation checks for longitude and latitude coordinates_cases = [ (150.0, 100.0, "Latitude must be between -90 and 90 degrees."), @@ -330,15 +414,14 @@ def test_whois_model_fields_validation(self): with self.assertRaises(ValidationError) as context_manager: point = Point(longitude, latitude, srid=4326) self._create_whois_info(coordinates=point) - try: - self.assertEqual( - context_manager.exception.message_dict["coordinates"][0], - expected_msg, - ) - except AssertionError: - self.fail("ValidationError message not equal to expected message.") + self.assertEqual( + context_manager.exception.message_dict["coordinates"][0], + expected_msg, + ) +# SESSION_ENGINE set to DB to avoid conflicts in parallel tests +@override_settings(SESSION_ENGINE="django.contrib.sessions.backends.db") class TestWHOISTransaction( CreateWHOISMixin, WHOISTransactionMixin, TransactionTestCase ): @@ -348,27 +431,72 @@ class TestWHOISTransaction( _WHOIS_TASKS_INFO_LOGGER = "openwisp_controller.config.whois.tasks.logger.info" _WHOIS_TASKS_WARN_LOGGER = "openwisp_controller.config.whois.tasks.logger.warning" _WHOIS_TASKS_ERR_LOGGER = "openwisp_controller.config.whois.tasks.logger.error" + _WHOIS_TASK_NAME = "openwisp_controller.config.whois.tasks.fetch_whois_details" def setUp(self): super().setUp() self.admin = self._get_admin() + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_process_whois_details_handles_missing_coordinates(self, mock_client): + """Ensure WHOIS processing tolerates missing coordinates in response.""" + connect_whois_handlers() + mocked_response = self._mocked_client_response() + # simulate missing coordinates + mocked_response.location = None + mock_client.return_value.city.return_value = mocked_response + device = self._create_device(last_ip="172.217.22.14") + whois_details = device.whois_service.process_whois_details(device.last_ip) + self.assertIsNone(whois_details.get("coordinates")) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) @mock.patch("openwisp_controller.config.whois.tasks.fetch_whois_details.delay") def test_whois_task_called(self, mocked_lookup_task): connect_whois_handlers() self._task_called(mocked_lookup_task) + Device.objects.all().delete() + WHOISInfo.objects.all().delete() + org = self._get_org() + org.config_settings.whois_enabled = True + org.config_settings.save() + + with self.subTest("WHOIS lookup task called when last_ip is public"): + with mock.patch( + "django.core.cache.cache.get", return_value=None + ) as mocked_get, mock.patch("django.core.cache.cache.set") as mocked_set: + device = self._create_device(last_ip="172.217.22.14") + mocked_lookup_task.assert_called() + mocked_set.assert_called_once_with( + f"organization_config_{org.pk}", + org.config_settings, + timeout=Config._CHECKSUM_CACHE_TIMEOUT, + ) + mocked_get.assert_called() + mocked_lookup_task.reset_mock() + + with self.subTest( + "WHOIS lookup task called when last_ip is changed and is public" + ): + with mock.patch("django.core.cache.cache.get") as mocked_get, mock.patch( + "django.core.cache.cache.set" + ) as mocked_set: + device.last_ip = "172.217.22.10" + device.save() + device.refresh_from_db() + mocked_lookup_task.assert_called() + mocked_set.assert_not_called() + mocked_get.assert_called() + mocked_lookup_task.reset_mock() - Device.objects.all().delete() # Clear existing devices - device = self._create_device() with self.subTest( "WHOIS lookup task not called when last_ip has related WhoIsInfo" ): - device.organization.config_settings.whois_enabled = True - device.organization.config_settings.save() device.last_ip = "172.217.22.14" self._create_whois_info(ip_address=device.last_ip) device.save() + device.refresh_from_db() + device.organization.config_settings.refresh_from_db() mocked_lookup_task.assert_not_called() mocked_lookup_task.reset_mock() @@ -409,9 +537,12 @@ def test_whois_multiple_orgs(self, mocked_task): with self.subTest("Test task calls when last_ip is changed and is public"): device1.last_ip = "172.217.22.12" device1.save() + device1.refresh_from_db() mocked_task.assert_called() mocked_task.reset_mock() device2.last_ip = "172.217.22.13" + device2.save() + device2.refresh_from_db() mocked_task.assert_not_called() mocked_task.reset_mock() @@ -474,26 +605,27 @@ def test_whois_creation(self, mock_client, mock_info): connect_whois_handlers() def _verify_whois_details(instance, ip_address): - self.assertEqual(instance.isp, "Google LLC") - self.assertEqual(instance.asn, "15169") - self.assertEqual(instance.timezone, "America/Los_Angeles") + mock_response = self._mocked_client_response() self.assertEqual( - instance.address, - { - "city": "Mountain View", - "country": "United States", - "continent": "North America", - "postal": "94043", - }, + instance.isp, mock_response.traits.autonomous_system_organization ) - self.assertEqual(instance.cidr, "172.217.22.0/24") - self.assertEqual(instance.ip_address, ip_address) self.assertEqual( - instance.formatted_address, - "Mountain View, United States, North America, 94043", + instance.asn, str(mock_response.traits.autonomous_system_number) ) - self.assertEqual(instance.coordinates.x, 150.0) - self.assertEqual(instance.coordinates.y, 50.0) + self.assertEqual(instance.timezone, mock_response.location.time_zone) + mock_address = { + "city": mock_response.city.name, + "country": mock_response.country.name, + "continent": mock_response.continent.name, + "postal": mock_response.postal.code, + } + self.assertEqual(instance.address, mock_address) + self.assertEqual(instance.cidr, "172.217.22.0/24") + self.assertEqual(instance.ip_address, ip_address) + formatted_address = WHOISInfo(address=mock_address).formatted_address + self.assertEqual(instance.formatted_address, formatted_address) + self.assertEqual(instance.coordinates.x, mock_response.location.longitude) + self.assertEqual(instance.coordinates.y, mock_response.location.latitude) # mocking the response from the geoip2 client mock_client.return_value.city.return_value = self._mocked_client_response() @@ -515,6 +647,7 @@ def _verify_whois_details(instance, ip_address): old_ip_address = device.last_ip device.last_ip = "172.217.22.10" device.save() + device.refresh_from_db() self.assertEqual(mock_info.call_count, 1) mock_info.reset_mock() device.refresh_from_db() @@ -522,7 +655,6 @@ def _verify_whois_details(instance, ip_address): _verify_whois_details( device.whois_service.get_device_whois_info(), device.last_ip ) - # details related to old ip address should be deleted self.assertEqual( WHOISInfo.objects.filter(ip_address=old_ip_address).count(), 0 @@ -540,14 +672,13 @@ def _verify_whois_details(instance, ip_address): ) device.last_ip = "172.217.22.11" device.save() + device.refresh_from_db() self.assertEqual(mock_info.call_count, 1) mock_info.reset_mock() device.refresh_from_db() - _verify_whois_details( device.whois_service.get_device_whois_info(), device.last_ip ) - # details related to old ip address should be not be deleted self.assertEqual( WHOISInfo.objects.filter(ip_address=old_ip_address).count(), 1 @@ -561,7 +692,6 @@ def _verify_whois_details(instance, ip_address): device.delete(check_deactivated=False) self.assertEqual(mock_info.call_count, 0) mock_info.reset_mock() - # WHOIS related to the device's last_ip should be deleted self.assertEqual(WHOISInfo.objects.filter(ip_address=ip_address).count(), 1) @@ -582,7 +712,6 @@ def _verify_whois_details(instance, ip_address): device1.delete(check_deactivated=False) self.assertEqual(mock_info.call_count, 0) mock_info.reset_mock() - # WHOIS related to the device's last_ip should be deleted self.assertEqual(WHOISInfo.objects.filter(ip_address=ip_address).count(), 0) @@ -594,7 +723,6 @@ def test_whois_update(self, mock_client): mock_client.return_value.city.return_value = mocked_response threshold = app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + 1 new_time = timezone.now() - timedelta(days=threshold) - whois_obj = self._create_whois_info() WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) @@ -606,7 +734,28 @@ def test_whois_update(self, mock_client): self.assertEqual(whois_obj.asn, str(11111)) with self.subTest( - "Test WHOIS update when older than X days for existing device" + "Test WHOIS update not running for invalid ip address " + "even if older than X days" + ): + Device.objects.all().delete() + WHOISInfo.objects.all().delete() + # to check ip address fallbacks are working if device created by + # bypassing validations + device = Device.objects.create( + name="default.test.device", + organization=self._get_org(), + mac_address=self.TEST_MAC_ADDRESS, + last_ip="InvalidIP", + ) + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj, None) + device.save() + device.refresh_from_db() + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj, None) + + with self.subTest( + "Test WHOIS update not running if whois disabled even if older than X days" ): Device.objects.all().delete() WHOISInfo.objects.all().delete() @@ -616,16 +765,22 @@ def test_whois_update(self, mock_client): WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) mocked_response.traits.autonomous_system_number = 22222 mock_client.return_value.city.return_value = mocked_response + org = self._get_org() + org.config_settings.whois_enabled = False + org.config_settings.save() device.save() + device.refresh_from_db() whois_obj = device.whois_service.get_device_whois_info() - self.assertEqual(whois_obj.asn, str(22222)) + self.assertEqual(whois_obj, None) with self.subTest( - "Test WHOIS update when older than X days for existing device " - "from DeviceChecksum View" + "Test WHOIS update when older than X days for existing device" ): Device.objects.all().delete() WHOISInfo.objects.all().delete() + org = self._get_org() + org.config_settings.whois_enabled = True + org.config_settings.save() device = self._create_device(last_ip="172.217.22.11") whois_obj = device.whois_service.get_device_whois_info() self.assertEqual(whois_obj.asn, str(22222)) @@ -633,9 +788,57 @@ def test_whois_update(self, mock_client): mocked_response.traits.autonomous_system_number = 33333 mock_client.return_value.city.return_value = mocked_response device.save() + device.refresh_from_db() whois_obj = device.whois_service.get_device_whois_info() self.assertEqual(whois_obj.asn, str(33333)) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_device_last_ip_deferred_checks(self): + whois_obj = self._create_whois_info() + self._create_device(last_ip=whois_obj.ip_address) + # Deferred fields remained deferred + device = Device.objects.only("id", "created").first() + with self.subTest("Test deferred fields remained deferred after last_ip check"): + device._check_last_ip() + self.assertTrue(device._is_deferred("last_ip")) + + with self.subTest( + "Test update_whois does not run if last_ip is deferred" + ), mock.patch( + "openwisp_controller.config.whois.service.WHOISService.update_whois_info" + ) as mock_update_whois: + threshold = app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + 1 + new_time = timezone.now() - timedelta(days=threshold) + WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) + device._check_last_ip() + mock_update_whois.assert_not_called() + mock_update_whois.reset_mock() + + def test_create_or_update_whois_updates_modified_unchanged_details(self): + """ + Test that _create_or_update_whois updates the modified field + even when the WHOIS details are unchanged. + """ + whois_obj = self._create_whois_info(ip_address="172.217.22.50") + device = self._create_device(last_ip=whois_obj.ip_address) + old_modified = whois_obj.modified + whois_details = { + "isp": whois_obj.isp, + "asn": whois_obj.asn, + "timezone": whois_obj.timezone, + "address": whois_obj.address, + "cidr": whois_obj.cidr, + "coordinates": whois_obj.coordinates, + } + updated_obj, _ = device.whois_service._create_or_update_whois( + whois_details, whois_obj + ) + updated_obj.refresh_from_db() + self.assertGreater( + updated_obj.modified, + old_modified, + ) + # we need to allow the task to propagate exceptions to ensure # `on_failure` method is called and notifications are executed @override_settings(CELERY_TASK_EAGER_PROPAGATES=False) @@ -645,38 +848,213 @@ def test_whois_update(self, mock_client): @mock.patch(_WHOIS_TASKS_INFO_LOGGER) def test_whois_task_failure_notification(self, mock_info, mock_warn, mock_error): def assert_logging_on_exception( - exception, info_calls=0, warn_calls=0, error_calls=1 + exception, info_calls=0, warn_calls=0, error_calls=1, notification_count=1 ): with self.subTest( - f"Test notifications and logging when {exception.__name__} is raised" - ), mock.patch(self._WHOIS_GEOIP_CLIENT, side_effect=exception("test")): + f"Test notification and logging when {exception.__name__} is raised" + ), mock.patch(self._WHOIS_GEOIP_CLIENT) as mock_client: + mock_client.return_value.city.side_effect = exception("test") Device.objects.all().delete() # Clear existing devices device = self._create_device(last_ip="172.217.22.14") self.assertEqual(mock_info.call_count, info_calls) self.assertEqual(mock_warn.call_count, warn_calls) self.assertEqual(mock_error.call_count, error_calls) - self.assertEqual(notification_qs.count(), 1) - notification = notification_qs.first() - self.assertEqual(notification.actor, device) - self.assertEqual(notification.target, device) - self.assertEqual(notification.level, "error") - self.assertEqual(notification.type, "generic_message") - self.assertIn( - "Failed to fetch WHOIS details for device", - notification.message, - ) - self.assertIn(device.last_ip, notification.rendered_description) - + if notification_count > 0: + notification_qs = _notification_qs() + self.assertEqual(notification_qs.count(), notification_count) + notification = notification_qs.first() + self.assertEqual(notification.actor, device) + self.assertEqual(notification.target, device) + self.assertEqual(notification.level, "error") + self.assertEqual(notification.type, "generic_message") + self.assertIn( + "Failed to fetch WHOIS details for device", + notification.message, + ) + self.assertIn(device.last_ip, notification.rendered_description) mock_info.reset_mock() mock_warn.reset_mock() mock_error.reset_mock() - notification_qs.delete() + _notification_qs().delete() # Test for all possible exceptions that can be raised by the geoip2 client + # Notification are sent only one time when any of the following exceptions + # are raised first time. assert_logging_on_exception(errors.OutOfQueriesError) - assert_logging_on_exception(errors.AddressNotFoundError) - assert_logging_on_exception(errors.AuthenticationError) - assert_logging_on_exception(errors.PermissionRequiredError) + assert_logging_on_exception(errors.AddressNotFoundError, notification_count=0) + assert_logging_on_exception(errors.AuthenticationError, notification_count=0) + assert_logging_on_exception( + errors.PermissionRequiredError, notification_count=0 + ) + assert_logging_on_exception(RequestException, notification_count=0) + cache.clear() + + @override_settings(CACHES=MODIFIED_CACHE) + @override_settings(CELERY_TASK_EAGER_PROPAGATES=False) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_whois_task_failure_cache(self): + permanent_errors = [ + errors.AddressNotFoundError, + errors.OutOfQueriesError, + errors.AuthenticationError, + errors.PermissionRequiredError, + ] + + def trigger_error_and_assert_cached(exc, notification_count=0): + with mock.patch(self._WHOIS_GEOIP_CLIENT) as mock_client: + mock_client.return_value.city.side_effect = exc("test") + Device.objects.all().delete() + device = self._create_device(last_ip="172.217.22.14") + cache_key = f"{self._WHOIS_TASK_NAME}_last_operation" + self.assertEqual(cache.get(cache_key), "errored") + self.assertEqual(_notification_qs().count(), notification_count) + _notification_qs().delete() + return device + + # simulate that no matter which permanent error is raised first, + # the rest of the permanent errors should use the cache + for first_error in permanent_errors: + with self.subTest(f"Cache populated by {first_error.__name__}"): + cache.clear() + trigger_error_and_assert_cached(first_error, 1) + for subsequent_error in permanent_errors: + if subsequent_error is first_error: + continue + with self.subTest( + f"Cache reused when {subsequent_error.__name__} occurs " + f"after {first_error.__name__}" + ): + trigger_error_and_assert_cached(subsequent_error, 0) + + with self.subTest("Test cache updated on success"), mock.patch( + self._WHOIS_GEOIP_CLIENT + ) as mock_client: + Device.objects.all().delete() + mocked_response = self._mocked_client_response() + mocked_response.location = None + mock_client.return_value.city.return_value = mocked_response + self._create_device(last_ip="172.217.22.14") + cache_key = f"{self._WHOIS_TASK_NAME}_last_operation" + self.assertEqual(cache.get(cache_key), "success") + self.assertEqual(_notification_qs().count(), 0) + cache.clear() + + @override_settings(CELERY_TASK_EAGER_PROPAGATES=True) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch("openwisp_controller.config.whois.tasks.fetch_whois_details.retry") + def test_whois_task_retry_mechanism(self, mock_retry): + def assert_retry_on_exception(exception, should_retry): + with self.subTest( + f"Test retry mechanism when {exception.__name__} is raised" + ), mock.patch(self._WHOIS_GEOIP_CLIENT) as mock_client: + mock_client.return_value.city.side_effect = exception("test") + Device.objects.all().delete() + mock_retry.reset_mock() + mock_retry.side_effect = exception("test") + with self.assertRaises(exception): + self._create_device(last_ip="172.217.22.14") + if should_retry: + self.assertEqual(mock_retry.call_count, 1) + assert isinstance(mock_retry.call_args.kwargs["exc"], exception) + else: + self.assertEqual(mock_retry.call_count, 0) + + assert_retry_on_exception(errors.HTTPError, should_retry=True) + assert_retry_on_exception(errors.OutOfQueriesError, should_retry=False) + assert_retry_on_exception(errors.AddressNotFoundError, should_retry=False) + assert_retry_on_exception(errors.AuthenticationError, should_retry=False) + assert_retry_on_exception(errors.PermissionRequiredError, should_retry=False) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_TASKS_WARN_LOGGER) + def test_fetch_whois_details_device_not_found(self, mock_warn): + invalid_pk = uuid4() + fetch_whois_details(device_pk=invalid_pk, initial_ip_address="10.0.0.1") + mock_warn.assert_called_once_with( + f"Device {invalid_pk} not found, skipping WHOIS lookup" + ) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_fetch_whois_details_record_already_exists(self, mock_client): + whois_obj = self._create_whois_info() + device = self._create_device(last_ip=whois_obj.ip_address) + mock_client.return_value.city.return_value = self._mocked_client_response() + fetch_whois_details(device_pk=device.pk, initial_ip_address="10.0.0.1") + mock_client.assert_not_called() + + def test_send_whois_task_notification_with_invalid_device_pk(self): + invalid_pk = uuid4() + result = send_whois_task_notification( + device=invalid_pk, notify_type="whois_device_error" + ) + self.assertIsNone(result) + + def test_delete_whois_record_force(self): + whois_obj = self._create_whois_info() + ip_address = whois_obj.ip_address + delete_whois_record(ip_address=ip_address, force=True) + self.assertFalse(WHOISInfo.objects.filter(ip_address=ip_address).exists()) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_get_whois_info_device_whois_disabled(self): + org = self._get_org() + org.config_settings.whois_enabled = False + org.config_settings.save() + device = self._create_device(last_ip="172.217.22.14") + result = get_whois_info(pk=device.pk) + self.assertIsNone(result) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_get_whois_info_with_none_pk(self): + result = get_whois_info(pk=None) + self.assertIsNone(result) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_get_whois_info_device_not_found(self): + result = get_whois_info(pk=uuid4()) + self.assertIsNone(result) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_get_whois_info_not_found(self, mock_client): + mock_client.return_value.city.return_value = self._mocked_client_response() + org = self._get_org() + org.config_settings.whois_enabled = True + org.config_settings.save() + device = self._create_device(last_ip="172.217.22.14") + WHOISInfo.objects.all().delete() + result = get_whois_info(pk=device.pk) + self.assertIsNone(result) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_get_whois_info_returns_data_with_formatted_address(self, mock_client): + mock_client.return_value.city.return_value = self._mocked_client_response() + org = self._get_org() + device = self._create_device(last_ip="172.217.22.14") + WHOISInfo.objects.filter(ip_address=device.last_ip).delete() + org.config_settings.whois_enabled = True + org.config_settings.save() + WHOISInfo.objects.create( + ip_address=device.last_ip, + address={ + "city": "Mountain View", + "country": "United States", + "postal": "94043", + }, + ) + result = get_whois_info(pk=device.pk) + self.assertIsNotNone(result) + self.assertEqual( + result["formatted_address"], "Mountain View, United States, 94043" + ) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", False) + def test_get_whois_info_when_not_configured(self): + device = self._create_device(last_ip="172.217.22.14") + result = get_whois_info(pk=device.pk) + self.assertIsNone(result) @tag("selenium_tests") @@ -707,7 +1085,6 @@ def _assert_no_js_errors(): if cells := row.find_elements(By.TAG_NAME, "td"): self.assertEqual(cells[0].text, whois_obj.isp) self.assertEqual(cells[1].text, whois_obj.address["country"]) - details = self.find_element(By.CSS_SELECTOR, "details.whois") self.web_driver.execute_script( "arguments[0].setAttribute('open','')", details @@ -735,6 +1112,7 @@ def _assert_no_js_errors(): org = self._get_org() org.config_settings.whois_enabled = False org.config_settings.save(update_fields=["whois_enabled"]) + org.config_settings.refresh_from_db(fields=["whois_enabled"]) self.open(reverse("admin:config_device_change", args=[device.pk])) self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") @@ -746,8 +1124,44 @@ def _assert_no_js_errors(): org = self._get_org() org.config_settings.whois_enabled = True org.config_settings.save(update_fields=["whois_enabled"]) + org.config_settings.refresh_from_db(fields=["whois_enabled"]) WHOISInfo.objects.all().delete() self.open(reverse("admin:config_device_change", args=[device.pk])) self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") _assert_no_js_errors() + + with self.subTest("Check XSS protection in WHOIS details admin view"): + whois_data = { + "ip_address": device.last_ip, + "isp": "", + "timezone": "", + "address": { + "city": "Mountain View", + "country": "", + "continent": "North America", + "postal": "94043", + }, + } + WHOISInfo.objects.all().delete() + self._create_whois_info(**whois_data) + try: + self.open(reverse("admin:config_device_change", args=[device.pk])) + table = self.find_element(By.CSS_SELECTOR, "table.whois-table") + rows = table.find_elements(By.TAG_NAME, "tr") + for row in rows: + if cells := row.find_elements(By.TAG_NAME, "td"): + self.assertIn("onerror", cells[0].text) + self.assertIn("script", cells[1].text) + details = self.find_element(By.CSS_SELECTOR, "details.whois") + self.web_driver.execute_script( + "arguments[0].setAttribute('open','')", details + ) + additional_text = details.find_elements( + By.CSS_SELECTOR, ".additional-text" + ) + self.assertIn("script", additional_text[1].text) + self.assertIn("script", additional_text[2].text) + _assert_no_js_errors() + except UnexpectedAlertPresentException: + self.fail("XSS vulnerability detected in WHOIS details admin view.") diff --git a/openwisp_controller/config/whois/tests/utils.py b/openwisp_controller/config/whois/tests/utils.py index 03822270a..ce36e5a4d 100644 --- a/openwisp_controller/config/whois/tests/utils.py +++ b/openwisp_controller/config/whois/tests/utils.py @@ -27,7 +27,6 @@ def _create_whois_info(self, **kwargs): cidr="172.217.22.0/24", coordinates=Point(150, 50, srid=4326), ) - options.update(kwargs) w = WHOISInfo(**options) w.full_clean() @@ -59,39 +58,27 @@ def _mocked_client_response(): def _task_called(self, mocked_task, task_name="WHOIS lookup"): org = self._get_org() - - with self.subTest(f"{task_name} task called when last_ip is public"): - with mock.patch( - "django.core.cache.cache.get", side_effect=[None, org.config_settings] - ) as mocked_get, mock.patch("django.core.cache.cache.set") as mocked_set: - device = self._create_device(last_ip="172.217.22.14") - mocked_task.assert_called() - mocked_set.assert_called_once() - mocked_get.assert_called() - mocked_task.reset_mock() - - with self.subTest( - f"{task_name} task called when last_ip is changed and is public" - ): - with mock.patch("django.core.cache.cache.get") as mocked_get, mock.patch( - "django.core.cache.cache.set" - ) as mocked_set: - device.last_ip = "172.217.22.10" - device.save() - mocked_task.assert_called() - mocked_set.assert_not_called() - mocked_get.assert_called() + device = self._create_device(last_ip="172.217.22.14") mocked_task.reset_mock() with self.subTest(f"{task_name} task not called when last_ip not updated"): device.name = "default.test.Device2" device.save() + device.refresh_from_db() mocked_task.assert_not_called() mocked_task.reset_mock() with self.subTest(f"{task_name} task not called when last_ip is private"): device.last_ip = "10.0.0.1" device.save() + device.refresh_from_db() + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest(f"{task_name} task not called when last_ip is invalid"): + device.last_ip = "invalid_ip" + device.save() + device.refresh_from_db() mocked_task.assert_not_called() mocked_task.reset_mock() @@ -100,6 +87,7 @@ def _task_called(self, mocked_task, task_name="WHOIS lookup"): org.config_settings.whois_enabled = False # Invalidates old org config settings cache org.config_settings.save(update_fields=["whois_enabled"]) + org.config_settings.refresh_from_db(fields=["whois_enabled"]) device = self._create_device(last_ip="172.217.22.14") mocked_task.assert_not_called() mocked_task.reset_mock() @@ -110,6 +98,7 @@ def _task_called(self, mocked_task, task_name="WHOIS lookup"): org.config_settings.whois_enabled = True # Invalidates old org config settings cache org.config_settings.save(update_fields=["whois_enabled"]) + org.config_settings.refresh_from_db(fields=["whois_enabled"]) # config is required for checksum view to work device.refresh_from_db() self._create_config(device=device) @@ -125,7 +114,8 @@ def _task_called(self, mocked_task, task_name="WHOIS lookup"): mocked_task.reset_mock() with self.subTest( - f"{task_name} task not called via DeviceChecksumView for no WHOIS record" + f"{task_name} task not called via DeviceChecksumView " + "if no WHOIS record and IP unchanged" ): WHOISInfo.objects.all().delete() device.refresh_from_db() @@ -145,6 +135,25 @@ def _task_called(self, mocked_task, task_name="WHOIS lookup"): device.refresh_from_db() org.config_settings.whois_enabled = False org.config_settings.save(update_fields=["whois_enabled"]) + org.config_settings.refresh_from_db(fields=["whois_enabled"]) + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest( + f"{task_name} task not called explicitly via DeviceChecksumView for " + "stale records" + ), mock.patch( + "openwisp_controller.config.whois.service.WHOISService.is_older", + return_value=True, + ): + WHOISInfo.objects.all().delete() + self._create_whois_info(ip_address=device.last_ip) response = self.client.get( reverse("controller:device_checksum", args=[device.pk]), {"key": device.key}, diff --git a/openwisp_controller/config/whois/utils.py b/openwisp_controller/config/whois/utils.py index c1b0430e9..1a51a6d44 100644 --- a/openwisp_controller/config/whois/utils.py +++ b/openwisp_controller/config/whois/utils.py @@ -1,4 +1,5 @@ from django.utils.translation import gettext_lazy as _ +from geoip2 import errors from openwisp_notifications.signals import notify from swapper import load_model @@ -14,37 +15,32 @@ ), "description": _("WHOIS details could not be fetched for ip: {ip_address}."), }, - "estimated_location_error": { - "level": "error", - "type": "estimated_location_info", - "message": _( - "Unable to create estimated location for device " - "[{notification.target}]({notification.target_link}). " - "Please assign/create a location manually." - ), - "description": _("Multiple devices found for IP: {ip_address}"), - }, - "estimated_location_created": { - "type": "estimated_location_info", - "description": _("Estimated Location {notification.verb} for IP: {ip_address}"), - }, - "estimated_location_updated": { - "type": "estimated_location_info", - "message": _( - "Estimated location [{notification.actor}]({notification.actor_link})" - " for device" - " [{notification.target}]({notification.target_link})" - " updated successfully." - ), - "description": _("Estimated Location updated for IP: {ip_address}"), - }, +} + +EXCEPTION_MESSAGES = { + errors.AddressNotFoundError: _( + "No WHOIS information found for IP address {ip_address}" + ), + errors.AuthenticationError: _( + "Authentication failed for GeoIP2 service. " + "Check your OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT and " + "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY settings." + ), + errors.OutOfQueriesError: _( + "Your account has run out of queries for the GeoIP2 service." + ), + errors.PermissionRequiredError: _( + "Your account does not have permission to access this service." + ), } def send_whois_task_notification(device, notify_type, actor=None): Device = load_model("config", "Device") if not isinstance(device, Device): - device = Device.objects.get(pk=device) + device = Device.objects.filter(pk=device).first() + if not device: + return notify_details = MESSAGE_MAP[notify_type] notify.send( sender=actor or device, diff --git a/openwisp_controller/geo/admin.py b/openwisp_controller/geo/admin.py index ffab0c99b..f08960b8c 100644 --- a/openwisp_controller/geo/admin.py +++ b/openwisp_controller/geo/admin.py @@ -102,26 +102,43 @@ class LocationAdmin(MultitenantAdminMixin, AbstractLocationAdmin): list_select_related = ("organization",) change_form_template = "admin/geo/location/change_form.html" + # Adding is_estimated field via 'get_' methods to allow testing in + # isolation as class level insertions are evaluated at import time + # making it unsuitable as per current testing setup. + def get_list_display(self, request): + list_display = list(super().get_list_display(request)) # makes a copy + if config_app_settings.WHOIS_CONFIGURED: + list_display.insert(list_display.index("created"), "is_estimated") + return list_display + + def get_list_filter(self, request): + list_filter = list(super().get_list_filter(request)) # makes a copy + if config_app_settings.WHOIS_CONFIGURED: + list_filter.append("is_estimated") + return list_filter + def get_fields(self, request, obj=None): - fields = super().get_fields(request, obj) - org_id = obj.organization_id if obj else None - if not WHOISService.check_estimate_location_configured(org_id): - if "is_estimated" in fields: - fields.remove("is_estimated") + fields = list(super().get_fields(request, obj)) # makes a copy + # do not show the is_estimated field on add pages + # or if the estimated location feature is not enabled for the organization + if not obj or not WHOISService.check_estimated_location_enabled( + obj.organization_id + ): + fields.remove("is_estimated") return fields def get_readonly_fields(self, request, obj=None): fields = super().get_readonly_fields(request, obj) - org_id = obj.organization_id if obj else None - if obj and WHOISService.check_estimate_location_configured(org_id): - fields = fields + ("is_estimated",) + if obj and WHOISService.check_estimated_location_enabled(obj.organization_id): + fields = ("is_estimated",) + fields # creates a new tuple return fields def change_view(self, request, object_id, form_url="", extra_context=None): obj = self.get_object(request, object_id) org_id = obj.organization_id if obj else None - estimated_configured = WHOISService.check_estimate_location_configured(org_id) - extra_context = {"estimated_configured": estimated_configured} + estimated_enabled = WHOISService.check_estimated_location_enabled(org_id) + extra_context = extra_context or {} + extra_context["estimated_enabled"] = estimated_enabled return super().change_view(request, object_id, form_url, extra_context) diff --git a/openwisp_controller/geo/api/serializers.py b/openwisp_controller/geo/api/serializers.py index 0626efdab..3d7cc61e3 100644 --- a/openwisp_controller/geo/api/serializers.py +++ b/openwisp_controller/geo/api/serializers.py @@ -12,7 +12,7 @@ from ...serializers import BaseSerializer from ..estimated_location.mixins import ( - EstimatedLocationGeoJsonSerializer, + EstimatedLocationGeoJsonMixin, EstimatedLocationMixin, ) @@ -36,7 +36,7 @@ class Meta: class GeoJsonLocationSerializer( - EstimatedLocationGeoJsonSerializer, gis_serializers.GeoFeatureModelSerializer + EstimatedLocationGeoJsonMixin, gis_serializers.GeoFeatureModelSerializer ): device_count = IntegerField() @@ -166,7 +166,8 @@ def validate(self, data): def to_representation(self, instance): request = self.context["request"] data = super().to_representation(instance) - floorplans = instance.floorplan_set.all().order_by("-modified") + # floorplan_set is already prefetched and ordered in the view + floorplans = instance.floorplan_set.all() floorplan_list = [] for floorplan in floorplans: dict_ = { @@ -232,7 +233,7 @@ def update(self, instance, validated_data): class NestedtLocationSerializer( - EstimatedLocationGeoJsonSerializer, gis_serializers.GeoFeatureModelSerializer + EstimatedLocationGeoJsonMixin, gis_serializers.GeoFeatureModelSerializer ): class Meta: model = Location diff --git a/openwisp_controller/geo/api/views.py b/openwisp_controller/geo/api/views.py index 9b64ca084..ae87c34df 100644 --- a/openwisp_controller/geo/api/views.py +++ b/openwisp_controller/geo/api/views.py @@ -1,5 +1,5 @@ from django.core.exceptions import ObjectDoesNotExist, ValidationError -from django.db.models import Count +from django.db.models import Count, Prefetch from django.http import Http404 from django.utils.translation import gettext_lazy as _ from django_filters import rest_framework as filters @@ -52,11 +52,15 @@ class Meta(OrganizationManagedFilter.Meta): model = Location fields = OrganizationManagedFilter.Meta.fields + ["is_mobile", "type"] - def __init__(self, data=None, queryset=None, *, request=None, prefix=None): - super().__init__(data, queryset, request=request, prefix=prefix) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # This is evaluated at runtime, which makes it suited + # For the automated testing strategy we are using. + # Defining this at class definition does not allow flexible testing. if config_app_settings.WHOIS_CONFIGURED: self.filters["is_estimated"] = filters.BooleanFilter( - field_name="is_estimated" + field_name="is_estimated", + label=_("Is geographic location estimated?"), ) @@ -223,8 +227,10 @@ class GeoJsonLocationList( Shows only locations which are assigned to devices. """ - queryset = Location.objects.filter(devicelocation__isnull=False).annotate( - device_count=Count("devicelocation") + queryset = ( + Location.objects.filter(devicelocation__isnull=False) + .annotate(device_count=Count("devicelocation")) + .order_by("-created") ) serializer_class = GeoJsonLocationSerializer pagination_class = GeoJsonLocationListPagination @@ -319,7 +325,12 @@ class FloorPlanDetailView( class LocationListCreateView(ProtectedAPIMixin, generics.ListCreateAPIView): serializer_class = LocationSerializer - queryset = Location.objects.order_by("-created") + queryset = Location.objects.prefetch_related( + Prefetch( + "floorplan_set", + queryset=FloorPlan.objects.order_by("-created"), + ) + ).order_by("-created") pagination_class = ListViewPagination filter_backends = [filters.DjangoFilterBackend] filterset_class = LocationOrganizationFilter diff --git a/openwisp_controller/geo/base/models.py b/openwisp_controller/geo/base/models.py index c1225f838..c26aca6cb 100644 --- a/openwisp_controller/geo/base/models.py +++ b/openwisp_controller/geo/base/models.py @@ -1,8 +1,7 @@ -import re +from typing import ClassVar from django.contrib.gis.db import models from django.core.exceptions import ValidationError -from django.utils.translation import gettext from django.utils.translation import gettext_lazy as _ from django_loci.base.models import ( AbstractFloorPlan, @@ -16,9 +15,14 @@ class BaseLocation(OrgMixin, AbstractLocation): - _changed_checked_fields = ["is_estimated", "address", "geometry"] + _changed_checked_fields: ClassVar[list[str]] = [ + "is_estimated", + "address", + "geometry", + ] is_estimated = models.BooleanField( + _("Is Estimated?"), default=False, help_text=_("Whether the location's coordinates are estimated."), ) @@ -41,12 +45,14 @@ def _set_initial_values_for_changed_checked_fields(self): def clean(self): # Raise validation error if `is_estimated` is True but estimated feature is # disabled. + estimated_status_changed = ( + self._initial_is_estimated is not models.DEFERRED + and self._initial_is_estimated != self.is_estimated + ) if ( - (self._state.adding or self._initial_is_estimated != self.is_estimated) + (self._state.adding or estimated_status_changed) and self.is_estimated - and not WHOISService.check_estimate_location_configured( - self.organization_id - ) + and not WHOISService.check_estimated_location_enabled(self.organization_id) ): raise ValidationError( { @@ -64,27 +70,35 @@ def save(self, *args, _set_estimated=False, **kwargs): Parameters: _set_estimated: Boolean flag to indicate if this save is being performed by the estimated location system. When False (default), - manual edits will clear the estimated status. + manual edits will clear the estimated status (only if estimated location + feature is enabled). *args, **kwargs: Arguments passed to the parent save method. Returns: The result of the parent save method. """ - if WHOISService.check_estimate_location_configured(self.organization_id): - if not _set_estimated and ( - self._initial_address != self.address - or self._initial_geometry != self.geometry - ): + changed_fields = set() + if WHOISService.check_estimated_location_enabled(self.organization_id): + address_changed = ( + self._initial_address is not models.DEFERRED + and self._initial_address != self.address + ) + geometry_changed = ( + self._initial_geometry is not models.DEFERRED + and self._initial_geometry != self.geometry + ) + if not _set_estimated and (address_changed or geometry_changed): self.is_estimated = False - estimated_string = gettext("Estimated Location") - if self.name and estimated_string in self.name: - # remove string starting with "(Estimated Location" - self.name = re.sub( - rf"\s\({estimated_string}.*", "", self.name, flags=re.IGNORECASE - ) - else: + changed_fields = {"is_estimated"} + # Manual changes to is_estimated discarded if feature not enabled + elif self._initial_is_estimated is not models.DEFERRED: self.is_estimated = self._initial_is_estimated - return super().save(*args, **kwargs) + changed_fields = {"is_estimated"} + if update_fields := kwargs.get("update_fields"): + kwargs["update_fields"] = set(update_fields) | changed_fields + result = super().save(*args, **kwargs) + self._set_initial_values_for_changed_checked_fields() + return result class BaseFloorPlan(OrgMixin, AbstractFloorPlan): diff --git a/openwisp_controller/geo/estimated_location/handlers.py b/openwisp_controller/geo/estimated_location/handlers.py index 60ea6f44a..2b66a6d94 100644 --- a/openwisp_controller/geo/estimated_location/handlers.py +++ b/openwisp_controller/geo/estimated_location/handlers.py @@ -4,6 +4,8 @@ from openwisp_controller.config import settings as config_app_settings +from .utils import MESSAGE_MAP + def register_estimated_location_notification_types(): """ @@ -19,18 +21,10 @@ def register_estimated_location_notification_types(): register_notification_type( "estimated_location_info", { + **MESSAGE_MAP["estimated_location_created"], "verbose_name": _("Estimated Location INFO"), "verb": _("created"), - "level": "info", - "email_subject": _( - "Estimated Location: Created for device {notification.target}" - ), - "message": _( - "Estimated location [{notification.actor}]({notification.actor_link})" - " for device" - " [{notification.target}]({notification.target_link})" - " {notification.verb} successfully." - ), + "email_subject": _("Estimated location created for {notification.target}"), "target_link": ( "openwisp_controller.geo.estimated_location.utils" ".get_device_location_notification_target_url" diff --git a/openwisp_controller/geo/estimated_location/mixins.py b/openwisp_controller/geo/estimated_location/mixins.py index f4d2fc93b..222eaa8b7 100644 --- a/openwisp_controller/geo/estimated_location/mixins.py +++ b/openwisp_controller/geo/estimated_location/mixins.py @@ -1,6 +1,8 @@ from openwisp_controller.config.whois.service import WHOISService +# These mixins are required as estimated location is an organization level feature. +# Also, adding it this way makes it read-only as well. class EstimatedLocationMixin: """ Serializer mixin to add estimated location field to the serialized data @@ -9,21 +11,22 @@ class EstimatedLocationMixin: def to_representation(self, obj): data = super().to_representation(obj) - if WHOISService.check_estimate_location_configured(obj.organization_id): + if WHOISService.check_estimated_location_enabled(obj.organization_id): data["is_estimated"] = obj.is_estimated else: data.pop("is_estimated", None) return data -class EstimatedLocationGeoJsonSerializer(EstimatedLocationMixin): +class EstimatedLocationGeoJsonMixin: """ - Extension of EstimatedLocationMixin for GeoJSON serialization. + Serializer mixin to add estimated location field to the serialized GeoJSON data + if the estimated location feature is configured and enabled for the organization. """ def to_representation(self, obj): - data = super(EstimatedLocationMixin, self).to_representation(obj) - if WHOISService.check_estimate_location_configured(obj.organization_id): + data = super().to_representation(obj) + if WHOISService.check_estimated_location_enabled(obj.organization_id): data["properties"]["is_estimated"] = obj.is_estimated else: data["properties"].pop("is_estimated", None) diff --git a/openwisp_controller/geo/estimated_location/tasks.py b/openwisp_controller/geo/estimated_location/tasks.py index c900af12c..58eb863d1 100644 --- a/openwisp_controller/geo/estimated_location/tasks.py +++ b/openwisp_controller/geo/estimated_location/tasks.py @@ -28,7 +28,10 @@ def _handle_attach_existing_location( .exclude(pk=device.pk) .exists() ) - if existing_device_location: + if ( + existing_device_location + and existing_device_location.location != device_location.location + ): existing_location = existing_device_location.location device_location.location = existing_location device_location.full_clean() @@ -61,6 +64,18 @@ def _handle_attach_existing_location( **whois_obj._get_defaults_for_estimated_location(), "organization_id": device.organization_id, } + # Create new location only if location is changed. + if ( + attached_devices_exists + and current_location + and current_location.geometry == location_defaults.get("geometry") + and current_location.name == location_defaults.get("name") + ): + logger.debug( + f"Estimated location unchanged for {device.pk}" + f" for IP: {ip_address}, keeping existing location" + ) + return # create new location if no location exists for device or the estimated location # of device is shared. whois_service = device.whois_service @@ -73,7 +88,7 @@ def _handle_attach_existing_location( ) -@shared_task +@shared_task(name="whois_estimated_location_task") def manage_estimated_locations(device_pk, ip_address): """ Creates/updates estimated location for a device based on the latitude and @@ -82,7 +97,7 @@ def manage_estimated_locations(device_pk, ip_address): the given ip_address. Does not alters the existing location if it is not estimated. - - If the current device has no location or location is estimate, either update + - If the current device has no location or location is estimated, either update to an existing location; if it exists, else - A new location is created if current device has no location, or @@ -94,18 +109,34 @@ def manage_estimated_locations(device_pk, ip_address): Device = load_model("config", "Device") DeviceLocation = load_model("geo", "DeviceLocation") - device = Device.objects.select_related("devicelocation__location").get(pk=device_pk) - - devices_with_location = ( - Device.objects.only("devicelocation") + try: + device = Device.objects.select_related("devicelocation__location").get( + pk=device_pk + ) + except Device.DoesNotExist: + logger.warning( + f"Device {device_pk} not found, skipping manage_estimated_locations" + ) + return + devices_with_location = list( + # "devicelocation" and "devicelocation__location" must be in only() to + # prevent Django from deferring them, which would conflict with + # select_related(). Django raises FieldError if a relation field is + # both deferred and traversed via select_related. + Device.objects.only( + "id", "name", "last_ip", "devicelocation", "devicelocation__location" + ) .select_related("devicelocation__location") - .filter(organization_id=device.organization_id) - .filter(last_ip=ip_address, devicelocation__location__isnull=False) - .exclude(pk=device.pk) + .filter( + organization_id=device.organization_id, + last_ip=ip_address, + devicelocation__location__isnull=False, + ) + # evaluated to LIMIT query, we need to know if there's more than 1 result + .exclude(pk=device.pk)[:2] ) - # multiple devices can have same last_ip in cases like usage of proxy - if devices_with_location.count() > 1: + if len(devices_with_location) > 1: send_whois_task_notification( device=device, notify_type="estimated_location_error" ) @@ -114,16 +145,19 @@ def manage_estimated_locations(device_pk, ip_address): f"last_ip {ip_address}. Please resolve the conflict manually." ) return - + # if device doesn't have a location yet, initialize a draft if not (device_location := getattr(device, "devicelocation", None)): device_location = DeviceLocation(content_object=device) - current_location = device_location.location - if not current_location or current_location.is_estimated: - existing_device_location = getattr( - devices_with_location.first(), "devicelocation", None - ) + # existing device location + try: + existing_device_location = getattr( + devices_with_location[0], "devicelocation", None + ) + # no existing device location + except IndexError: + existing_device_location = None _handle_attach_existing_location( device, device_location, ip_address, existing_device_location ) diff --git a/openwisp_controller/geo/estimated_location/tests/tests.py b/openwisp_controller/geo/estimated_location/tests/tests.py index 8760a7796..70794f44c 100644 --- a/openwisp_controller/geo/estimated_location/tests/tests.py +++ b/openwisp_controller/geo/estimated_location/tests/tests.py @@ -2,6 +2,7 @@ import importlib from datetime import timedelta from unittest import mock +from uuid import uuid4 from django.contrib.gis.geos import GEOSGeometry from django.core.exceptions import ImproperlyConfigured, ValidationError @@ -14,6 +15,7 @@ from openwisp_controller.config import settings as config_app_settings from openwisp_controller.config.whois.handlers import connect_whois_handlers from openwisp_controller.config.whois.tests.utils import WHOISTransactionMixin +from openwisp_controller.geo import estimated_location from ....tests.utils import TestAdminMixin from ...tests.utils import TestGeoMixin @@ -21,6 +23,7 @@ from ..tasks import manage_estimated_locations from .utils import TestEstimatedLocationMixin +Config = load_model("config", "Config") Device = load_model("config", "Device") Location = load_model("geo", "Location") DeviceLocation = load_model("geo", "DeviceLocation") @@ -28,7 +31,9 @@ Notification = load_model("openwisp_notifications", "Notification") OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") -notification_qs = Notification.objects.all() + +def _notification_qs(): + return Notification.objects.all() class TestEstimatedLocation(TestAdminMixin, TestCase): @@ -37,7 +42,8 @@ class TestEstimatedLocation(TestAdminMixin, TestCase): OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY="test_key", ) def test_estimated_location_configuration_setting(self): - # reload app_settings to apply the overridden settings + # reload app_settings to apply the overridden + self.addCleanup(importlib.reload, config_app_settings) importlib.reload(config_app_settings) with self.subTest( "ImproperlyConfigured raised when ESTIMATED_LOCATION_ENABLED is True " @@ -59,16 +65,11 @@ def test_estimated_location_configuration_setting(self): org_settings_obj.whois_enabled = False org_settings_obj.estimated_location_enabled = True org_settings_obj.full_clean() - try: - self.assertEqual( - context_manager.exception.message_dict[ - "estimated_location_enabled" - ][0], - "Estimated Location feature requires " - + "WHOIS Lookup feature to be enabled.", - ) - except AssertionError: - self.fail("ValidationError message not equal to expected message.") + self.assertEqual( + context_manager.exception.message_dict["estimated_location_enabled"][0], + "Estimated Location feature requires " + + "WHOIS Lookup feature to be enabled.", + ) with self.subTest( "Test Estimated Location field visible on admin when " @@ -116,13 +117,10 @@ def test_estimated_location_field(self): org.refresh_from_db() with self.assertRaises(ValidationError) as context_manager: self._create_location(organization=org, is_estimated=True) - try: - self.assertEqual( - context_manager.exception.message_dict["is_estimated"][0], - "Estimated Location feature required to be configured.", - ) - except AssertionError: - self.fail("ValidationError message not equal to expected message.") + self.assertEqual( + context_manager.exception.message_dict["is_estimated"][0], + "Estimated Location feature required to be configured.", + ) @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) def test_estimated_location_admin(self): @@ -131,33 +129,70 @@ def test_estimated_location_admin(self): self.client.force_login(admin) org = self._get_org() location = self._create_location(organization=org, is_estimated=True) - path = reverse("admin:geo_location_change", args=[location.pk]) - response = self.client.get(path) - self.assertContains(response, "field-is_estimated") - self.assertContains( - response, "Whether the location's coordinates are estimated." - ) + change_path = reverse("admin:geo_location_change", args=[location.pk]) + add_path = reverse("admin:geo_location_add") + + with self.subTest( + "is-estimated field visible when estimated location is enabled" + ): + response = self.client.get(change_path) + self.assertContains(response, "field-is_estimated") + self.assertContains( + response, "Whether the location's coordinates are estimated." + ) + with self.subTest( + "is-estimated field not visible in add pages because auto-managed" + ): + response = self.client.get(add_path) + self.assertNotContains(response, "field-is_estimated") + org.config_settings.estimated_location_enabled = False org.config_settings.save() + org.config_settings.refresh_from_db() + + with self.subTest( + "is-estimated field hidden when estimated location is disabled" + ): + response = self.client.get(change_path) + self.assertNotContains(response, "field-is_estimated") + self.assertNotContains( + response, "Whether the location's coordinates are estimated." + ) + with self.subTest( + "double-check is-estimated field is not " + "leaking if estimated location is disabled" + ): + response = self.client.get(add_path) + self.assertNotContains(response, "field-is_estimated") + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False) + def test_estimated_location_admin_add_whois_disabled(self): + admin = self._create_admin() + self.client.force_login(admin) + path = reverse("admin:geo_location_add") response = self.client.get(path) self.assertNotContains(response, "field-is_estimated") - self.assertNotContains( - response, "Whether the location's coordinates are estimated." - ) class TestEstimatedLocationTransaction( - TestEstimatedLocationMixin, WHOISTransactionMixin, TransactionTestCase + TestEstimatedLocationMixin, WHOISTransactionMixin, TestGeoMixin, TransactionTestCase ): + location_model = Location + object_location_model = DeviceLocation + _WHOIS_GEOIP_CLIENT = ( "openwisp_controller.config.whois.service.geoip2_webservice.Client" ) _ESTIMATED_LOCATION_INFO_LOGGER = ( "openwisp_controller.geo.estimated_location.tasks.logger.info" ) + _ESTIMATED_LOCATION_WARNING_LOGGER = ( + "openwisp_controller.geo.estimated_location.tasks.logger.warning" + ) _ESTIMATED_LOCATION_ERROR_LOGGER = ( "openwisp_controller.geo.estimated_location.tasks.logger.error" ) + _WHOIS_TASK_NAME = "openwisp_controller.config.whois.tasks.fetch_whois_details" def setUp(self): super().setUp() @@ -170,7 +205,7 @@ def setUp(self): @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) @mock.patch( - "openwisp_controller.geo.estimated_location.tasks.manage_estimated_locations.delay" # noqa + "openwisp_controller.config.whois.service.WHOISService.trigger_estimated_location_task" # noqa: E501 ) @mock.patch(_WHOIS_GEOIP_CLIENT) def test_estimated_location_task_called( @@ -186,8 +221,50 @@ def test_estimated_location_task_called( ) Device.objects.all().delete() - device = self._create_device() - self._create_config(device=device) + WHOISInfo.objects.all().delete() + org = self._get_org() + org.config_settings.whois_enabled = True + org.config_settings.estimated_location_enabled = True + org.config_settings.save() + + with self.subTest("Estimated location task called when last_ip is public"): + with mock.patch( + "django.core.cache.cache.get", return_value=None + ) as mocked_get, mock.patch("django.core.cache.cache.set") as mocked_set: + device = self._create_device(last_ip="172.217.22.14") + mocked_estimated_location_task.assert_called() + expected_cache_set_calls = [ + mock.call( + f"organization_config_{org.pk}", + org.config_settings, + timeout=Config._CHECKSUM_CACHE_TIMEOUT, + ), + mock.call( + f"{self._WHOIS_TASK_NAME}_last_operation", "success", None + ), + ] + mocked_set.assert_has_calls(expected_cache_set_calls) + mocked_get.assert_called() + mocked_estimated_location_task.reset_mock() + + with self.subTest( + "Estimated location task called when last_ip is changed and is public" + ): + with mock.patch("django.core.cache.cache.get") as mocked_get, mock.patch( + "django.core.cache.cache.set" + ) as mocked_set: + device.last_ip = "172.217.22.10" + device.save() + device.refresh_from_db() + mocked_estimated_location_task.assert_called() + expected_cache_set_calls = [ + mock.call( + f"{self._WHOIS_TASK_NAME}_last_operation", "success", None + ), + ] + mocked_set.assert_has_calls(expected_cache_set_calls) + mocked_get.assert_called() + mocked_estimated_location_task.reset_mock() with self.subTest( "Estimated location task called when last_ip has related WhoIsInfo" @@ -195,12 +272,12 @@ def test_estimated_location_task_called( with mock.patch("django.core.cache.cache.get") as mocked_get, mock.patch( "django.core.cache.cache.set" ) as mocked_set: - device.organization.config_settings.whois_enabled = True - device.organization.config_settings.estimated_location_enabled = True - device.organization.config_settings.save() + self._create_config(device=device) device.last_ip = "172.217.22.14" self._create_whois_info(ip_address=device.last_ip) device.save() + device.refresh_from_db() + device.organization.config_settings.refresh_from_db() mocked_set.assert_not_called() # The cache `get` is called twice, once for `whois_enabled` and # once for `estimated_location_enabled` @@ -210,7 +287,7 @@ def test_estimated_location_task_called( with self.subTest( "Estimated location task not called via DeviceChecksumView when " - "last_ip has no related WhoIsInfo" + "last_ip already has related WhoIsInfo" ): WHOISInfo.objects.all().delete() self._create_whois_info(ip_address=device.last_ip) @@ -231,7 +308,9 @@ def test_estimated_location_task_called( WHOISInfo.objects.filter(pk=whois_obj.pk).update( modified=timezone.now() - timedelta(days=threshold) ) + self._create_object_location(content_object=device) device.save() + device.refresh_from_db() mocked_estimated_location_task.assert_not_called() mocked_estimated_location_task.reset_mock() response = self.client.get( @@ -254,43 +333,17 @@ def test_estimated_location_task_called( mocked_response.city.name = "New city" mocked_client.return_value.city.return_value = mocked_response device.save() - mocked_estimated_location_task.assert_called() - mocked_estimated_location_task.reset_mock() - mocked_response.city.name = "New city 2" - mocked_client.return_value.city.return_value = mocked_response - response = self.client.get( - reverse("controller:device_checksum", args=[device.pk]), - {"key": device.key}, - REMOTE_ADDR=device.last_ip, - ) - self.assertEqual(response.status_code, 200) - mocked_estimated_location_task.assert_called() - - mocked_response.location.latitude = 60 - mocked_client.return_value.city.return_value = mocked_response - device.save() - mocked_estimated_location_task.assert_called() - mocked_estimated_location_task.reset_mock() - mocked_response.location.longitude = 160 - mocked_client.return_value.city.return_value = mocked_response - response = self.client.get( - reverse("controller:device_checksum", args=[device.pk]), - {"key": device.key}, - REMOTE_ADDR=device.last_ip, - ) - self.assertEqual(response.status_code, 200) + device.refresh_from_db() mocked_estimated_location_task.assert_called() mocked_estimated_location_task.reset_mock() @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch("openwisp_controller.config.whois.service.send_whois_task_notification") @mock.patch( - "openwisp_controller.config.whois.service.send_whois_task_notification" # noqa - ) - @mock.patch( - "openwisp_controller.geo.estimated_location.tasks.send_whois_task_notification" # noqa + "openwisp_controller.geo.estimated_location.tasks.send_whois_task_notification" ) @mock.patch( - "openwisp_controller.geo.estimated_location.tasks.manage_estimated_locations.delay" # noqa + "openwisp_controller.config.whois.service.WHOISService.trigger_estimated_location_task" # noqa: E501 ) @mock.patch(_ESTIMATED_LOCATION_INFO_LOGGER) @mock.patch(_WHOIS_GEOIP_CLIENT) @@ -302,19 +355,17 @@ def test_estimated_location_creation_and_update( def _verify_location_details(device, mocked_response): location = device.devicelocation.location mocked_location = mocked_response.location - address = ", ".join( - [ - mocked_response.city.name, - mocked_response.country.name, - mocked_response.continent.name, - mocked_response.postal.code, - ] - ) + mock_address = { + "city": mocked_response.city.name, + "country": mocked_response.country.name, + "continent": mocked_response.continent.name, + "postal": mocked_response.postal.code, + } + address = WHOISInfo(address=mock_address).formatted_address ip_address = mocked_response.ip_address or device.last_ip - location_name = ( - ",".join(address.split(",")[:2]) - + f" (Estimated Location: {ip_address})" - ) + location_name = WHOISInfo( + address=mock_address, ip_address=ip_address + )._location_name self.assertEqual(location.name, location_name) self.assertEqual(location.address, address) self.assertEqual( @@ -330,9 +381,8 @@ def _verify_location_details(device, mocked_response): with self.subTest("Test Estimated location created when device is created"): device = self._create_device(last_ip="172.217.22.14") - with self.assertNumQueries(14): + with self.assertNumQueries(13): manage_estimated_locations(device.pk, device.last_ip) - location = device.devicelocation.location mocked_response.ip_address = device.last_ip self.assertEqual(location.is_estimated, True) @@ -352,9 +402,9 @@ def _verify_location_details(device, mocked_response): mocked_response.city.name = "New City" mock_client.return_value.city.return_value = mocked_response device.save() - with self.assertNumQueries(8): - manage_estimated_locations(device.pk, device.last_ip) device.refresh_from_db() + with self.assertNumQueries(7): + manage_estimated_locations(device.pk, device.last_ip) location = device.devicelocation.location mocked_response.ip_address = device.last_ip @@ -368,18 +418,41 @@ def _verify_location_details(device, mocked_response): ) mock_info.reset_mock() + with self.subTest("Test Estimated location Name when address not available"): + device.last_ip = "172.217.22.11" + mocked_response.city.name = "" + mocked_response.country.name = "" + mocked_response.continent.name = "" + mocked_response.postal.code = "" + mock_client.return_value.city.return_value = mocked_response + device.save() + device.refresh_from_db() + with self.assertNumQueries(7): + manage_estimated_locations(device.pk, device.last_ip) + + location = device.devicelocation.location + mocked_response.ip_address = device.last_ip + self.assertEqual(location.is_estimated, True) + self.assertEqual(location.is_mobile, False) + _verify_location_details(device, mocked_response) + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device.pk}" + f" for IP: {device.last_ip}" + ) + mock_info.reset_mock() + with self.subTest( "Test Non Estimated Location not updated when last ip is updated" ): mocked_response.ip_address = device.last_ip - device.last_ip = "172.217.22.11" + device.last_ip = "172.217.22.12" device.devicelocation.location.is_estimated = False mock_client.return_value.city.return_value = self._mocked_client_response() device.devicelocation.location.save(_set_estimated=True) device.save() + device.refresh_from_db() with self.assertNumQueries(2): manage_estimated_locations(device.pk, device.last_ip) - device.refresh_from_db() location = device.devicelocation.location self.assertEqual(location.is_estimated, False) @@ -404,7 +477,7 @@ def _verify_location_details(device, mocked_response): mac_address="11:22:33:44:55:66", last_ip="172.217.22.10", ) - with self.assertNumQueries(8): + with self.assertNumQueries(7): manage_estimated_locations(device2.pk, device2.last_ip) self.assertEqual( @@ -433,13 +506,13 @@ def _verify_location_details(device, mocked_response): device2.last_ip = "172.217.22.10" device2.save() # 3 queries related to notifications cleanup - with self.assertNumQueries(16): + device2.refresh_from_db() + with self.assertNumQueries(15): manage_estimated_locations(device2.pk, device2.last_ip) mock_info.assert_called_once_with( f"Estimated location saved successfully for {device2.pk}" f" for IP: {device2.last_ip}" ) - device2.refresh_from_db() self.assertEqual( device1.devicelocation.location.pk, device2.devicelocation.location.pk @@ -466,13 +539,13 @@ def _verify_location_details(device, mocked_response): old_location.save() device2.last_ip = "172.217.22.10" device2.save() + device2.refresh_from_db() with self.assertNumQueries(2): manage_estimated_locations(device2.pk, device2.last_ip) mock_info.assert_called_once_with( f"Non Estimated location already set for {device2.pk}. Update" f" location manually as per IP: {device2.last_ip}" ) - device2.refresh_from_db() self.assertNotEqual( device1.devicelocation.location.pk, device2.devicelocation.location.pk @@ -499,24 +572,118 @@ def _verify_location_details(device, mocked_response): ) device2.last_ip = "172.217.22.11" device2.save() - with self.assertNumQueries(14): + device2.refresh_from_db() + with self.assertNumQueries(13): manage_estimated_locations(device2.pk, device2.last_ip) mock_info.assert_called_once_with( f"Estimated location saved successfully for {device2.pk}" f" for IP: {device2.last_ip}" ) - device2.refresh_from_db() self.assertNotEqual( device1.devicelocation.location.pk, device2.devicelocation.location.pk ) mock_info.reset_mock() @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_ESTIMATED_LOCATION_WARNING_LOGGER) + def test_manage_estimated_location_device_not_found(self, mock_warn): + invalid_pk = uuid4() + manage_estimated_locations(device_pk=invalid_pk, ip_address="10.0.0.1") + mock_warn.assert_called_once_with( + f"Device {invalid_pk} not found, skipping manage_estimated_locations" + ) + + @mock.patch( + "openwisp_controller.config.whois.service.current_app.send_task", + side_effect=TestEstimatedLocationMixin.run_task, + ) + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_estimated_location_handling_on_whois_update( + self, mock_client, mock_send_task + ): + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + threshold = config_app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + 1 + new_time = timezone.now() - timedelta(days=threshold) + org = self._get_org() + org.config_settings.estimated_location_enabled = False + org.config_settings.save() + device = self._create_device(last_ip="172.217.22.10") + with self.assertRaises(Device.devicelocation.RelatedObjectDoesNotExist): + # Accessing devicelocation to verify it doesn't exist (raises if not) + device.devicelocation + org.config_settings.estimated_location_enabled = True + org.config_settings.save() + whois_obj = device.whois_service.get_device_whois_info() + WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) + device.name = "test.new.name" + device.save() + device.refresh_from_db() + # location created so can safely access devicelocation + # Accessing devicelocation to verify it exists (raises if not) + device.devicelocation + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch( + "openwisp_controller.config.whois.service.current_app.send_task", + side_effect=TestEstimatedLocationMixin.run_task, + ) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_unchanged_whois_data_no_location_recreation(self, mock_client, _): + """Ensure identical WHOIS results do not recreate a shared Location when + devices reuse the same IP.""" + connect_whois_handlers() + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + shared_ip = "20.49.19.19" + device1 = self._create_device( + name="device-a", + mac_address="00:11:22:33:44:55", + last_ip=shared_ip, + ) + device2 = self._create_device( + name="device-b", + mac_address="00:11:22:33:44:66", + last_ip=shared_ip, + ) + original_location = device1.devicelocation.location + self.assertEqual(original_location.pk, device2.devicelocation.location.pk) + location_count = Location.objects.count() + notification_count = _notification_qs().count() + # Clear the last ip for both devices, so setting them again + # will trigger the WHOIS lookup flow. + for device in (device1, device2): + device.last_ip = "" + device.save(update_fields=["last_ip"]) + device.refresh_from_db() + # We set the same shared IP again. This simulates device fetching checksum. + for device in (device1, device2): + device.last_ip = shared_ip + device.save(update_fields=["last_ip"]) + device.refresh_from_db() + # The location object should remain unchanged since the WHOIS data is the same. + self.assertEqual(original_location.pk, device1.devicelocation.location.pk) + self.assertEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + self.assertEqual(Location.objects.count(), location_count) + self.assertTrue(Location.objects.filter(pk=original_location.pk).exists()) + self.assertEqual(_notification_qs().count(), notification_count) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch( + "openwisp_controller.config.whois.service.current_app.send_task", + side_effect=TestEstimatedLocationMixin.run_task, + ) @mock.patch(_ESTIMATED_LOCATION_INFO_LOGGER) @mock.patch(_ESTIMATED_LOCATION_ERROR_LOGGER) @mock.patch(_WHOIS_GEOIP_CLIENT) - def test_estimated_location_notification(self, mock_client, mock_error, mock_info): + def test_estimated_location_notification( + self, mock_client, mock_error, mock_info, _ + ): def _verify_notification(device, messages, notify_level="info"): + notification_qs = _notification_qs() self.assertEqual(notification_qs.count(), 1) notification = notification_qs.first() device_location = getattr(device, "devicelocation", None) @@ -540,7 +707,7 @@ def _verify_notification(device, messages, notify_level="info"): _verify_notification(device1, messages) with self.subTest("Test Notification for location update"): - notification_qs.delete() + _notification_qs().delete() # will have same location as first device device2 = self._create_device( name="11:22:33:44:55:66", @@ -551,9 +718,7 @@ def _verify_notification(device, messages, notify_level="info"): _verify_notification(device2, messages) with self.subTest("Test Error Notification for conflicting locations"): - device2.last_ip = device1.last_ip - device2.save() - notification_qs.delete() + _notification_qs().delete() mock_info.reset_mock() mock_error.reset_mock() device3 = self._create_device( @@ -566,12 +731,50 @@ def _verify_notification(device, messages, notify_level="info"): f"Multiple devices with locations found with same " f"last_ip {device3.last_ip}. Please resolve the conflict manually." ) - messages = ["Unable to create estimated location for device"] + messages = [ + "Unable to create estimated location for device", + "Please assign/create a location manually.", + ] _verify_notification(device3, messages, "error") @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch("openwisp_controller.config.whois.service.send_whois_task_notification") + @mock.patch( + "openwisp_controller.geo.estimated_location.tasks.send_whois_task_notification" + ) + @mock.patch( + "openwisp_controller.config.whois.service.WHOISService.trigger_estimated_location_task" # noqa: E501 + ) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_manage_estimated_locations_no_coordinates_warning( + self, mock_client, _mocked_task, _mocked_notify, _mocked_notify2 + ): + with mock.patch.object(estimated_location.tasks.logger, "warning") as mock_warn: + connect_whois_handlers() + mock_client.return_value.city.return_value = self._mocked_client_response() + device = self._create_device(last_ip="172.217.22.14") + WHOISInfo.objects.filter(ip_address=device.last_ip).delete() + WHOISInfo.objects.create( + ip_address=device.last_ip, + address={ + "city": "Mountain View", + "country": "United States", + "postal": "94043", + }, + ) + manage_estimated_locations(device.pk, device.last_ip) + mock_warn.assert_called_once_with( + f"Coordinates not available for {device.pk} for IP: {device.last_ip}. " + "Estimated location cannot be determined." + ) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch( + "openwisp_controller.config.whois.service.current_app.send_task", + side_effect=TestEstimatedLocationMixin.run_task, + ) @mock.patch(_WHOIS_GEOIP_CLIENT) - def test_estimate_location_status_remove(self, mock_client): + def test_estimate_location_status_remove(self, mock_client, _): mocked_response = self._mocked_client_response() mock_client.return_value.city.return_value = mocked_response device = self._create_device(last_ip="172.217.22.10") @@ -585,10 +788,12 @@ def test_estimate_location_status_remove(self, mock_client): ): org.config_settings.estimated_location_enabled = False org.config_settings.save() + org.config_settings.refresh_from_db() location.geometry = GEOSGeometry("POINT(12.512124 41.898903)", srid=4326) location.save() + location.refresh_from_db() self.assertTrue(location.is_estimated) - self.assertIn(f"(Estimated Location: {device.last_ip})", location.name) + self.assertIn(f": {device.last_ip}", location.name) with self.subTest( "Test Estimated Status unchanged if Estimated feature is enabled" @@ -596,10 +801,12 @@ def test_estimate_location_status_remove(self, mock_client): ): org.config_settings.estimated_location_enabled = True org.config_settings.save() + org.config_settings.refresh_from_db() location._set_initial_values_for_changed_checked_fields() location.type = "outdoor" location.is_mobile = True location.save() + location.refresh_from_db() self.assertTrue(location.is_estimated) with self.subTest( @@ -607,9 +814,12 @@ def test_estimate_location_status_remove(self, mock_client): " and desired fields changed" ): location.geometry = GEOSGeometry("POINT(15.512124 45.898903)", srid=4326) - location.save() + location.save(update_fields=["geometry"]) + location.refresh_from_db() self.assertFalse(location.is_estimated) - self.assertNotIn(f"(Estimated Location: {device.last_ip})", location.name) + # Note: Name is no longer automatically cleaned up when + # is_estimated becomes False. Users must update the name manually + # if desired class TestEstimatedLocationFieldFilters( @@ -623,14 +833,6 @@ def setUp(self): admin = self._create_admin() self.client.force_login(admin) - def _create_device_location(self, **kwargs): - options = dict() - options.update(kwargs) - device_location = self.object_location_model(**options) - device_location.full_clean() - device_location.save() - return device_location - @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) def test_estimated_location_api_status_configured(self): org1 = self._get_org() @@ -646,21 +848,20 @@ def test_estimated_location_api_status_configured(self): org2_location = self._create_location(name="org2-location", organization=org2) org1_device = self._create_device(organization=org1) org2_device = self._create_device(organization=org2) - self._create_device_location(content_object=org1_device, location=org1_location) - self._create_device_location(content_object=org2_device, location=org2_location) + self._create_object_location(content_object=org1_device, location=org1_location) + self._create_object_location(content_object=org2_device, location=org2_location) with self.subTest("Test Estimated Location in Locations List"): path = reverse("geo_api:list_location") - with self.assertNumQueries(5): + with self.assertNumQueries(4): response = self.client.get(path) self.assertEqual(response.status_code, 200) self.assertEqual(response.data["count"], 2) self.assertContains(response, org1_location.id) self.assertContains(response, org2_location.id) - location1 = response.data["results"][1] - location2 = response.data["results"][0] - self.assertIn("is_estimated", location1) - self.assertNotIn("is_estimated", location2) + results_by_id = {item["id"]: item for item in response.data["results"]} + self.assertIn("is_estimated", results_by_id[str(org1_location.id)]) + self.assertNotIn("is_estimated", results_by_id[str(org2_location.id)]) with self.subTest("Test Estimated Location in Device Locations List"): path = reverse("geo_api:device_location", args=[org1_device.pk]) @@ -668,7 +869,6 @@ def test_estimated_location_api_status_configured(self): response = self.client.get(path) self.assertEqual(response.status_code, 200) self.assertIn("is_estimated", response.data["location"]["properties"]) - path = reverse("geo_api:device_location", args=[org2_device.pk]) with self.assertNumQueries(4): response = self.client.get(path) @@ -681,20 +881,19 @@ def test_estimated_location_api_status_configured(self): response = self.client.get(path) self.assertEqual(response.status_code, 200) self.assertEqual(response.data["count"], 2) - for i in response.data["features"]: - if i["id"] == org1_location.id: - self.assertIn("is_estimated", i["properties"]) - self.assertTrue(i["properties"]["is_estimated"]) - elif i["id"] == org2_location.id: - self.assertNotIn("is_estimated", i["properties"]) - self.assertFalse(i["properties"]["is_estimated"]) + results_by_id = {item["id"]: item for item in response.data["features"]} + location1_result = results_by_id[str(org1_location.id)] + location2_result = results_by_id[str(org2_location.id)] + self.assertIn("is_estimated", location1_result["properties"]) + self.assertTrue(location1_result["properties"]["is_estimated"]) + self.assertNotIn("is_estimated", location2_result["properties"]) @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False) def test_estimated_location_api_status_not_configured(self): org = self._get_org() location = self._create_location(name="org1-location", organization=org) device = self._create_device(organization=org) - self._create_device_location(content_object=device, location=location) + self._create_object_location(content_object=device, location=location) with self.subTest("Test Estimated status not in Locations List"): path = reverse("geo_api:list_location") @@ -721,7 +920,6 @@ def test_estimated_location_api_status_not_configured(self): self.assertEqual(response.data["count"], 1) location_features = response.data["features"][0] self.assertNotIn("is_estimated", location_features["properties"]) - self.assertFalse(location_features["properties"].get("is_estimated", False)) @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) def test_estimated_location_filter_list_api(self): @@ -732,13 +930,15 @@ def test_estimated_location_filter_list_api(self): location2 = self._create_location( name="location2", is_estimated=False, organization=org ) + location3 = self._create_location( + name="location3", is_estimated=False, organization=org + ) device1 = self._create_device() device2 = self._create_device( name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66" ) - self._create_device_location(content_object=device1, location=location1) - self._create_device_location(content_object=device2, location=location2) - + self._create_object_location(content_object=device1, location=location1) + self._create_object_location(content_object=device2, location=location2) path = reverse("geo_api:list_location") with self.subTest( @@ -751,16 +951,17 @@ def test_estimated_location_filter_list_api(self): self.assertEqual(response.data["count"], 1) self.assertContains(response, location1.id) self.assertNotContains(response, location2.id) + self.assertNotContains(response, location3.id) with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False): with self.subTest( "Test Estimated Location filter not available in location list " "when WHOIS not configured" ): - with self.assertNumQueries(5): + with self.assertNumQueries(4): response = self.client.get(path, {"is_estimated": True}) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data["count"], 2) + self.assertEqual(response.data["count"], 3) self.assertContains(response, location1.id) self.assertContains(response, location2.id) @@ -789,6 +990,39 @@ def test_estimated_location_filter_list_api(self): self.assertContains(response, device1.id) self.assertContains(response, device2.id) + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + def test_location_admin_estimated_field(self): + org = self._get_org() + estimated_location = self._create_location( + name="location1", is_estimated=True, organization=org + ) + estimated_device = self._create_device() + self._create_object_location( + content_object=estimated_device, location=estimated_location + ) + path = reverse("admin:geo_location_changelist") + response = self.client.get(path) + + with self.subTest("Test location Admin estimated field displayed"): + self.assertContains(response, "column-is_estimated") + + with self.subTest("Test location admin estimated field sorting enabled"): + self.assertContains(response, "sortable column-is_estimated") + + with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False): + response = self.client.get(path) + with self.subTest( + "Test location admin estimated field not displayed " + "when WHOIS not configured" + ): + self.assertNotContains(response, "column-is_estimated") + + with self.subTest( + "Test location admin estimated field sorting not enabled " + "when WHOIS not configured" + ): + self.assertNotContains(response, "sortable column-is_estimated") + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) def test_estimated_location_filter_admin(self): org = self._get_org() @@ -799,7 +1033,6 @@ def test_estimated_location_filter_admin(self): indoor_location = self._create_location( name="location3", organization=org, type="indoor" ) - estimated_device = self._create_device() outdoor_device = self._create_device( name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66" @@ -807,54 +1040,61 @@ def test_estimated_location_filter_admin(self): indoor_device = self._create_device( name="11:22:33:44:55:77", mac_address="11:22:33:44:55:77" ) - self._create_device_location( + self._create_object_location( content_object=estimated_device, location=estimated_location ) - self._create_device_location( + self._create_object_location( content_object=outdoor_device, location=outdoor_location ) - self._create_device_location( + self._create_object_location( content_object=indoor_device, location=indoor_location ) - path = reverse("admin:config_device_changelist") + with self.subTest("Test All Locations Filter"): response = self.client.get(path) self.assertContains(response, estimated_device.id) self.assertContains(response, outdoor_device.id) self.assertContains(response, indoor_device.id) + self.assertContains(response, "3 Devices") with self.subTest("Test Estimated Location Filter"): response = self.client.get(path, {"with_geo": "estimated"}) self.assertContains(response, estimated_device.id) self.assertNotContains(response, outdoor_device.id) self.assertNotContains(response, indoor_device.id) + self.assertContains(response, "1 Device") with self.subTest("Test Outdoor Location Filter"): response = self.client.get(path, {"with_geo": "outdoor"}) self.assertContains(response, outdoor_device.id) self.assertNotContains(response, estimated_device.id) self.assertNotContains(response, indoor_device.id) + self.assertContains(response, "1 Device") with self.subTest("Test Indoor Location Filter"): response = self.client.get(path, {"with_geo": "indoor"}) self.assertContains(response, indoor_device.id) self.assertNotContains(response, outdoor_device.id) self.assertNotContains(response, estimated_device.id) + self.assertContains(response, "1 Device") - with self.subTest("Test Indoor Location Filter"): + with self.subTest("Test No Location Filter"): response = self.client.get(path, {"with_geo": "false"}) self.assertNotContains(response, indoor_device.id) self.assertNotContains(response, outdoor_device.id) self.assertNotContains(response, estimated_device.id) + self.assertContains(response, "0 Devices") with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False): with self.subTest( "Test Estimated Location Admin specific filters not available" " when WHOIS not configured" ): - for i in ["estimated", "outdoor", "indoor"]: - response = self.client.get(path, {"with_geo": i}) - self.assertContains(response, estimated_device.id) - self.assertContains(response, outdoor_device.id) - self.assertContains(response, indoor_device.id) + for filter_value in ["estimated", "outdoor", "indoor"]: + with self.subTest(filter_value=filter_value): + response = self.client.get(path, {"with_geo": filter_value}) + self.assertContains(response, estimated_device.id) + self.assertContains(response, outdoor_device.id) + self.assertContains(response, indoor_device.id) + self.assertContains(response, "3 Devices") diff --git a/openwisp_controller/geo/estimated_location/tests/utils.py b/openwisp_controller/geo/estimated_location/tests/utils.py index 77bb618ba..0ca26f335 100644 --- a/openwisp_controller/geo/estimated_location/tests/utils.py +++ b/openwisp_controller/geo/estimated_location/tests/utils.py @@ -1,15 +1,34 @@ from swapper import load_model +from openwisp_controller.config.whois.tasks import fetch_whois_details from openwisp_controller.config.whois.tests.utils import CreateWHOISMixin +from ..tasks import manage_estimated_locations + OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") class TestEstimatedLocationMixin(CreateWHOISMixin): def setUp(self): + # skip the org config settings creation from the parent mixin super(CreateWHOISMixin, self).setUp() OrganizationConfigSettings.objects.create( organization=self._get_org(), whois_enabled=True, estimated_location_enabled=True, ) + + # helper to mock send_task to directly call the task function synchronously + @staticmethod + def run_task(name, args=None, kwargs=None, **_): + # This mock intercepts all Celery send_task calls and executes them + # synchronously for testing purposes. We need to handle both task names + # because when a device is created/saved, the WHOIS lookup task may be + # triggered first, which then triggers the estimated location task. + # Without handling both, the assertion would fail and break the test flow. + if name == "whois_estimated_location_task": + return manage_estimated_locations(*args or [], **kwargs or {}) + elif name == "openwisp_controller.config.whois.tasks.fetch_whois_details": + return fetch_whois_details(*args or [], **kwargs or {}) + # Let other tasks pass through (they may be mocked elsewhere) + return None diff --git a/openwisp_controller/geo/estimated_location/utils.py b/openwisp_controller/geo/estimated_location/utils.py index ddd43e837..f7516afdc 100644 --- a/openwisp_controller/geo/estimated_location/utils.py +++ b/openwisp_controller/geo/estimated_location/utils.py @@ -1,6 +1,49 @@ -from openwisp_notifications.utils import _get_object_link +from django.utils.translation import gettext_lazy as _ + +from openwisp_controller.config.whois.utils import MESSAGE_MAP + +# Mutating the existing MESSAGE_MAP to include estimated location messages +MESSAGE_MAP.update( + { + "estimated_location_error": { + "level": "error", + "type": "estimated_location_info", + "message": _( + "Unable to create estimated location for device " + "[{notification.target}]({notification.target_link}). " + "Please assign/create a location manually." + ), + "description": _("Multiple devices found for IP: {ip_address}"), + }, + "estimated_location_created": { + "type": "estimated_location_info", + "level": "info", + "message": _( + "Estimated location [{notification.actor}]({notification.actor_link})" + " for device" + " [{notification.target}]({notification.target_link})" + " {notification.verb} successfully." + ), + "description": _("Geographic coordinates inferred from IP: {ip_address}"), + }, + "estimated_location_updated": { + "type": "estimated_location_info", + "level": "info", + "message": _( + "Estimated location [{notification.actor}]({notification.actor_link})" + " for device" + " [{notification.target}]({notification.target_link})" + " updated successfully." + ), + "description": _("Geographic coordinates updated for IP: {ip_address}"), + }, + } +) def get_device_location_notification_target_url(obj, field, absolute_url=True): + # importing here to avoid "AppRegistryNotReady" + from openwisp_notifications.utils import _get_object_link + url = _get_object_link(obj._related_object(field), absolute_url) return f"{url}#devicelocation-group" diff --git a/openwisp_controller/geo/migrations/0004_location_is_estimated.py b/openwisp_controller/geo/migrations/0004_location_is_estimated.py index 1fcdfca68..45ac400da 100644 --- a/openwisp_controller/geo/migrations/0004_location_is_estimated.py +++ b/openwisp_controller/geo/migrations/0004_location_is_estimated.py @@ -16,6 +16,7 @@ class Migration(migrations.Migration): field=models.BooleanField( default=False, help_text=("Whether the location's coordinates are estimated."), + verbose_name="Is Estimated?", ), ), ] diff --git a/openwisp_controller/geo/templates/admin/geo/location/change_form.html b/openwisp_controller/geo/templates/admin/geo/location/change_form.html index af2fc9ceb..5c1bd85a4 100644 --- a/openwisp_controller/geo/templates/admin/geo/location/change_form.html +++ b/openwisp_controller/geo/templates/admin/geo/location/change_form.html @@ -1,9 +1,9 @@ {% extends "admin/django_loci/location_change_form.html" %} -{% load i18n admin_urls %} +{% load i18n %} {% block messages %} {{ block.super }} - {% if original and estimated_configured and original.is_estimated %} + {% if original and estimated_enabled and original.is_estimated %}
  • {% trans "This location is estimated based on the device's last IP address. Please refine it for greater accuracy." %}
diff --git a/openwisp_controller/geo/tests/test_api.py b/openwisp_controller/geo/tests/test_api.py index 4ebff1c9b..d8743fa8c 100644 --- a/openwisp_controller/geo/tests/test_api.py +++ b/openwisp_controller/geo/tests/test_api.py @@ -1,6 +1,7 @@ import json import tempfile import uuid +from unittest.mock import patch from django.contrib.auth import get_user_model from django.contrib.gis.geos import Point @@ -12,6 +13,7 @@ from rest_framework.authtoken.models import Token from swapper import load_model +from openwisp_controller.config import settings as config_app_settings from openwisp_controller.config.tests.utils import ( CreateConfigTemplateMixin, CreateDeviceMixin, @@ -533,7 +535,7 @@ def test_filter_location_list(self): path = reverse("geo_api:list_location") with self.subTest("Test without organization filtering"): - with self.assertNumQueries(5): + with self.assertNumQueries(4): response = self.client.get(path) self.assertEqual(response.status_code, 200) self.assertEqual(response.data["count"], 2) @@ -583,6 +585,7 @@ def test_filter_location_list(self): self.assertContains(response, org1_location.id) self.assertNotContains(response, org2_location.id) + @patch.object(config_app_settings, "WHOIS_CONFIGURED", False) def test_post_location_list(self): path = reverse("geo_api:list_location") coords = json.loads(Point(2, 23).geojson) @@ -614,6 +617,7 @@ def test_get_location_detail(self): else: self.fail("NoReverseMatch not raised as expected") + @patch.object(config_app_settings, "WHOIS_CONFIGURED", False) def test_put_location_detail(self): l1 = self._create_location() path = reverse("geo_api:detail_location", args=[l1.pk]) @@ -691,6 +695,7 @@ def test_delete_location_detail(self): response = self.client.delete(path) self.assertEqual(response.status_code, 204) + @patch.object(config_app_settings, "WHOIS_CONFIGURED", False) def test_create_location_with_floorplan(self): path = reverse("geo_api:list_location") fl_image = self._get_simpleuploadedfile() diff --git a/requirements.txt b/requirements.txt index 5a8d14148..386f4b9a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,4 +17,4 @@ shortuuid~=1.0.13 netaddr~=1.3.0 django-import-export~=4.3.14 jsonfield>=3.1.0,<4.0.0 -geoip2~=5.1.0 +geoip2>=5.1.0,<6.0.0 diff --git a/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py b/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py index 707a44bf5..d9125877c 100644 --- a/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py +++ b/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py @@ -1,7 +1,6 @@ # Generated by Django 5.2.1 on 2025-06-26 17:48 import django.utils.timezone -import jsonfield.fields import model_utils.fields from django.db import migrations, models @@ -46,22 +45,21 @@ class Migration(migrations.Migration): blank=True, help_text="Organization for ASN", max_length=100 ), ), - ("asn", models.CharField(blank=True, help_text="ASN", max_length=6)), + ("asn", models.CharField(blank=True, help_text="ASN", max_length=10)), ( "timezone", models.CharField(blank=True, help_text="Time zone", max_length=35), ), ( "address", - jsonfield.fields.JSONField( - blank=True, default=dict, help_text="Address" - ), + models.JSONField(blank=True, default=dict, help_text="Address"), ), - ("cidr", models.CharField(blank=True, help_text="CIDR", max_length=20)), + ("cidr", models.CharField(blank=True, help_text="CIDR", max_length=49)), ("details", models.CharField(blank=True, max_length=64, null=True)), ], options={ "abstract": False, + "swappable": "CONFIG_WHOISINFO_MODEL", }, ), migrations.AddField( diff --git a/tests/openwisp2/sample_geo/migrations/0004_location_is_estimated.py b/tests/openwisp2/sample_geo/migrations/0004_location_is_estimated.py index e3ad65809..1c2e65e19 100644 --- a/tests/openwisp2/sample_geo/migrations/0004_location_is_estimated.py +++ b/tests/openwisp2/sample_geo/migrations/0004_location_is_estimated.py @@ -16,6 +16,7 @@ class Migration(migrations.Migration): field=models.BooleanField( default=False, help_text="Whether the location's coordinates are estimated.", + verbose_name="Is Estimated?", ), ), ]