diff --git a/docs/developer/extending.rst b/docs/developer/extending.rst index eb29af94a..eda5c0b05 100644 --- a/docs/developer/extending.rst +++ b/docs/developer/extending.rst @@ -344,6 +344,7 @@ Once you have created the models, add the following to your CONFIG_VPNCLIENT_MODEL = "sample_config.VpnClient" CONFIG_ORGANIZATIONCONFIGSETTINGS_MODEL = "sample_config.OrganizationConfigSettings" CONFIG_ORGANIZATIONLIMITS_MODEL = "sample_config.OrganizationLimits" + CONFIG_WHOISINFO_MODEL = "sample_config.WHOISInfo" DJANGO_X509_CA_MODEL = "sample_pki.Ca" DJANGO_X509_CERT_MODEL = "sample_pki.Cert" GEO_LOCATION_MODEL = "sample_geo.Location" diff --git a/docs/index.rst b/docs/index.rst index 3372e6eae..8bdb49d88 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -47,6 +47,8 @@ the OpenWISP architecture. user/vxlan-wireguard.rst user/zerotier.rst user/openvpn.rst + user/whois.rst + user/estimated-location.rst user/subnet-division-rules.rst user/rest-api.rst user/settings.rst diff --git a/docs/user/estimated-location.rst b/docs/user/estimated-location.rst new file mode 100644 index 000000000..55c9188da --- /dev/null +++ b/docs/user/estimated-location.rst @@ -0,0 +1,123 @@ +Estimated Location +================== + +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/is-estimated-flag.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/is-estimated-flag.png + :alt: Estimated location flag + +.. important:: + + The **Estimated Location** feature is **disabled by default**. + + Before enabling it, the :ref:`WHOIS Lookup feature + ` must be enabled. + + Then set :ref:`OPENWISP_CONTROLLER_ESTIMATED_LOCATION_ENABLED` to + ``True``. + +.. contents:: **Table of contents**: + :depth: 1 + :local: + +Overview +-------- + +This feature automatically creates or updates a device's location based on +latitude and longitude information retrieved from the :doc:`whois` +feature. + +It is very useful for those users who have devices scattered across +different geographic regions and would like some help to place the devices +on the map, while being gently reminded to improve the precision of the +location with a direct link for doing so. + +It also significantly reduces the effort required to assign a geographic +location manually when many devices are deployed in large buildings like +schools, offices, hospitals, libraries, etc. Improve the precision of the +estimated location just once and all the other devices sharing the same +public IP will automatically inherit the same location. + +The feature is not useful in the following scenarios: + +- Most devices are deployed in one single location. +- Most devices are mobile (e.g. moving vehicles). + +Visibility of Estimated Status +------------------------------ + +The estimated status of a location is visible in the admin interface in +several ways: + +- The location name will mention the *IP address* from which it was + estimated. +- A *warning message* appears at the top of the location list page as in + the image below. + + .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/estimated-warning.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/estimated-warning.png + :alt: Estimated location warning + +- The *Is Estimated?* flag is displayed both in the location list page and + in the location detail page, as in the images below. + + .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/admin-list.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/admin-list.png + :alt: Estimated location admin list + + .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/is-estimated-flag.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/is-estimated-flag.png + :alt: Estimated location flag + +- The device list page also allows filtering devices which are associated + with estimated locations as shown in the image below. + + .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/filter-devices-by-estimated-location.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/filter-devices-by-estimated-location.png + :alt: Filter devices associated to estimated locations + +Any change to the geographic coordinates of an estimated location will set +the ``is_estimated`` field to ``False``. + +When manually increasing the precision of estimated locations, it is +highly recommended to also change the auto-generated location name. + +In the REST API, the ``is_estimated`` field is visible in the :ref:`Device +Location `, :ref:`Location list +`, :ref:`Location Detail +` and :ref:`Location list (GeoJSON) +` endpoints if the feature is enabled. The +field can also be used for filtering in the location list endpoints, +including the GeoJSON endpoint, and in the :ref:`Device List +`. + +Triggers and Record Management +------------------------------ + +The feature is triggered automatically when all the following conditions +are met: + +- A WHOIS lookup is performed. +- The last IP is a public IP address. +- Both WHOIS lookup and Estimated Location features are enabled for the + device's organization. + +If no matching location exists, a new estimated location is created using +coordinates from the WHOIS record. If an estimated location already +exists, it will be updated with the new coordinates. + +If another device with the same IP already has a location, the system will +assign the same location for any device having the same IP and not being +assigned to any other location. + +If two devices share the same IP and are assigned to the same location, +and one of them updates its last IP, the system will create a new +estimated location for that device. + +When multiple devices with the same IP already have a location assigned +but the locations differ, the system will send a notification to network +administrators asking to manually resolve the ambiguity. + +When WHOIS records are updated as described in :ref:`the WHOIS Lookup +section `, any related estimated +location will also be updated, if needed and only if the estimated +location has not been manually modified to increase precision. diff --git a/docs/user/intro.rst b/docs/user/intro.rst index 98c4b0abe..1c4b123ae 100644 --- a/docs/user/intro.rst +++ b/docs/user/intro.rst @@ -35,6 +35,8 @@ following features: - **VPN management**: automatically provision VPN tunnel configurations, including cryptographic keys and IP addresses, e.g.: :doc:`OpenVPN `, :doc:`WireGuard ` +- :doc:`whois`: display information about the public IP address used by + devices to communicate with OpenWISP - :doc:`import-export` It exposes various :doc:`REST API endpoints `. @@ -96,6 +98,9 @@ The geographic app is based on `django-loci geographic coordinates of the devices, as well as their indoor coordinates on floor plan images. +It also provides an :doc:`estimated-location` feature which automatically +creates or updates device locations based on WHOIS data. + It exposes various :doc:`REST API endpoints `. Subnet Division App diff --git a/docs/user/rest-api.rst b/docs/user/rest-api.rst index a4e15febe..2ec0b48ed 100644 --- a/docs/user/rest-api.rst +++ b/docs/user/rest-api.rst @@ -68,6 +68,22 @@ List Devices GET /api/v1/controller/device/ +.. _device_list_whois: + +**WHOIS Details** + +If :doc:`WHOIS Lookup feature ` is enabled, each device in the list +response will also include a ``whois_info`` field with related brief WHOIS +information. + +.. _device_list_estimated_filters: + +**Estimated Location Filters** + +If :doc:`Estimated Location feature ` is enabled, +devices can be filtered based on the estimated nature of their location +using the ``geo_is_estimated`` filter. + **Available filters** You can filter a list of devices based on their configuration status using @@ -145,6 +161,13 @@ Get Device Detail GET /api/v1/controller/device/{id}/ +.. _device_detail_whois: + +**WHOIS Details** + +If :doc:`WHOIS Lookup feature ` is enabled, the response will also +include a ``whois_info`` field with related detailed WHOIS information. + Download Device Configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -529,6 +552,13 @@ of certificate's organization as show in the example below: GET /api/v1/controller/cert/{common_name}/group/?org={org1_slug},{org2_slug} +.. |est_loc| replace:: Estimated Location feature + +.. _est_loc: estimated-location.html + +.. |estimated_details| replace:: If |est_loc|_ is enabled, the location + response will also include ``is_estimated`` status field. + Get Device Location ~~~~~~~~~~~~~~~~~~~ @@ -536,6 +566,12 @@ Get Device Location GET /api/v1/controller/device/{id}/location/ +.. _device_location_estimated: + +**Estimated Status** + +|estimated_details| + .. _create_device_location: Create Device Location @@ -772,6 +808,14 @@ List Locations GET /api/v1/controller/location/ +.. _location_list_estimated: + +**Estimated Status** + +|estimated_details| + +Locations can also be filtered using the ``is_estimated`` filter. + **Available filters** You can filter using ``organization_id`` or ``organization_slug`` to get @@ -853,6 +897,12 @@ Get Location Details GET /api/v1/controller/location/{pk}/ +.. _location_detail_estimated: + +**Estimated Status** + +|estimated_details| + Change Location Details ~~~~~~~~~~~~~~~~~~~~~~~ @@ -895,6 +945,14 @@ List Locations with Devices Deployed (in GeoJSON Format) GET /api/v1/controller/location/geojson/ +.. _location_geojson_estimated: + +**Estimated Status** + +|estimated_details| + +Locations can also be filtered using the ``is_estimated`` filter. + **Available filters** You can filter using ``organization_id`` or ``organization_slug`` to get diff --git a/docs/user/settings.rst b/docs/user/settings.rst index ee32e4875..aa59425ef 100644 --- a/docs/user/settings.rst +++ b/docs/user/settings.rst @@ -761,3 +761,92 @@ recoverable failures, improving the reliability of the system. For more information on these settings, you can refer to the `the celery documentation regarding automatic retries for known errors. `_ + +.. _openwisp_controller_whois_enabled: + +``OPENWISP_CONTROLLER_WHOIS_ENABLED`` +------------------------------------- + +============ ========= +**type**: ``bool`` +**default**: ``False`` +============ ========= + +Allows enabling the optional :doc:`WHOIS Lookup feature `. + +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-setting.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-setting.png + :alt: WHOIS admin setting + +After enabling this feature, you have to set +:ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT +` and +:ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY +`. + +.. warning:: + + If these three settings are not configured as expected, an + ``ImproperlyConfigured`` exception will be raised. + +.. _openwisp_controller_whois_geoip_account: + +``OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT`` +------------------------------------------- + +============ ======= +**type**: ``str`` +**default**: ``""`` +============ ======= + +MaxMind Account ID required for the :doc:`WHOIS Lookup feature `. + +.. _openwisp_controller_whois_geoip_key: + +``OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY`` +--------------------------------------- + +============ ======= +**type**: ``str`` +**default**: ``""`` +============ ======= + +MaxMind License Key required for the :doc:`WHOIS Lookup feature `. + +.. _openwisp_controller_whois_refresh_threshold_days: + +``OPENWISP_CONTROLLER_WHOIS_REFRESH_THRESHOLD_DAYS`` +---------------------------------------------------- + +============ ======= +**type**: ``int`` +**default**: ``90`` +============ ======= + +Specifies the number of days after which the WHOIS information for a +device is considered stale and eligible for refresh. + +.. _openwisp_controller_estimated_location_enabled: + +``OPENWISP_CONTROLLER_ESTIMATED_LOCATION_ENABLED`` +-------------------------------------------------- + +============ ========= +**type**: ``bool`` +**default**: ``False`` +============ ========= + +Allows enabling the optional :doc:`Estimated Location feature +`. + +.. warning:: + + :ref:`OPENWISP_CONTROLLER_WHOIS_ENABLED + ` must be set to ``True`` before + enabling this feature. Enabling estimated locations while + ``WHOIS_ENABLED=False`` will raise ``ImproperlyConfigured`` at startup + time. + +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/admin-setting.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/admin-setting.png + :alt: Estimated Location setting diff --git a/docs/user/whois.rst b/docs/user/whois.rst new file mode 100644 index 000000000..264b2b163 --- /dev/null +++ b/docs/user/whois.rst @@ -0,0 +1,129 @@ +WHOIS Lookup +============ + +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-details.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-details.png + :alt: WHOIS admin details + +.. important:: + + The **WHOIS Lookup** feature is **disabled by default**. + + To enable it, follow the :ref:`controller_setup_whois_lookup` below. + +.. contents:: **Table of contents**: + :depth: 1 + :local: + +Overview +-------- + +This feature displays information about the public IP address used by +devices to communicate with OpenWISP (via the ``last_ip`` field). It helps +identify the geographic location and ISP associated with the IP address, +which can be useful for troubleshooting network issues. + +.. note:: + + Once WHOIS Lookups are enabled, no manual intervention is needed: + everything is handled automatically, including the refresh of old + data. See :ref:`controller_whois_auto_management` for more + information. + +The retrieved information pertains to the Autonomous System (ASN) +associated with the device's public IP address and includes: + +- ASN (Autonomous System Number) +- Organization name that owns the ASN +- CIDR block assigned to the ASN +- Physical address registered to the ASN +- Timezone of the ASN's registered location +- Coordinates (Latitude and Longitude) + +.. note:: + + This data also serves as a base for the :doc:`Estimated Location + feature <./estimated-location>`. + +.. _controller_setup_whois_lookup: + +Setup Instructions +------------------ + +1. Create a MaxMind account: `Sign up here + `_. + + If you already have an account, just **Sign In**. + +2. Go to **Manage License Keys** in your MaxMind dashboard. +3. Generate a new license key and name it as you prefer. +4. Copy both the **Account ID** and **License Key**. +5. Set the following settings accordingly: + + - Set :ref:`OPENWISP_CONTROLLER_WHOIS_ENABLED` to ``True``. + - Set :ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT` to **Account ID**. + - Set :ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY` to **License Key**. + +6. Restart the application/containers if using ansible-openwisp2 or + docker. +7. Run the ``clear_last_ip`` management command to clear the last IP + address of any active device which doesn't have WHOIS info yet across + all organizations (which will trigger the WHOIS lookup at the next + config checksum check). + + - If using ansible-openwisp2 (default directory is /opt/openwisp2, + unless changed in Ansible playbook configuration): + + .. code-block:: bash + + source /opt/openwisp2/env/bin/activate + python /opt/openwisp2/manage.py clear_last_ip + + - If using docker: + + .. code-block:: bash + + docker exec -it sh + python manage.py clear_last_ip + +Viewing WHOIS Lookup Data +------------------------- + +Once the WHOIS Lookup feature is enabled and WHOIS data is available, the +retrieved details can be viewed as follows: + +- **Device Admin**: on the device's admin page, the WHOIS data is + displayed alongside the device's last IP address. + +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-details.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-details.png + :alt: WHOIS admin details + +- **Device REST API**: Refer to :ref:`Device List ` and + :ref:`Device Detail `. + +.. _controller_whois_auto_management: + +Triggers and Record Management +------------------------------ + +A WHOIS lookup is triggered automatically when: + +- A new device registers and the last IP is a public IP address. +- A device's last IP address changes and is a public IP address. +- A device fetches its checksum **and** either no WHOIS record exists for + the IP or the existing record is older than the :ref:`configured + threshold `. + +The lookup will only run if the device's last IP address is **public** and +WHOIS lookup is **enabled** for the device's organization. + +When a device updates its last IP address, a WHOIS lookup is triggered for +the **new IP** and the **WHOIS record for the old IP** is deleted, unless +any active devices are associated with that IP address. + +.. note:: + + When a device with an associated WHOIS record is deleted, its WHOIS + record is automatically removed, but only if no other active devices + are associated with the same IP address. diff --git a/openwisp_controller/config/admin.py b/openwisp_controller/config/admin.py index 9c758fd06..7c328a353 100644 --- a/openwisp_controller/config/admin.py +++ b/openwisp_controller/config/admin.py @@ -31,6 +31,7 @@ from swapper import load_model from openwisp_controller.config.views import get_default_values, get_relevant_templates +from openwisp_controller.config.whois.utils import get_whois_info from openwisp_users.admin import OrganizationAdmin from openwisp_users.multitenancy import MultitenantOrgFilter from openwisp_utils.admin import ( @@ -49,6 +50,7 @@ logger = logging.getLogger(__name__) prefix = "config/" +whois_prefix = "whois/" Config = load_model("config", "Config") Device = load_model("config", "Device") DeviceGroup = load_model("config", "DeviceGroup") @@ -569,12 +571,21 @@ class DeviceAdmin(MultitenantAdminMixin, BaseConfigAdmin, UUIDAdmin): fields.insert(0, "hardware_id") list_select_related = ("config", "organization") - class Media(BaseConfigAdmin.Media): + # overriding media property to allow testing in isolation + # as class Media is evaluated at import time making the + # settings not overridable in tests + @property + def media(self): js = BaseConfigAdmin.Media.js + [ f"{prefix}js/tabs.js", f"{prefix}js/management_ip.js", f"{prefix}js/relevant_templates.js", ] + css = BaseConfigAdmin.Media.css["all"] + if app_settings.WHOIS_CONFIGURED: + js.append(f"{whois_prefix}js/whois.js") + css += (f"{whois_prefix}css/whois.css",) + return super().media + forms.Media(js=js, css={"all": css}) def has_change_permission(self, request, obj=None): perm = super().has_change_permission(request) @@ -928,6 +939,10 @@ def get_extra_context(self, pk=None): ), } ) + # passing the whois details to the context to avoid + # the need to make an additional request in the js + if data := get_whois_info(pk): + ctx["device_whois_details"] = data return ctx def add_view(self, request, form_url="", extra_context=None): @@ -1362,18 +1377,29 @@ def has_delete_permission(self, request, obj): limits_inline_position = 0 -if getattr(app_settings, "REGISTRATION_ENABLED", True): - class ConfigSettingsForm(AlwaysHasChangedMixin, forms.ModelForm): - class Meta: - widgets = {"context": FlatJsonWidget} - class ConfigSettingsInline(admin.StackedInline): - model = OrganizationConfigSettings - form = ConfigSettingsForm +class ConfigSettingsForm(AlwaysHasChangedMixin, forms.ModelForm): + class Meta: + widgets = {"context": FlatJsonWidget} + + +class ConfigSettingsInline(admin.StackedInline): + model = OrganizationConfigSettings + form = ConfigSettingsForm + + def get_fields(self, request, obj=None): + fields = [] + if app_settings.REGISTRATION_ENABLED: + fields += ["registration_enabled", "shared_secret"] + if app_settings.WHOIS_CONFIGURED: + fields += ["whois_enabled", "estimated_location_enabled"] + fields += ["context"] + return fields + - OrganizationAdmin.save_on_top = True - OrganizationAdmin.inlines.insert(0, ConfigSettingsInline) - limits_inline_position = 1 +OrganizationAdmin.save_on_top = True +OrganizationAdmin.inlines.insert(0, ConfigSettingsInline) +limits_inline_position = 1 OrganizationAdmin.inlines.insert(limits_inline_position, OrganizationLimitsInline) diff --git a/openwisp_controller/config/api/serializers.py b/openwisp_controller/config/api/serializers.py index 7e67d8f5f..f07627e60 100644 --- a/openwisp_controller/config/api/serializers.py +++ b/openwisp_controller/config/api/serializers.py @@ -9,6 +9,7 @@ from ...serializers import BaseSerializer from .. import settings as app_settings +from ..whois.mixins import WHOISMixin Template = load_model("config", "Template") Vpn = load_model("config", "Vpn") @@ -274,7 +275,7 @@ class DeviceDetailConfigSerializer(BaseConfigSerializer): templates = FilterTemplatesByOrganization(many=True) -class DeviceDetailSerializer(DeviceConfigSerializer): +class DeviceDetailSerializer(WHOISMixin, DeviceConfigSerializer): config = DeviceDetailConfigSerializer(allow_null=True) is_deactivated = serializers.BooleanField(read_only=True) diff --git a/openwisp_controller/config/apps.py b/openwisp_controller/config/apps.py index 973b46c63..4354fcd10 100644 --- a/openwisp_controller/config/apps.py +++ b/openwisp_controller/config/apps.py @@ -29,6 +29,7 @@ vpn_peers_changed, vpn_server_modified, ) +from .whois.handlers import connect_whois_handlers # ensure Device.hardware_id field is not flagged as unique # (because it's flagged as unique_together with organization) @@ -50,6 +51,7 @@ def ready(self, *args, **kwargs): self.register_dashboard_charts() self.register_menu_groups() self.notification_cache_update() + connect_whois_handlers() def __setmodels__(self): self.device_model = load_model("config", "Device") diff --git a/openwisp_controller/config/base/device.py b/openwisp_controller/config/base/device.py index 763f25ecc..d985ccaab 100644 --- a/openwisp_controller/config/base/device.py +++ b/openwisp_controller/config/base/device.py @@ -1,3 +1,4 @@ +from functools import cached_property from hashlib import md5 from django.core.exceptions import ObjectDoesNotExist, PermissionDenied, ValidationError @@ -18,6 +19,7 @@ management_ip_changed, ) from ..validators import device_name_validator, mac_address_validator +from ..whois.service import WHOISService from .base import BaseModel @@ -117,6 +119,13 @@ class Meta: verbose_name_plural = app_settings.DEVICE_VERBOSE_NAME[1] def __init__(self, *args, **kwargs): + if ( + app_settings.WHOIS_CONFIGURED + and "last_ip" not in self._changed_checked_fields + ): + # Initial value for last_ip is required in WHOIS + # to remove WHOIS info related to that ip address. + self._changed_checked_fields = [*self._changed_checked_fields, "last_ip"] super().__init__(*args, **kwargs) self._set_initial_values_for_changed_checked_fields() @@ -279,6 +288,8 @@ def save(self, *args, **kwargs): self.key = self.generate_key(shared_secret) state_adding = self._state.adding super().save(*args, **kwargs) + if app_settings.WHOIS_CONFIGURED: + self._check_last_ip(creating=state_adding) if state_adding and self.group and self.group.templates.exists(): self.create_default_config() # The value of "self._state.adding" will always be "False" @@ -299,7 +310,9 @@ def _check_changed_fields(self): self._get_initial_values_for_checked_fields() # Execute method for checked for each field in self._changed_checked_fields for field in self._changed_checked_fields: - getattr(self, f"_check_{field}_changed")() + method = getattr(self, f"_check_{field}_changed", None) + if callable(method): + method() def _is_deferred(self, field): """ @@ -509,3 +522,24 @@ def config_deactivated_clear_management_ip(cls, instance, *args, **kwargs): is changed to 'deactivated'. """ cls.objects.filter(pk=instance.device_id).update(management_ip="") + + @cached_property + def whois_service(self): + """ + Used as a shortcut to get WHOISService instance + for the device. + """ + return WHOISService(self) + + def _check_last_ip(self, creating=False): + """ + Process details and location related to last_ip if last_ip has + changed or is being set for the first time. + """ + if self._initial_last_ip == models.DEFERRED: + return + if creating or self.last_ip != self._initial_last_ip: + self.whois_service.process_ip_data_and_location() + else: + self.whois_service.update_whois_info() + self._initial_last_ip = self.last_ip diff --git a/openwisp_controller/config/base/multitenancy.py b/openwisp_controller/config/base/multitenancy.py index 9cdf15736..bceb5de42 100644 --- a/openwisp_controller/config/base/multitenancy.py +++ b/openwisp_controller/config/base/multitenancy.py @@ -2,12 +2,15 @@ from copy import deepcopy import swapper +from django.core.exceptions import ValidationError from django.db import models from django.utils.translation import gettext_lazy as _ from jsonfield import JSONField from openwisp_utils.base import KeyField, UUIDModel +from openwisp_utils.fields import FallbackBooleanChoiceField +from .. import settings as app_settings from ..exceptions import OrganizationDeviceLimitExceeded from ..tasks import bulk_invalidate_config_get_cached_checksum @@ -31,6 +34,16 @@ class AbstractOrganizationConfigSettings(UUIDModel): verbose_name=_("shared secret"), help_text=_("used for automatic registration of devices"), ) + whois_enabled = FallbackBooleanChoiceField( + help_text=_("Whether the WHOIS lookup feature is enabled"), + fallback=app_settings.WHOIS_ENABLED, + verbose_name=_("WHOIS Enabled"), + ) + estimated_location_enabled = FallbackBooleanChoiceField( + help_text=_("Whether the estimated location feature is enabled"), + fallback=app_settings.ESTIMATED_LOCATION_ENABLED, + verbose_name=_("Estimated Location Enabled"), + ) context = JSONField( blank=True, default=dict, @@ -53,6 +66,27 @@ def __str__(self): def get_context(self): return deepcopy(self.context) + def clean(self): + if not app_settings.WHOIS_CONFIGURED and self.whois_enabled: + raise ValidationError( + { + "whois_enabled": _( + "WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY must be set " + + "before enabling WHOIS feature." + ) + } + ) + if not self.whois_enabled and self.estimated_location_enabled: + raise ValidationError( + { + "estimated_location_enabled": _( + "Estimated Location feature requires " + "WHOIS Lookup feature to be enabled." + ) + } + ) + return super().clean() + def save( self, force_insert=False, force_update=False, using=None, update_fields=None ): diff --git a/openwisp_controller/config/base/whois.py b/openwisp_controller/config/base/whois.py new file mode 100644 index 000000000..6a2abac38 --- /dev/null +++ b/openwisp_controller/config/base/whois.py @@ -0,0 +1,165 @@ +from ipaddress import ip_address, ip_network + +from django.contrib.gis.db.models import PointField +from django.core.cache import cache +from django.core.exceptions import ValidationError +from django.db import models, transaction +from django.utils.translation import gettext_lazy as _ + +from openwisp_utils.base import TimeStampedEditableModel + +from ..whois.service import WHOISService +from ..whois.tasks import delete_whois_record + + +class AbstractWHOISInfo(TimeStampedEditableModel): + """ + Abstract model to store WHOIS information + for a device. + """ + + id = None + # Using ip_address as primary key to avoid redundant lookups + # and storage of duplicate WHOIS information per IP address. + # Whenever a device's last ip address changes, data related to + # previous IP address is deleted. If any device still has the + # previous IP address, they will trigger the lookup again + # ensuring latest WHOIS information is always available. + ip_address = models.GenericIPAddressField(db_index=True, primary_key=True) + isp = models.CharField( + max_length=100, + blank=True, + help_text=_("Organization for ASN"), + ) + asn = models.CharField( + max_length=10, + blank=True, + help_text=_("ASN"), + ) + timezone = models.CharField( + max_length=35, + blank=True, + help_text=_("Time zone"), + ) + address = models.JSONField( + default=dict, + help_text=_("Address"), + blank=True, + ) + cidr = models.CharField( + max_length=49, + blank=True, + help_text=_("CIDR"), + ) + coordinates = PointField( + null=True, + blank=True, + help_text=_("Coordinates"), + srid=4326, + ) + + class Meta: + abstract = True + + def clean(self): + if ip_address(self.ip_address).is_private: + raise ValidationError( + { + "ip_address": _( + "WHOIS information cannot be created for private IP addresses." + ) + } + ) + if self.cidr: + try: + # strict is set to False to allow CIDR without a mask + # e.g. 192.168.1.12/24 with strict False normalizes to + # 192.168.1.0/24 else it would raise an error. + ip_network(self.cidr, strict=False) + except ValueError as e: + raise ValidationError( + {"cidr": _("Invalid CIDR format: %(error)s") % {"error": str(e)}} + ) from e + if self.coordinates: + if not (-90 <= self.coordinates.y <= 90): + raise ValidationError( + {"coordinates": _("Latitude must be between -90 and 90 degrees.")} + ) + if not (-180 <= self.coordinates.x <= 180): + raise ValidationError( + { + "coordinates": _( + "Longitude must be between -180 and 180 degrees." + ) + } + ) + return super().clean() + + @staticmethod + def device_whois_info_delete_handler(instance, **kwargs): + """ + Delete WHOIS information for a device when the last IP address is removed or + when device is deleted. + """ + last_ip = instance.last_ip + if last_ip and instance._get_organization__config_settings().whois_enabled: + transaction.on_commit(lambda: delete_whois_record.delay(last_ip)) + + # this method is kept here instead of in OrganizationConfigSettings because + # currently the caching is used only for WHOIS feature + @staticmethod + def invalidate_org_settings_cache(instance, **kwargs): + """ + Invalidate the cache for Organization settings on update/delete of + Organization settings instance. + """ + org_id = instance.organization_id + cache.delete(WHOISService.get_cache_key(org_id)) + + @property + def formatted_address(self): + """ + Used as default formatter for address field. + 'filter' is used to remove any None values + """ + return ", ".join( + filter( + None, + [ + self.address.get("city"), + self.address.get("country"), + self.address.get("continent"), + self.address.get("postal"), + ], + ) + ) + + @property + def _location_name(self): + """ + Used to get location name based on the address and IP. + """ + address = self.formatted_address + if address: + parts = [part.strip() for part in address.split(",")[:2] if part.strip()] + location = ", ".join(parts) + # Use named placeholders so translators receive the template + return _("%(location)s: %(ip)s") % { + "location": location, + "ip": self.ip_address, + } + # Use named placeholder for consistency + return _("Estimated Location: %(ip)s") % {"ip": self.ip_address} + + def _get_defaults_for_estimated_location(self): + """ + Used to get default values for creating or updating + an estimated location based on the WHOIS information. + """ + return { + "name": self._location_name, + "type": "outdoor", + "is_mobile": False, + "geometry": self.coordinates, + "address": self.formatted_address, + } diff --git a/openwisp_controller/config/management/__init__.py b/openwisp_controller/config/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/config/management/commands/__init__.py b/openwisp_controller/config/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/config/management/commands/clear_last_ip.py b/openwisp_controller/config/management/commands/clear_last_ip.py new file mode 100644 index 000000000..55ae9ba20 --- /dev/null +++ b/openwisp_controller/config/management/commands/clear_last_ip.py @@ -0,0 +1,26 @@ +from django.core.management.base import BaseCommand + +from openwisp_controller.config.whois.commands import clear_last_ip_command + + +class Command(BaseCommand): + help = ( + "Clears the last IP address (if set) for active devices without WHOIS records" + " across all organizations." + ) + + def add_arguments(self, parser): + parser.add_argument( + "--noinput", + "--no-input", + action="store_false", + dest="interactive", + help="Do NOT prompt the user for input of any kind.", + ) + + def handle(self, *args, **options): + clear_last_ip_command( + stdout=self.stdout, + stderr=self.stderr, + interactive=options["interactive"], + ) diff --git a/openwisp_controller/config/migrations/0061_config_checksum_db.py b/openwisp_controller/config/migrations/0061_config_checksum_db.py index a572951cf..34319cae3 100644 --- a/openwisp_controller/config/migrations/0061_config_checksum_db.py +++ b/openwisp_controller/config/migrations/0061_config_checksum_db.py @@ -1,7 +1,6 @@ # Generated by Django for issue #1113 optimization from django.db import migrations, models -from swapper import load_model def populate_checksum_db(apps, schema_editor): @@ -13,7 +12,7 @@ def populate_checksum_db(apps, schema_editor): hence we use Config.objects.bulk_update() instead of Config.update_status_if_checksum_changed(). """ - Config = load_model("config", "Config") + Config = apps.get_model("config", "Config") chunk_size = 100 updated_configs = [] qs = ( diff --git a/openwisp_controller/config/migrations/0062_whoisinfo.py b/openwisp_controller/config/migrations/0062_whoisinfo.py new file mode 100644 index 000000000..8899f49c3 --- /dev/null +++ b/openwisp_controller/config/migrations/0062_whoisinfo.py @@ -0,0 +1,80 @@ +# Generated by Django 5.2.1 on 2025-06-26 02:13 + +import django.utils.timezone +import model_utils.fields +from django.db import migrations, models + +import openwisp_utils.fields + + +class Migration(migrations.Migration): + dependencies = [ + ("config", "0061_config_checksum_db"), + ] + + operations = [ + migrations.CreateModel( + name="WHOISInfo", + fields=[ + ( + "created", + model_utils.fields.AutoCreatedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="created", + ), + ), + ( + "modified", + model_utils.fields.AutoLastModifiedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="modified", + ), + ), + ( + "ip_address", + models.GenericIPAddressField( + db_index=True, primary_key=True, serialize=False + ), + ), + ( + "asn", + models.CharField(blank=True, help_text="ASN", max_length=10), + ), + ( + "timezone", + models.CharField(blank=True, help_text="Time zone", max_length=35), + ), + ( + "address", + models.JSONField(blank=True, default=dict, help_text="Address"), + ), + ("cidr", models.CharField(blank=True, help_text="CIDR", max_length=49)), + ( + "isp", + models.CharField( + blank=True, + help_text="Organization for ASN", + max_length=100, + ), + ), + ], + options={ + "abstract": False, + "swappable": "CONFIG_WHOISINFO_MODEL", + }, + ), + migrations.AddField( + model_name="organizationconfigsettings", + name="whois_enabled", + field=openwisp_utils.fields.FallbackBooleanChoiceField( + blank=True, + default=None, + fallback=False, + help_text="Whether the WHOIS lookup feature is enabled", + null=True, + verbose_name="WHOIS Enabled", + ), + ), + ] diff --git a/openwisp_controller/config/migrations/0063_organizationconfigsettings_estimated_location_enabled_and_more.py b/openwisp_controller/config/migrations/0063_organizationconfigsettings_estimated_location_enabled_and_more.py new file mode 100644 index 000000000..bdb3dab64 --- /dev/null +++ b/openwisp_controller/config/migrations/0063_organizationconfigsettings_estimated_location_enabled_and_more.py @@ -0,0 +1,35 @@ +# Generated by Django 5.2.1 on 2025-07-10 18:09 + +import django.contrib.gis.db.models.fields +from django.db import migrations + +import openwisp_utils.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("config", "0062_whoisinfo"), + ] + + operations = [ + migrations.AddField( + model_name="organizationconfigsettings", + name="estimated_location_enabled", + field=openwisp_utils.fields.FallbackBooleanChoiceField( + blank=True, + default=None, + fallback=False, + help_text="Whether the estimated location feature is enabled", + null=True, + verbose_name="Estimated Location Enabled", + ), + ), + migrations.AddField( + model_name="whoisinfo", + name="coordinates", + field=django.contrib.gis.db.models.fields.PointField( + blank=True, help_text="Coordinates", null=True, srid=4326 + ), + ), + ] diff --git a/openwisp_controller/config/models.py b/openwisp_controller/config/models.py index 9795abba8..e36280319 100644 --- a/openwisp_controller/config/models.py +++ b/openwisp_controller/config/models.py @@ -10,6 +10,7 @@ from .base.tag import AbstractTaggedTemplate, AbstractTemplateTag from .base.template import AbstractTemplate from .base.vpn import AbstractVpn, AbstractVpnClient +from .base.whois import AbstractWHOISInfo class Device(AbstractDevice): @@ -111,3 +112,13 @@ class OrganizationLimits(AbstractOrganizationLimits): class Meta(AbstractOrganizationLimits.Meta): abstract = False swappable = swapper.swappable_setting("config", "OrganizationLimits") + + +class WHOISInfo(AbstractWHOISInfo): + """ + Stores WHOIS information for devices. + """ + + class Meta(AbstractWHOISInfo.Meta): + abstract = False + swappable = swapper.swappable_setting("config", "WHOISInfo") diff --git a/openwisp_controller/config/settings.py b/openwisp_controller/config/settings.py index 783bdcd49..6c591c489 100644 --- a/openwisp_controller/config/settings.py +++ b/openwisp_controller/config/settings.py @@ -1,6 +1,7 @@ import logging from django.conf import settings +from django.core.exceptions import ImproperlyConfigured from django.utils.translation import gettext_lazy as _ logger = logging.getLogger(__name__) @@ -67,3 +68,26 @@ def get_setting(option, default): "API_TASK_RETRY_OPTIONS", dict(max_retries=5, retry_backoff=True, retry_backoff_max=600, retry_jitter=True), ) +WHOIS_GEOIP_ACCOUNT = get_setting("WHOIS_GEOIP_ACCOUNT", "") +WHOIS_GEOIP_KEY = get_setting("WHOIS_GEOIP_KEY", "") +WHOIS_ENABLED = get_setting("WHOIS_ENABLED", False) +WHOIS_CONFIGURED = bool(WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY) +if WHOIS_ENABLED and not WHOIS_CONFIGURED: + raise ImproperlyConfigured( + "OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT and " + "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY must be set " + + "when OPENWISP_CONTROLLER_WHOIS_ENABLED is True." + ) +WHOIS_REFRESH_THRESHOLD_DAYS = get_setting("WHOIS_REFRESH_THRESHOLD_DAYS", 90) +if not ( + isinstance(WHOIS_REFRESH_THRESHOLD_DAYS, int) and WHOIS_REFRESH_THRESHOLD_DAYS > 0 +): + raise ImproperlyConfigured( + "OPENWISP_CONTROLLER_WHOIS_REFRESH_THRESHOLD_DAYS must be a positive integer" + ) +ESTIMATED_LOCATION_ENABLED = get_setting("ESTIMATED_LOCATION_ENABLED", False) +if ESTIMATED_LOCATION_ENABLED and not WHOIS_ENABLED: + raise ImproperlyConfigured( + "OPENWISP_CONTROLLER_WHOIS_ENABLED must be set to True before " + "setting OPENWISP_CONTROLLER_ESTIMATED_LOCATION_ENABLED to True." + ) diff --git a/openwisp_controller/config/static/whois/css/whois.css b/openwisp_controller/config/static/whois/css/whois.css new file mode 100644 index 000000000..2530cf296 --- /dev/null +++ b/openwisp_controller/config/static/whois/css/whois.css @@ -0,0 +1,110 @@ +:root { + --whois-bg-light: #f9fafb; + --whois-text-muted: rgba(0, 0, 0, 0.45); + --whois-border-radius: 5px; + --whois-padding: 15px; +} + +.whois-table, +.whois { + margin-top: 1.25em; + max-width: 500px; + width: 100%; + border-radius: var(--whois-border-radius); +} +.whois-table { + border-spacing: 0; + box-shadow: 0 0 0 1px #ccc; + border-style: none; +} +.whois-table th, +.whois-table td { + padding: var(--whois-padding) !important; + border: none; +} +.whois-table tr:first-child { + background-color: var(--whois-bg-light); +} +.whois-table tr:not(:first-child) { + background-color: white; +} +.whois-table th:first-of-type > div { + display: flex; + align-items: center; +} +/* For rounding the individual corners */ +.whois-table th:first-of-type { + border-top-left-radius: var(--whois-border-radius); + border-right: 1px solid var(--border-color); + border-bottom: 1px solid var(--border-color); +} +.whois-table th:last-of-type { + border-top-right-radius: var(--whois-border-radius); + border-bottom: 1px solid var(--border-color); +} +.whois-table tr:last-child td:first-child { + border-bottom-left-radius: var(--whois-border-radius); + border-right: 1px solid var(--border-color); +} +.whois-table tr:last-child td:last-child { + border-bottom-right-radius: var(--whois-border-radius); +} +.whois-table .ow-info-icon { + width: 1rem; + height: 1rem; + mask-repeat: no-repeat; + mask-position: center; + mask-size: contain; + display: inline-block; + flex-shrink: 0; + background: #777; + margin-left: 1em; +} +.whois-table .ow-info-icon:hover { + background: #007cba; +} + +.whois { + background: var(--whois-bg-light); +} +.whois summary { + background: white; + border-radius: var(--whois-border-radius); + list-style: none; + display: flex; + align-items: center; + color: var(--whois-text-muted) !important; + font-weight: 600; + cursor: pointer; +} +.whois summary > div { + display: flex; + width: 100%; + justify-content: space-between; + margin-left: 1.4em; +} +.whois summary::-webkit-details-marker { + display: none; +} +.whois summary .mg-arrow { + transform: scale(1.75); + display: inline; +} +.whois > div { + padding: var(--whois-padding); + font-size: 1em; +} +.whois .additional-text { + display: block; +} +.whois .additional-text + .additional-text { + margin-top: 5px; +} +.whois-globe { + flex-shrink: 0; + mask: url(../images/whois_globe.svg) no-repeat center; + -webkit-mask: url(../images/whois_globe.svg) no-repeat center; + width: 1.1rem; + height: 1.1rem; + background-color: var(--whois-text-muted); +} diff --git a/openwisp_controller/config/static/whois/images/whois_globe.svg b/openwisp_controller/config/static/whois/images/whois_globe.svg new file mode 100644 index 000000000..74116f1a6 --- /dev/null +++ b/openwisp_controller/config/static/whois/images/whois_globe.svg @@ -0,0 +1,3 @@ + + + diff --git a/openwisp_controller/config/static/whois/js/whois.js b/openwisp_controller/config/static/whois/js/whois.js new file mode 100644 index 000000000..7b8c20be7 --- /dev/null +++ b/openwisp_controller/config/static/whois/js/whois.js @@ -0,0 +1,62 @@ +"use strict"; + +var gettext = + window.gettext || + function (word) { + return word; + }; +// For XSS prevention +function escapeHtml(text) { + if (text == null) return ""; + const div = document.createElement("div"); + div.textContent = text; + return div.innerHTML; +} +django.jQuery(function ($) { + const $addForm = $(".add-form"); + const $deviceForm = $("#device_form"); + + if ( + $addForm.length || + !$deviceForm.length || + typeof deviceWHOISDetails === "undefined" + ) { + return; + } + const $parentDiv = $("#overview-group .field-last_ip div:last"); + const tooltipText = gettext( + "This is the Organization associated with registered ASN", + ); + + $parentDiv.after( + ` + + + + + + + + +
+
+ ${gettext("ISP")} + +
+ 		${gettext("Country")}
${escapeHtml(deviceWHOISDetails.isp)}${escapeHtml(deviceWHOISDetails.address.country)}
+
+ + +
+ ${gettext("Additional Details")} +
+ +
+ ${gettext("ASN")}: ${escapeHtml(deviceWHOISDetails.asn)} + ${gettext("Timezone")}: ${escapeHtml(deviceWHOISDetails.timezone)} + ${gettext("Address")}: ${escapeHtml(deviceWHOISDetails.formatted_address)} + ${gettext("CIDR")}: ${escapeHtml(deviceWHOISDetails.cidr)} +
+
`, + ); +}); diff --git a/openwisp_controller/config/templates/admin/config/change_form.html b/openwisp_controller/config/templates/admin/config/change_form.html index 5e0c993a0..c4099580e 100644 --- a/openwisp_controller/config/templates/admin/config/change_form.html +++ b/openwisp_controller/config/templates/admin/config/change_form.html @@ -12,6 +12,9 @@ {% endblock %} {% block extrahead %} + {% if device_whois_details %} + {{ device_whois_details|json_script:"device-whois-details" }} + {% endif %} {{ block.super}} {% endblock extrahead %} diff --git a/openwisp_controller/config/tests/test_admin.py b/openwisp_controller/config/tests/test_admin.py index 3be6f448a..aa9e0ebca 100644 --- a/openwisp_controller/config/tests/test_admin.py +++ b/openwisp_controller/config/tests/test_admin.py @@ -2267,14 +2267,17 @@ def _verify_template_queries(self, config, count): self.assertEqual(response.status_code, 200) # ensuring queries are consistent for different number of templates + @patch.object(app_settings, "WHOIS_CONFIGURED", False) def test_templates_fetch_queries_1(self): config = self._create_config(organization=self._get_org()) self._verify_template_queries(config, 1) + @patch.object(app_settings, "WHOIS_CONFIGURED", False) def test_templates_fetch_queries_5(self): config = self._create_config(organization=self._get_org()) self._verify_template_queries(config, 5) + @patch.object(app_settings, "WHOIS_CONFIGURED", False) def test_templates_fetch_queries_10(self): config = self._create_config(organization=self._get_org()) self._verify_template_queries(config, 10) diff --git a/openwisp_controller/config/tests/test_api.py b/openwisp_controller/config/tests/test_api.py index 068528610..fe12fe2cf 100644 --- a/openwisp_controller/config/tests/test_api.py +++ b/openwisp_controller/config/tests/test_api.py @@ -250,7 +250,9 @@ def test_device_create_with_devicegroup(self): def test_device_list_api(self): device = self._create_device() path = reverse("config_api:device_list") - with self.assertNumQueries(3): + with patch.object( + app_settings, "WHOIS_CONFIGURED", False + ), self.assertNumQueries(3): r = self.client.get(path) self.assertEqual(r.status_code, 200) with self.subTest("device list should show most recent first"): @@ -396,7 +398,9 @@ def test_device_filter_templates(self): def test_device_detail_api(self): d1 = self._create_device() path = reverse("config_api:device_detail", args=[d1.pk]) - with self.assertNumQueries(2): + with patch.object( + app_settings, "WHOIS_CONFIGURED", False + ), self.assertNumQueries(2): r = self.client.get(path) self.assertEqual(r.status_code, 200) self.assertEqual(r.data["config"], None) @@ -406,7 +410,9 @@ def test_device_detail_config_api(self): d1 = self._create_device() self._create_config(device=d1) path = reverse("config_api:device_detail", args=[d1.pk]) - with self.assertNumQueries(3): + with patch.object( + app_settings, "WHOIS_CONFIGURED", False + ), self.assertNumQueries(3): r = self.client.get(path) self.assertEqual(r.status_code, 200) self.assertNotEqual(r.data["config"], None) diff --git a/openwisp_controller/config/tests/test_device.py b/openwisp_controller/config/tests/test_device.py index eb053f237..9603606d3 100644 --- a/openwisp_controller/config/tests/test_device.py +++ b/openwisp_controller/config/tests/test_device.py @@ -524,11 +524,23 @@ def test_device_field_changed_checks(self): device.management_ip = "10.0.0.1" device.group_id = device_group.id device.organization_id = self._create_org().id + # assigning a random ip to last_ip + device.last_ip = "172.217.22.14" # Another query is generated due to "config.set_status_modified" # on name change with self.assertNumQueries(3): device._check_changed_fields() + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_changed_checked_fields_no_duplicates(self): + """Ensure `_changed_checked_fields` contains `last_ip` only once. + This prevents duplicates if __init__ is invoked multiple times. + """ + device = Device() + # Simulate __init__ being called again on the same instance + device.__init__() + self.assertEqual(device._changed_checked_fields.count("last_ip"), 1) + def test_exceed_organization_device_limit(self): org = self._get_org() org.config_limits.device_limit = 1 diff --git a/openwisp_controller/config/whois/__init__.py b/openwisp_controller/config/whois/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/config/whois/commands.py b/openwisp_controller/config/whois/commands.py new file mode 100644 index 000000000..dafbe92e8 --- /dev/null +++ b/openwisp_controller/config/whois/commands.py @@ -0,0 +1,66 @@ +from django.core.management.base import CommandError +from django.db.models import OuterRef, Subquery +from swapper import load_model + +from openwisp_controller.config import settings as app_settings + + +def clear_last_ip_command(stdout, stderr, interactive=True): + """ + Clears the last IP address (if set) for active devices without WHOIS records + across all organizations. + """ + if not app_settings.WHOIS_CONFIGURED: + raise CommandError( + "WHOIS lookup is not configured. Set " + "OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT " + "and OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY to enable this command." + ) + + Device = load_model("config", "Device") + WHOISInfo = load_model("config", "WHOISInfo") + + if interactive: # pragma: no cover + message = ["\n"] + message.append( + "This will clear the last IP of any active device which doesn't " + "have WHOIS info yet!\n" + ) + message.append( + "Are you sure you want to do this?\n\n" + "Type 'yes' to continue, or 'no' to cancel: " + ) + if input("".join(message)).lower() != "yes": + raise CommandError("Operation cancelled by user.") + + devices = ( + Device.objects.filter(_is_deactivated=False).select_related( + "organization__config_settings" + ) + # include the FK field 'organization' in .only() so the related + # `organization__config_settings` traversal is not deferred + .only("last_ip", "organization", "key") + ) + # Filter out devices that have WHOIS information for their last IP + devices = devices.exclude(last_ip=None).exclude( + last_ip__in=Subquery( + WHOISInfo.objects.filter(ip_address=OuterRef("last_ip")).values( + "ip_address" + ) + ), + ) + # We cannot use a queryset-level update here because it bypasses model save() + # and signals, which are required to properly invalidate related caches + # (e.g. DeviceChecksumView.get_device). To ensure correct behavior and + # future compatibility, each device is saved individually. + updated_devices = 0 + for device in devices.iterator(): + device.last_ip = None + device.save(update_fields=["last_ip"]) + updated_devices += 1 + if updated_devices: + stdout.write( + f"Cleared the last IP addresses for {updated_devices} active device(s)." + ) + else: + stdout.write("No active devices with last IP to clear.") diff --git a/openwisp_controller/config/whois/handlers.py b/openwisp_controller/config/whois/handlers.py new file mode 100644 index 000000000..6a785c4b6 --- /dev/null +++ b/openwisp_controller/config/whois/handlers.py @@ -0,0 +1,29 @@ +from django.db.models.signals import post_delete, post_save +from swapper import load_model + +from .. import settings as app_settings + + +def connect_whois_handlers(): + if not app_settings.WHOIS_CONFIGURED: + return + + Device = load_model("config", "Device") + WHOISInfo = load_model("config", "WHOISInfo") + OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + + post_delete.connect( + WHOISInfo.device_whois_info_delete_handler, + sender=Device, + dispatch_uid="device.delete_whois_info", + ) + post_save.connect( + WHOISInfo.invalidate_org_settings_cache, + sender=OrganizationConfigSettings, + dispatch_uid="invalidate_org_config_cache_on_org_config_save", + ) + post_delete.connect( + WHOISInfo.invalidate_org_settings_cache, + sender=OrganizationConfigSettings, + dispatch_uid="invalidate_org_config_cache_on_org_config_delete", + ) diff --git a/openwisp_controller/config/whois/mixins.py b/openwisp_controller/config/whois/mixins.py new file mode 100644 index 000000000..3cd70949b --- /dev/null +++ b/openwisp_controller/config/whois/mixins.py @@ -0,0 +1,20 @@ +from .. import settings as app_settings +from .serializers import WHOISSerializer + + +class WHOISMixin: + """Mixin to add WHOIS information to the device representation.""" + + serializer_class = WHOISSerializer + + def get_whois_info(self, obj): + whois_obj = obj.whois_service.get_device_whois_info() + if not whois_obj: + return None + return self.serializer_class(whois_obj).data + + def to_representation(self, obj): + data = super().to_representation(obj) + if app_settings.WHOIS_CONFIGURED and obj.whois_service.is_whois_enabled: + data["whois_info"] = self.get_whois_info(obj) + return data diff --git a/openwisp_controller/config/whois/serializers.py b/openwisp_controller/config/whois/serializers.py new file mode 100644 index 000000000..028fc9426 --- /dev/null +++ b/openwisp_controller/config/whois/serializers.py @@ -0,0 +1,16 @@ +from rest_framework import serializers +from swapper import load_model + +WHOISInfo = load_model("config", "WHOISInfo") + + +class WHOISSerializer(serializers.ModelSerializer): + """ + Serializer for detailed representation of WHOIS model. + """ + + address = serializers.JSONField() + + class Meta: + model = WHOISInfo + fields = "__all__" diff --git a/openwisp_controller/config/whois/service.py b/openwisp_controller/config/whois/service.py new file mode 100644 index 000000000..d6f72d016 --- /dev/null +++ b/openwisp_controller/config/whois/service.py @@ -0,0 +1,363 @@ +from datetime import timedelta +from ipaddress import ip_address as ip_addr + +import requests +from celery import current_app +from django.contrib.gis.geos import Point +from django.core.cache import cache +from django.db import transaction +from django.utils import timezone +from geoip2 import errors +from geoip2 import webservice as geoip2_webservice +from swapper import load_model + +from openwisp_controller.config import settings as app_settings + +from .tasks import fetch_whois_details +from .utils import EXCEPTION_MESSAGES, send_whois_task_notification + + +class WHOISService: + """ + A handler class for managing the WHOIS functionality. + """ + + def __init__(self, device): + self.device = device + + @staticmethod + def get_geoip_client(): + """ + Used to get a GeoIP2 web service client instance. + Host is based on the db that is used to fetch the details. + As we are using GeoLite2, 'geolite.info' host is used. + Refer: https://geoip2.readthedocs.io/en/latest/#sync-web-service-example + """ + return geoip2_webservice.Client( + account_id=app_settings.WHOIS_GEOIP_ACCOUNT, + license_key=app_settings.WHOIS_GEOIP_KEY, + host="geolite.info", + ) + + @staticmethod + def get_cache_key(org_id): + """ + Used to get cache key for caching org settings of a device. + """ + return f"organization_config_{org_id}" + + @staticmethod + def is_valid_public_ip_address(ip): + """ + Check if given IP address is a valid public IP address. + """ + try: + return ip and ip_addr(ip).is_global + except ValueError: + # ip_address() from the stdlib raises ValueError for malformed strings + return False + + @staticmethod + def _get_whois_info_from_db(ip_address): + """ + For getting existing WHOISInfo for given IP from db if present. + """ + WHOISInfo = load_model("config", "WHOISInfo") + + return WHOISInfo.objects.filter(ip_address=ip_address) + + @staticmethod + def is_older(dt): + """ + Check if given datetime is older than the refresh threshold. + Raises TypeError if datetime is naive (not timezone-aware). + """ + if not timezone.is_aware(dt): + raise TypeError("datetime must be timezone-aware") + return (timezone.now() - dt) >= timedelta( + days=app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + ) + + @staticmethod + def get_org_config_settings(org_id): + """ + Retrieve and cache organization-specific configuration settings. + + Returns a "read-only" OrganizationConfigSettings instance for the + given organization. + If no settings exist for the organization, returns an empty instance to allow + fallback to global defaults. + + OrganizationConfigSettings are cached for performance, using the same timeout + as DeviceChecksumView for consistency. + """ + OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + Config = load_model("config", "Config") + + cache_key = WHOISService.get_cache_key(org_id=org_id) + org_settings = cache.get(cache_key) + if org_settings is None: + try: + org_settings = OrganizationConfigSettings.objects.get( + organization=org_id + ) + except OrganizationConfigSettings.DoesNotExist: + # If organization settings do not exist, fall back to global setting + org_settings = OrganizationConfigSettings() + cache.set( + cache_key, + org_settings, + timeout=Config._CHECKSUM_CACHE_TIMEOUT, + ) + return org_settings + + @staticmethod + def check_estimated_location_enabled(org_id): + if not org_id: + return False + if not app_settings.WHOIS_CONFIGURED: + return False + org_settings = WHOISService.get_org_config_settings(org_id=org_id) + return org_settings.estimated_location_enabled + + @property + def is_whois_enabled(self): + """ + Check if the WHOIS lookup feature is enabled. + """ + if not app_settings.WHOIS_CONFIGURED: + return False + org_settings = self.get_org_config_settings(org_id=self.device.organization.pk) + return org_settings.whois_enabled + + @property + def is_estimated_location_enabled(self): + """ + Check if the Estimated location feature is enabled. + """ + if not app_settings.WHOIS_CONFIGURED: + return False + org_settings = self.get_org_config_settings(org_id=self.device.organization.pk) + return org_settings.estimated_location_enabled + + def process_whois_details(self, ip_address): + """ + Fetch WHOIS details for a given IP address and return only + the relevant information. + """ + ip_client = self.get_geoip_client() + try: + data = ip_client.city(ip_address=ip_address) + # Catching all possible exceptions raised by the geoip2 client + # and raising them with appropriate messages to be handled by the task + # retry mechanism. + except ( + errors.AddressNotFoundError, + errors.AuthenticationError, + errors.OutOfQueriesError, + errors.PermissionRequiredError, + ) as e: + exc_type = type(e) + message = EXCEPTION_MESSAGES.get(exc_type) + if exc_type is errors.AddressNotFoundError: + message = message.format(ip_address=ip_address) + raise exc_type(message) from e + except requests.RequestException: + raise + else: + # The attributes are always present in the response, + # but they can be None, so added fallbacks. + address = { + "city": data.city.name or "", + "country": data.country.name or "", + "continent": data.continent.name or "", + "postal": str(data.postal.code or ""), + } + # Coordinates may be None in WHOIS response + # WHOISInfo.timezone is a non-nullable CharField, so store empty + # string when missing to avoid IntegrityError on save. + time_zone, coordinates = "", None + if location := data.location: + if location.latitude is not None and location.longitude is not None: + coordinates = Point( + location.longitude, location.latitude, srid=4326 + ) + time_zone = location.time_zone or "" + return { + "isp": str(data.traits.autonomous_system_organization or ""), + "asn": str(data.traits.autonomous_system_number or ""), + "timezone": time_zone, + "address": address, + "coordinates": coordinates, + "cidr": str(data.traits.network or ""), + "ip_address": ip_address, + } + + def _need_whois_lookup(self, new_ip): + """ + This is used to determine if the WHOIS lookup should be triggered + when the device is saved. + + The lookup is not triggered if: + - The new IP address is None or it is a private IP address. + - The WHOIS information of new ip is present and is not older than + X days (defined by "WHOIS_REFRESH_THRESHOLD_DAYS"). + - WHOIS is disabled in the organization settings. (query from db) + """ + # Check cheap conditions first before hitting the database + if not self.is_whois_enabled: + return False + if not self.is_valid_public_ip_address(new_ip): + return False + whois_obj = self._get_whois_info_from_db(ip_address=new_ip).first() + if whois_obj and not self.is_older(whois_obj.modified): + return False + return True + + def _need_estimated_location_management(self, new_ip): + """ + Used to determine if Estimated locations need to be created/updated + or not during WHOIS lookup. + """ + if not self.is_whois_enabled: + return False + if not self.is_valid_public_ip_address(new_ip): + return False + if not self.is_estimated_location_enabled: + return False + return True + + def trigger_estimated_location_task(self, ip_address): + """Helper method to trigger the estimated location task.""" + current_app.send_task( + "whois_estimated_location_task", + kwargs={"device_pk": self.device.pk, "ip_address": ip_address}, + ) + + def get_device_whois_info(self): + """ + If the WHOIS lookup feature is enabled and the device ``last_ip`` + is a public IP, it fetches WHOIS information for the network device. + """ + ip_address = self.device.last_ip + if not (self.is_valid_public_ip_address(ip_address) and self.is_whois_enabled): + return None + return self._get_whois_info_from_db(ip_address=ip_address).first() + + def process_ip_data_and_location(self, force_lookup=False): + """ + Trigger WHOIS lookup based on the conditions of `_need_whois_lookup` + and also manage estimated locations based on the conditions of + `_need_estimated_location_management`. + Tasks are triggered on commit to ensure redundant data is not created. + """ + new_ip = self.device.last_ip + initial_ip = self.device._initial_last_ip + if force_lookup or self._need_whois_lookup(new_ip): + transaction.on_commit( + lambda: fetch_whois_details.delay( + device_pk=self.device.pk, + initial_ip_address=initial_ip, + ) + ) + # To handle the case when WHOIS already exists as in that case + # WHOIS lookup is not triggered but we still need to + # manage estimated locations. + elif self._need_estimated_location_management(new_ip): + transaction.on_commit( + lambda: self.trigger_estimated_location_task( + ip_address=new_ip, + ) + ) + + def update_whois_info(self): + """ + Update existing WHOIS data for the device + when the data is older than + ``OPENWISP_CONTROLLER_WHOIS_REFRESH_THRESHOLD_DAYS``. + """ + ip_address = self.device.last_ip + if not self.is_valid_public_ip_address(ip_address): + return + if not self.is_whois_enabled: + return + whois_obj = WHOISService._get_whois_info_from_db(ip_address=ip_address).first() + if whois_obj and self.is_older(whois_obj.modified): + transaction.on_commit( + lambda: fetch_whois_details.delay( + device_pk=self.device.pk, + initial_ip_address=None, + ) + ) + + def _create_or_update_whois(self, whois_details, whois_instance=None): + """ + Used to update an existing WHOIS instance; else, creates a new one. + Returns the updated or created WHOIS instance along with update fields. + """ + WHOISInfo = load_model("config", "WHOISInfo") + update_fields = [] + if whois_instance: + for attr, value in whois_details.items(): + # whois_details already coerce to string most values + if getattr(whois_instance, attr) != value: + update_fields.append(attr) + setattr(whois_instance, attr, value) + # bump modified time so staleness check + # doesn't re-trigger the lookup + update_fields.append("modified") + whois_instance.modified = timezone.now() + whois_instance.save(update_fields=update_fields) + else: + whois_instance = WHOISInfo(**whois_details) + whois_instance.full_clean() + whois_instance.save(force_insert=True) + return whois_instance, update_fields + + def _create_or_update_estimated_location( + self, location_defaults, attached_devices_exists + ): + """ + Create or update estimated location for the device based on the + given location defaults. + """ + Location = load_model("geo", "Location") + DeviceLocation = load_model("geo", "DeviceLocation") + + if not (device_location := getattr(self.device, "devicelocation", None)): + device_location = DeviceLocation(content_object=self.device) + + current_location = device_location.location + + if not current_location or ( + attached_devices_exists and current_location.is_estimated + ): + with transaction.atomic(): + current_location = Location(**location_defaults, is_estimated=True) + current_location.full_clean() + current_location.save(_set_estimated=True) + device_location.location = current_location + device_location.full_clean() + device_location.save() + send_whois_task_notification( + device=self.device, + notify_type="estimated_location_created", + actor=current_location, + ) + elif current_location.is_estimated: + update_fields = [] + for attr, value in location_defaults.items(): + if getattr(current_location, attr) != value: + setattr(current_location, attr, value) + update_fields.append(attr) + if update_fields: + with transaction.atomic(): + current_location.save( + update_fields=update_fields, _set_estimated=True + ) + send_whois_task_notification( + device=self.device, + notify_type="estimated_location_updated", + actor=current_location, + ) + return current_location diff --git a/openwisp_controller/config/whois/tasks.py b/openwisp_controller/config/whois/tasks.py new file mode 100644 index 000000000..78955600e --- /dev/null +++ b/openwisp_controller/config/whois/tasks.py @@ -0,0 +1,130 @@ +import logging + +from celery import shared_task +from django.core.cache import cache +from django.db import transaction +from geoip2 import errors +from swapper import load_model + +from openwisp_utils.tasks import OpenwispCeleryTask + +from .. import settings as app_settings +from .utils import send_whois_task_notification + +logger = logging.getLogger(__name__) + + +class WHOISCeleryRetryTask(OpenwispCeleryTask): + """ + Base class for OpenWISP Celery tasks with retry support on failure. + """ + + # this is the exception related to networking errors + # that should trigger a retry of the task. + autoretry_for = (errors.HTTPError,) + + def on_success(self, retval, task_id, args, kwargs): + """Mark the task as successfully completed.""" + task_key = f"{self.name}_last_operation" + cache.set(task_key, "success", None) + return super().on_success(retval, task_id, args, kwargs) + + def on_failure(self, exc, task_id, args, kwargs, einfo): + """ + Notify the user about the failure of the WHOIS task. + + Notifications are sent only once when task fails for the first time. + Subsequent failures do not trigger notifications until a successful run occurs. + """ + device_pk = kwargs.get("device_pk") or (args[0] if args else None) + if device_pk is not None: + # All exceptions are treated globally to prevent notification spam. + # The cache key is global (not per-device) to avoid spamming admins + # with multiple notifications for the same recurring issue. + task_key = f"{self.name}_last_operation" + last_operation = cache.get(task_key) + if last_operation != "errored": + cache.set(task_key, "errored", None) + send_whois_task_notification( + device=device_pk, notify_type="whois_device_error" + ) + logger.error(f"WHOIS lookup failed. Details: {exc}") + return super().on_failure(exc, task_id, args, kwargs, einfo) + + +# device_pk is used when task fails to report for which device failure occurred +@shared_task( + bind=True, + base=WHOISCeleryRetryTask, + **app_settings.API_TASK_RETRY_OPTIONS, +) +def fetch_whois_details(self, device_pk, initial_ip_address): + """ + Fetches the WHOIS details of the given IP address + and creates/updates the WHOIS record. + """ + Device = load_model("config", "Device") + WHOISInfo = load_model("config", "WHOISInfo") + + try: + device = Device.objects.select_related("devicelocation").get(pk=device_pk) + except Device.DoesNotExist: + logger.warning(f"Device {device_pk} not found, skipping WHOIS lookup") + return + new_ip_address = device.last_ip + whois_service = device.whois_service + # If there is existing WHOIS older record then it needs to be updated + whois_obj = WHOISInfo.objects.filter(ip_address=new_ip_address).first() + if whois_obj and not whois_service.is_older(whois_obj.modified): + return + # WARNING: execute HTTP requests before transaction lock is acquired + fetched_details = whois_service.process_whois_details(new_ip_address) + + with transaction.atomic(): + whois_obj, update_fields = whois_service._create_or_update_whois( + fetched_details, whois_obj + ) + logger.info(f"Successfully fetched WHOIS details for {new_ip_address}.") + if initial_ip_address: + transaction.on_commit( + # execute synchronously as we're already in a background task + lambda: delete_whois_record(ip_address=initial_ip_address) + ) + if not device._get_organization__config_settings().estimated_location_enabled: + return + # the estimated location task should not run if old record is updated + # and location related fields are not updated + device_location = getattr(device, "devicelocation", None) + if ( + device_location + and device_location.location + and update_fields + and not any(i in update_fields for i in ["address", "coordinates"]) + ): + return + transaction.on_commit( + lambda: whois_service.trigger_estimated_location_task( + ip_address=new_ip_address, + ) + ) + + +@shared_task +def delete_whois_record(ip_address, force=False): + """ + Deletes the WHOIS record for the device's last IP address. + This is used when the device is deleted or its last IP address is changed. + 'force' parameter is used to delete the record without checking for linked devices. + """ + Device = load_model("config", "Device") + WHOISInfo = load_model("config", "WHOISInfo") + queryset = WHOISInfo.objects.filter(ip_address=ip_address) + if force: + queryset.delete() + else: + if ( + not Device.objects.filter(_is_deactivated=False) + .filter(last_ip=ip_address) + .exists() + ): + queryset.delete() diff --git a/openwisp_controller/config/whois/tests/__init__.py b/openwisp_controller/config/whois/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/config/whois/tests/tests.py b/openwisp_controller/config/whois/tests/tests.py new file mode 100644 index 000000000..c3622ec96 --- /dev/null +++ b/openwisp_controller/config/whois/tests/tests.py @@ -0,0 +1,1167 @@ +import copy +import importlib +from datetime import timedelta +from io import StringIO +from unittest import mock +from uuid import uuid4 + +from django.conf import settings +from django.contrib.gis.geos import Point +from django.contrib.staticfiles.testing import StaticLiveServerTestCase +from django.core.cache import cache +from django.core.exceptions import ImproperlyConfigured, ValidationError +from django.core.management import CommandError, call_command +from django.db.models.signals import post_delete, post_save +from django.test import TestCase, TransactionTestCase, override_settings, tag +from django.urls import reverse +from django.utils import timezone +from geoip2 import errors +from requests.exceptions import RequestException +from selenium.common.exceptions import UnexpectedAlertPresentException +from selenium.webdriver.common.by import By +from swapper import load_model + +from openwisp_utils.tests import SeleniumTestMixin + +from ....tests.utils import TestAdminMixin +from ... import settings as app_settings +from ..handlers import connect_whois_handlers +from ..service import WHOISService +from ..tasks import delete_whois_record, fetch_whois_details +from ..utils import get_whois_info, send_whois_task_notification +from .utils import CreateWHOISMixin, WHOISTransactionMixin + +Config = load_model("config", "Config") +Device = load_model("config", "Device") +WHOISInfo = load_model("config", "WHOISInfo") +Notification = load_model("openwisp_notifications", "Notification") +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + +MODIFIED_CACHE = copy.deepcopy(settings.CACHES) +# add key_prefix to avoid conflicts in parallel tests +MODIFIED_CACHE["default"]["KEY_PREFIX"] = "whois_failure" + + +def _notification_qs(): + return Notification.objects.all() + + +# SESSION_ENGINE set to DB to avoid conflicts in parallel tests +@override_settings(SESSION_ENGINE="django.contrib.sessions.backends.db") +class TestWHOIS(CreateWHOISMixin, TestAdminMixin, TestCase): + # Signals are connected when apps are loaded, + # and if WHOIS is Configured all related WHOIS + # handlers are also connected. Thus we need to + # disconnect them. + def _disconnect_signals(self): + post_delete.disconnect( + WHOISInfo.device_whois_info_delete_handler, + sender=Device, + dispatch_uid="device.delete_whois_info", + ) + post_save.disconnect( + WHOISInfo.invalidate_org_settings_cache, + sender=OrganizationConfigSettings, + dispatch_uid="invalidate_org_config_cache_on_org_config_save", + ) + post_delete.disconnect( + WHOISInfo.invalidate_org_settings_cache, + sender=OrganizationConfigSettings, + dispatch_uid="invalidate_org_config_cache_on_org_config_delete", + ) + + @override_settings( + OPENWISP_CONTROLLER_WHOIS_ENABLED=False, + OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT=None, + OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY=None, + ) + def test_whois_configuration_setting(self): + self._disconnect_signals() + self.addCleanup(importlib.reload, app_settings) + # ensure organization exists for admin page checks + org = self._get_org() + # reload app_settings to apply the overridden settings + importlib.reload(app_settings) + + with self.subTest("Test Signals not connected when WHOIS_CONFIGURED is False"): + # should not connect any handlers since WHOIS_CONFIGURED is False + connect_whois_handlers() + assert not any( + "device.delete_whois_info" in str(r[0]) for r in post_delete.receivers + ) + assert not any( + "invalidate_org_config_cache_on_org_config_save" in str(r[0]) + for r in post_save.receivers + ) + assert not any( + "invalidate_org_config_cache_on_org_config_delete" in str(r[0]) + for r in post_delete.receivers + ) + + with self.subTest( + "Test WHOIS field hidden on admin when WHOIS_CONFIGURED is False" + ): + self._login() + url = reverse( + "admin:openwisp_users_organization_change", + args=[org.pk], + ) + response = self.client.get(url) + self.assertNotContains(response, 'name="config_settings-0-whois_enabled"') + + with self.subTest( + "Test ImproperlyConfigured raised when WHOIS_CONFIGURED is False" + ): + with override_settings(OPENWISP_CONTROLLER_WHOIS_ENABLED=True): + with self.assertRaises(ImproperlyConfigured): + # reload app_settings to apply the overridden settings + importlib.reload(app_settings) + + with override_settings( + OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT="test_account_id", + OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY="test_license_key", + ): + importlib.reload(app_settings) + with self.subTest( + "Test WHOIS_CONFIGURED is True when both settings are set" + ): + self.assertTrue(app_settings.WHOIS_CONFIGURED) + + with self.subTest("Test Signals connected when WHOIS_CONFIGURED is True"): + connect_whois_handlers() + + assert any( + "device.delete_whois_info" in str(r[0]) + for r in post_delete.receivers + ) + assert any( + "invalidate_org_config_cache_on_org_config_save" in str(r[0]) + for r in post_save.receivers + ) + assert any( + "invalidate_org_config_cache_on_org_config_delete" in str(r[0]) + for r in post_delete.receivers + ) + + with self.subTest( + "Test WHOIS field visible on admin when WHOIS_CONFIGURED is True" + ): + self._login() + url = reverse( + "admin:openwisp_users_organization_change", + args=[org.pk], + ) + response = self.client.get(url) + self.assertContains(response, 'name="config_settings-0-whois_enabled"') + + def test_is_older_requires_timezone_aware(self): + """Verify is_older raises TypeError for naive datetimes.""" + naive_dt = timezone.now().replace(tzinfo=None) + with self.assertRaises(TypeError): + WHOISService.is_older(naive_dt) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_whois_enabled(self): + OrganizationConfigSettings.objects.all().delete() + device = self._create_device() + + with self.subTest( + "Test WHOIS fallback when Organization settings do not exist" + ): + self.assertEqual( + device.whois_service.is_whois_enabled, app_settings.WHOIS_ENABLED + ) + org_settings_obj = OrganizationConfigSettings( + organization=self._get_org(), whois_enabled=True + ) + + with self.subTest("Test WHOIS not configured does not allow enabling WHOIS"): + with mock.patch.object( + app_settings, "WHOIS_CONFIGURED", False + ), self.assertRaises(ValidationError) as context_manager: + org_settings_obj.full_clean() + self.assertEqual( + context_manager.exception.message_dict["whois_enabled"][0], + "WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY must be set " + + "before enabling WHOIS feature.", + ) + + with self.subTest("Test setting WHOIS enabled to True"): + org_settings_obj.whois_enabled = True + org_settings_obj.full_clean() + org_settings_obj.save(update_fields=["whois_enabled"]) + org_settings_obj.refresh_from_db(fields=["whois_enabled"]) + self.assertTrue(org_settings_obj.whois_enabled) + + with self.subTest("Test setting WHOIS enabled to False"): + org_settings_obj.whois_enabled = False + org_settings_obj.full_clean() + org_settings_obj.save(update_fields=["whois_enabled"]) + org_settings_obj.refresh_from_db(fields=["whois_enabled"]) + self.assertFalse(org_settings_obj.whois_enabled) + + with self.subTest( + "Test setting WHOIS enabled to None fallbacks to global setting" + ): + # reload app_settings to ensure latest settings are applied + importlib.reload(app_settings) + org_settings_obj.whois_enabled = None + org_settings_obj.full_clean() + org_settings_obj.save(update_fields=["whois_enabled"]) + org_settings_obj.refresh_from_db(fields=["whois_enabled"]) + self.assertEqual( + org_settings_obj.whois_enabled, + app_settings.WHOIS_ENABLED, + ) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_whois_details_device_api(self): + """ + Test the WHOIS details API endpoint. + """ + whois_obj = self._create_whois_info() + device = self._create_device(last_ip=whois_obj.ip_address) + self._login() + + with self.subTest( + "Device List API does not have whois_info (removed to avoid N+1)" + ): + response = self.client.get(reverse("config_api:device_list")) + self.assertEqual(response.status_code, 200) + self.assertNotIn("whois_info", response.data["results"][0]) + + with self.subTest( + "Device Detail API has whois_info when WHOIS_CONFIGURED is True" + ): + response = self.client.get( + reverse("config_api:device_detail", args=[device.pk]) + ) + self.assertEqual(response.status_code, 200) + self.assertIn("whois_info", response.data) + api_whois_info = response.data["whois_info"] + self.assertEqual(api_whois_info["isp"], whois_obj.isp) + self.assertEqual(api_whois_info["cidr"], whois_obj.cidr) + self.assertEqual(api_whois_info["asn"], whois_obj.asn) + self.assertEqual(api_whois_info["timezone"], whois_obj.timezone) + self.assertEqual(api_whois_info["address"], whois_obj.address) + + with self.subTest( + "Device List API does not have whois_info when no WHOIS Info exists" + ): + device.last_ip = "172.217.22.24" + device.save() + device.refresh_from_db() + response = self.client.get(reverse("config_api:device_list")) + self.assertEqual(response.status_code, 200) + self.assertNotIn("whois_info", response.data["results"][0]) + + with self.subTest( + "Device Detail API has whois_info as None when no WHOIS Info exists" + ): + response = self.client.get( + reverse("config_api:device_detail", args=[device.pk]) + ) + self.assertEqual(response.status_code, 200) + self.assertIn("whois_info", response.data) + self.assertIsNone(response.data["whois_info"]) + + with mock.patch.object(app_settings, "WHOIS_CONFIGURED", False): + with self.subTest( + "Device List API has no whois_info when WHOIS_CONFIGURED is False" + ): + response = self.client.get(reverse("config_api:device_list")) + self.assertEqual(response.status_code, 200) + self.assertNotIn("whois_info", response.data["results"][0]) + with self.subTest( + "Device Detail API has no whois_info when WHOIS_CONFIGURED is False" + ): + response = self.client.get( + reverse("config_api:device_detail", args=[device.pk]) + ) + self.assertEqual(response.status_code, 200) + self.assertNotIn("whois_info", response.data) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_device_list_api_whois_no_nplus1(self): + """ + Test that WHOIS info doesn't cause N+1 queries in device list API. + Should use constant queries regardless of device count. + """ + self._login() + path = reverse("config_api:device_list") + # Create 3 devices with WHOIS info + for i in range(3): + device = self._create_device( + name=f"device{i}", + mac_address=f"00:11:22:33:44:{i:02x}", + last_ip=f"172.217.22.{i + 1}", + ) + WHOISInfo.objects.create( + ip_address=device.last_ip, + isp="Test ISP", + asn="12345", + address={"city": "Test City", "country": "Test Country"}, + cidr=f"172.217.22.{i + 1}/32", + ) + with self.subTest(f"Device List API with WHOIS info: {i}"): + with self.assertNumQueries(4): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data["results"]), i + 1) + for result in response.data["results"]: + self.assertNotIn("whois_info", result) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_last_ip_management_command(self): + out = StringIO() + device = self._create_device(last_ip="172.217.22.11") + args = ["--noinput"] + call_command("clear_last_ip", *args, stdout=out, stderr=StringIO()) + self.assertIn( + "Cleared the last IP addresses for 1 active device(s).", out.getvalue() + ) + device.refresh_from_db() + self.assertIsNone(device.last_ip) + out.seek(0) + out.truncate(0) + call_command("clear_last_ip", *args, stdout=out, stderr=StringIO()) + self.assertIn("No active devices with last IP to clear.", out.getvalue()) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_last_ip_management_command_queries(self): + out = StringIO() + self._create_device(last_ip="172.217.22.11") + self._create_device( + name="default.test.device2", + last_ip="172.217.22.12", + mac_address="11:22:33:44:55:66", + ) + self._create_device( + name="default.test.device3", + last_ip="172.217.22.13", + mac_address="22:33:44:55:66:77", + ) + self._create_device( + name="default.test.device4", mac_address="66:33:44:55:66:77" + ) + args = ["--noinput"] + # 4 queries (3 for each device's last_ip update) and 1 for fetching devices + with self.assertNumQueries(4): + call_command("clear_last_ip", *args, stdout=out, stderr=StringIO()) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_last_ip_management_command_invalidates_cache(self): + device = self._create_device(last_ip="172.217.22.11") + self._create_config(device=device) + call_command("clear_last_ip", "--noinput", stdout=StringIO()) + device.refresh_from_db() + self.assertEqual(device.last_ip, None) + # We will use the DeviceChecksumView to set the last_ip again to + # the same value to verify that the command invalidates the cache + # and the view is able to send the same IP again. + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR="172.217.22.11", + ) + self.assertEqual(response.status_code, 200) + device.refresh_from_db() + self.assertEqual(device.last_ip, "172.217.22.11") + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", False) + def test_clear_last_ip_command_not_enabled(self): + """Test that clear_last_ip command raises error when WHOIS is not configured.""" + out = StringIO() + err = StringIO() + with self.assertRaises(CommandError) as context: + call_command("clear_last_ip", "--noinput", stdout=out, stderr=err) + self.assertIn("WHOIS lookup is not configured", str(context.exception)) + + +class TestWHOISInfoModel(CreateWHOISMixin, TestCase): + def test_whois_model_fields_validation(self): + """ + Test db_constraints and validators for WHOISInfo model fields. + """ + with self.assertRaises(ValidationError): + self._create_whois_info(isp="a" * 101) + with self.assertRaises(ValidationError) as context_manager: + self._create_whois_info(ip_address="127.0.0.1") + self.assertEqual( + context_manager.exception.message_dict["ip_address"][0], + "WHOIS information cannot be created for private IP addresses.", + ) + with self.assertRaises(ValidationError): + self._create_whois_info(timezone="a" * 36) + with self.assertRaises(ValidationError) as context_manager: + self._create_whois_info(cidr="InvalidCIDR") + # Not using assertEqual here because we are adding error message raised by + # ipaddress module to the ValidationError message. + self.assertIn( + "Invalid CIDR format: 'InvalidCIDR'", + context_manager.exception.message_dict["cidr"][0], + ) + with self.assertRaises(ValidationError): + self._create_whois_info(asn="InvalidASNNumber") + # Common validation checks for longitude and latitude + coordinates_cases = [ + (150.0, 100.0, "Latitude must be between -90 and 90 degrees."), + (150.0, -100.0, "Latitude must be between -90 and 90 degrees."), + (200.0, 80.0, "Longitude must be between -180 and 180 degrees."), + (-200.0, -80.0, "Longitude must be between -180 and 180 degrees."), + ] + for longitude, latitude, expected_msg in coordinates_cases: + with self.assertRaises(ValidationError) as context_manager: + point = Point(longitude, latitude, srid=4326) + self._create_whois_info(coordinates=point) + self.assertEqual( + context_manager.exception.message_dict["coordinates"][0], + expected_msg, + ) + + +# SESSION_ENGINE set to DB to avoid conflicts in parallel tests +@override_settings(SESSION_ENGINE="django.contrib.sessions.backends.db") +class TestWHOISTransaction( + CreateWHOISMixin, WHOISTransactionMixin, TransactionTestCase +): + _WHOIS_GEOIP_CLIENT = ( + "openwisp_controller.config.whois.service.geoip2_webservice.Client" + ) + _WHOIS_TASKS_INFO_LOGGER = "openwisp_controller.config.whois.tasks.logger.info" + _WHOIS_TASKS_WARN_LOGGER = "openwisp_controller.config.whois.tasks.logger.warning" + _WHOIS_TASKS_ERR_LOGGER = "openwisp_controller.config.whois.tasks.logger.error" + _WHOIS_TASK_NAME = "openwisp_controller.config.whois.tasks.fetch_whois_details" + + def setUp(self): + super().setUp() + self.admin = self._get_admin() + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_process_whois_details_handles_missing_coordinates(self, mock_client): + """Ensure WHOIS processing tolerates missing coordinates in response.""" + connect_whois_handlers() + mocked_response = self._mocked_client_response() + # simulate missing coordinates + mocked_response.location = None + mock_client.return_value.city.return_value = mocked_response + device = self._create_device(last_ip="172.217.22.14") + whois_details = device.whois_service.process_whois_details(device.last_ip) + self.assertIsNone(whois_details.get("coordinates")) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch("openwisp_controller.config.whois.tasks.fetch_whois_details.delay") + def test_whois_task_called(self, mocked_lookup_task): + connect_whois_handlers() + self._task_called(mocked_lookup_task) + Device.objects.all().delete() + WHOISInfo.objects.all().delete() + org = self._get_org() + org.config_settings.whois_enabled = True + org.config_settings.save() + + with self.subTest("WHOIS lookup task called when last_ip is public"): + with mock.patch( + "django.core.cache.cache.get", return_value=None + ) as mocked_get, mock.patch("django.core.cache.cache.set") as mocked_set: + device = self._create_device(last_ip="172.217.22.14") + mocked_lookup_task.assert_called() + mocked_set.assert_called_once_with( + f"organization_config_{org.pk}", + org.config_settings, + timeout=Config._CHECKSUM_CACHE_TIMEOUT, + ) + mocked_get.assert_called() + mocked_lookup_task.reset_mock() + + with self.subTest( + "WHOIS lookup task called when last_ip is changed and is public" + ): + with mock.patch("django.core.cache.cache.get") as mocked_get, mock.patch( + "django.core.cache.cache.set" + ) as mocked_set: + device.last_ip = "172.217.22.10" + device.save() + device.refresh_from_db() + mocked_lookup_task.assert_called() + mocked_set.assert_not_called() + mocked_get.assert_called() + mocked_lookup_task.reset_mock() + + with self.subTest( + "WHOIS lookup task not called when last_ip has related WhoIsInfo" + ): + device.last_ip = "172.217.22.14" + self._create_whois_info(ip_address=device.last_ip) + device.save() + device.refresh_from_db() + device.organization.config_settings.refresh_from_db() + mocked_lookup_task.assert_not_called() + mocked_lookup_task.reset_mock() + + with self.subTest( + "WHOIS lookup task not called via DeviceChecksumView when " + "last_ip has related WhoIsInfo" + ): + WHOISInfo.objects.all().delete() + self._create_whois_info(ip_address=device.last_ip) + self._create_config(device=device) + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_lookup_task.assert_not_called() + mocked_lookup_task.reset_mock() + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch("openwisp_controller.config.whois.tasks.fetch_whois_details.delay") + def test_whois_multiple_orgs(self, mocked_task): + org2 = self._create_org(name="test org2", slug="test-org2") + OrganizationConfigSettings.objects.create( + organization=org2, whois_enabled=False + ) + + with self.subTest("Test task calls when device created with public last_ip"): + device1 = self._create_device( + last_ip="172.217.22.10", organization=self._get_org() + ) + mocked_task.assert_called() + mocked_task.reset_mock() + device2 = self._create_device(last_ip="172.217.22.11", organization=org2) + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest("Test task calls when last_ip is changed and is public"): + device1.last_ip = "172.217.22.12" + device1.save() + device1.refresh_from_db() + mocked_task.assert_called() + mocked_task.reset_mock() + device2.last_ip = "172.217.22.13" + device2.save() + device2.refresh_from_db() + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest("Test fetching WHOIS details"): + whois_obj1 = self._create_whois_info(ip_address=device1.last_ip) + self._create_whois_info(ip_address=device2.last_ip) + self.assertEqual(whois_obj1, device1.whois_service.get_device_whois_info()) + self.assertIsNone(device2.whois_service.get_device_whois_info()) + + with self.subTest("Test task calls in DeviceChecksumView when last_ip changes"): + # config is required for checksum view to work + self._create_config(device=device1) + self._create_config(device=device2) + response = self.client.get( + reverse("controller:device_checksum", args=[device1.pk]), + {"key": device1.key}, + REMOTE_ADDR="172.217.22.20", + ) + device1.refresh_from_db() + self.assertEqual(response.status_code, 200) + mocked_task.assert_called() + mocked_task.reset_mock() + response = self.client.get( + reverse("controller:device_checksum", args=[device2.pk]), + {"key": device2.key}, + REMOTE_ADDR="172.217.22.30", + ) + device2.refresh_from_db() + self.assertEqual(response.status_code, 200) + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest( + "Task not called via DeviceChecksumView when a device has no WHOIS record" + ): + WHOISInfo.objects.all().delete() + response = self.client.get( + reverse("controller:device_checksum", args=[device1.pk]), + {"key": device1.key}, + REMOTE_ADDR=device1.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_not_called() + mocked_task.reset_mock() + response = self.client.get( + reverse("controller:device_checksum", args=[device2.pk]), + {"key": device2.key}, + REMOTE_ADDR=device2.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_not_called() + mocked_task.reset_mock() + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_TASKS_INFO_LOGGER) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_whois_creation(self, mock_client, mock_info): + # helper function for asserting the model details with + # mocked api response + connect_whois_handlers() + + def _verify_whois_details(instance, ip_address): + mock_response = self._mocked_client_response() + self.assertEqual( + instance.isp, mock_response.traits.autonomous_system_organization + ) + self.assertEqual( + instance.asn, str(mock_response.traits.autonomous_system_number) + ) + self.assertEqual(instance.timezone, mock_response.location.time_zone) + mock_address = { + "city": mock_response.city.name, + "country": mock_response.country.name, + "continent": mock_response.continent.name, + "postal": mock_response.postal.code, + } + self.assertEqual(instance.address, mock_address) + self.assertEqual(instance.cidr, "172.217.22.0/24") + self.assertEqual(instance.ip_address, ip_address) + formatted_address = WHOISInfo(address=mock_address).formatted_address + self.assertEqual(instance.formatted_address, formatted_address) + self.assertEqual(instance.coordinates.x, mock_response.location.longitude) + self.assertEqual(instance.coordinates.y, mock_response.location.latitude) + + # mocking the response from the geoip2 client + mock_client.return_value.city.return_value = self._mocked_client_response() + + with self.subTest("Test WHOIS create when device is created"): + device = self._create_device(last_ip="172.217.22.14") + self.assertEqual(mock_info.call_count, 1) + mock_info.reset_mock() + device.refresh_from_db() + + _verify_whois_details( + device.whois_service.get_device_whois_info(), device.last_ip + ) + + with self.subTest( + "Test WHOIS create & deletion of old record when last ip is updated" + " when no other devices are linked to the old ip address" + ): + old_ip_address = device.last_ip + device.last_ip = "172.217.22.10" + device.save() + device.refresh_from_db() + self.assertEqual(mock_info.call_count, 1) + mock_info.reset_mock() + device.refresh_from_db() + + _verify_whois_details( + device.whois_service.get_device_whois_info(), device.last_ip + ) + # details related to old ip address should be deleted + self.assertEqual( + WHOISInfo.objects.filter(ip_address=old_ip_address).count(), 0 + ) + + with self.subTest( + "Test WHOIS create & deletion of old record when last ip is updated" + " when other devices are linked to the old ip address" + ): + old_ip_address = device.last_ip + self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.11", + ) + device.last_ip = "172.217.22.11" + device.save() + device.refresh_from_db() + self.assertEqual(mock_info.call_count, 1) + mock_info.reset_mock() + device.refresh_from_db() + _verify_whois_details( + device.whois_service.get_device_whois_info(), device.last_ip + ) + # details related to old ip address should be not be deleted + self.assertEqual( + WHOISInfo.objects.filter(ip_address=old_ip_address).count(), 1 + ) + + with self.subTest( + "Test WHOIS not deleted when device is deleted and" + " other active devices are linked to the last_ip" + ): + ip_address = device.last_ip + device.delete(check_deactivated=False) + self.assertEqual(mock_info.call_count, 0) + mock_info.reset_mock() + # WHOIS related to the device's last_ip should be deleted + self.assertEqual(WHOISInfo.objects.filter(ip_address=ip_address).count(), 1) + + Device.objects.all().delete() + with self.subTest( + "Test WHOIS deleted when device is deleted and" + " no other active devices are linked to the last_ip" + ): + device1 = self._create_device(last_ip="172.217.22.11") + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.11", + ) + device2.deactivate() + mock_info.reset_mock() + ip_address = device1.last_ip + device1.delete(check_deactivated=False) + self.assertEqual(mock_info.call_count, 0) + mock_info.reset_mock() + # WHOIS related to the device's last_ip should be deleted + self.assertEqual(WHOISInfo.objects.filter(ip_address=ip_address).count(), 0) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_whois_update(self, mock_client): + connect_whois_handlers() + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + threshold = app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + 1 + new_time = timezone.now() - timedelta(days=threshold) + whois_obj = self._create_whois_info() + WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) + + with self.subTest("Test WHOIS update when older than X days for new device"): + mocked_response.traits.autonomous_system_number = 11111 + mock_client.return_value.city.return_value = mocked_response + device = self._create_device(last_ip=whois_obj.ip_address) + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj.asn, str(11111)) + + with self.subTest( + "Test WHOIS update not running for invalid ip address " + "even if older than X days" + ): + Device.objects.all().delete() + WHOISInfo.objects.all().delete() + # to check ip address fallbacks are working if device created by + # bypassing validations + device = Device.objects.create( + name="default.test.device", + organization=self._get_org(), + mac_address=self.TEST_MAC_ADDRESS, + last_ip="InvalidIP", + ) + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj, None) + device.save() + device.refresh_from_db() + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj, None) + + with self.subTest( + "Test WHOIS update not running if whois disabled even if older than X days" + ): + Device.objects.all().delete() + WHOISInfo.objects.all().delete() + device = self._create_device(last_ip="172.217.22.11") + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj.asn, str(11111)) + WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) + mocked_response.traits.autonomous_system_number = 22222 + mock_client.return_value.city.return_value = mocked_response + org = self._get_org() + org.config_settings.whois_enabled = False + org.config_settings.save() + device.save() + device.refresh_from_db() + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj, None) + + with self.subTest( + "Test WHOIS update when older than X days for existing device" + ): + Device.objects.all().delete() + WHOISInfo.objects.all().delete() + org = self._get_org() + org.config_settings.whois_enabled = True + org.config_settings.save() + device = self._create_device(last_ip="172.217.22.11") + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj.asn, str(22222)) + WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) + mocked_response.traits.autonomous_system_number = 33333 + mock_client.return_value.city.return_value = mocked_response + device.save() + device.refresh_from_db() + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj.asn, str(33333)) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_device_last_ip_deferred_checks(self): + whois_obj = self._create_whois_info() + self._create_device(last_ip=whois_obj.ip_address) + # Deferred fields remained deferred + device = Device.objects.only("id", "created").first() + with self.subTest("Test deferred fields remained deferred after last_ip check"): + device._check_last_ip() + self.assertTrue(device._is_deferred("last_ip")) + + with self.subTest( + "Test update_whois does not run if last_ip is deferred" + ), mock.patch( + "openwisp_controller.config.whois.service.WHOISService.update_whois_info" + ) as mock_update_whois: + threshold = app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + 1 + new_time = timezone.now() - timedelta(days=threshold) + WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) + device._check_last_ip() + mock_update_whois.assert_not_called() + mock_update_whois.reset_mock() + + def test_create_or_update_whois_updates_modified_unchanged_details(self): + """ + Test that _create_or_update_whois updates the modified field + even when the WHOIS details are unchanged. + """ + whois_obj = self._create_whois_info(ip_address="172.217.22.50") + device = self._create_device(last_ip=whois_obj.ip_address) + old_modified = whois_obj.modified + whois_details = { + "isp": whois_obj.isp, + "asn": whois_obj.asn, + "timezone": whois_obj.timezone, + "address": whois_obj.address, + "cidr": whois_obj.cidr, + "coordinates": whois_obj.coordinates, + } + updated_obj, _ = device.whois_service._create_or_update_whois( + whois_details, whois_obj + ) + updated_obj.refresh_from_db() + self.assertGreater( + updated_obj.modified, + old_modified, + ) + + # we need to allow the task to propagate exceptions to ensure + # `on_failure` method is called and notifications are executed + @override_settings(CELERY_TASK_EAGER_PROPAGATES=False) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_TASKS_ERR_LOGGER) + @mock.patch(_WHOIS_TASKS_WARN_LOGGER) + @mock.patch(_WHOIS_TASKS_INFO_LOGGER) + def test_whois_task_failure_notification(self, mock_info, mock_warn, mock_error): + def assert_logging_on_exception( + exception, info_calls=0, warn_calls=0, error_calls=1, notification_count=1 + ): + with self.subTest( + f"Test notification and logging when {exception.__name__} is raised" + ), mock.patch(self._WHOIS_GEOIP_CLIENT) as mock_client: + mock_client.return_value.city.side_effect = exception("test") + Device.objects.all().delete() # Clear existing devices + device = self._create_device(last_ip="172.217.22.14") + self.assertEqual(mock_info.call_count, info_calls) + self.assertEqual(mock_warn.call_count, warn_calls) + self.assertEqual(mock_error.call_count, error_calls) + if notification_count > 0: + notification_qs = _notification_qs() + self.assertEqual(notification_qs.count(), notification_count) + notification = notification_qs.first() + self.assertEqual(notification.actor, device) + self.assertEqual(notification.target, device) + self.assertEqual(notification.level, "error") + self.assertEqual(notification.type, "generic_message") + self.assertIn( + "Failed to fetch WHOIS details for device", + notification.message, + ) + self.assertIn(device.last_ip, notification.rendered_description) + mock_info.reset_mock() + mock_warn.reset_mock() + mock_error.reset_mock() + _notification_qs().delete() + + # Test for all possible exceptions that can be raised by the geoip2 client + # Notification are sent only one time when any of the following exceptions + # are raised first time. + assert_logging_on_exception(errors.OutOfQueriesError) + assert_logging_on_exception(errors.AddressNotFoundError, notification_count=0) + assert_logging_on_exception(errors.AuthenticationError, notification_count=0) + assert_logging_on_exception( + errors.PermissionRequiredError, notification_count=0 + ) + assert_logging_on_exception(RequestException, notification_count=0) + cache.clear() + + @override_settings(CACHES=MODIFIED_CACHE) + @override_settings(CELERY_TASK_EAGER_PROPAGATES=False) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_whois_task_failure_cache(self): + permanent_errors = [ + errors.AddressNotFoundError, + errors.OutOfQueriesError, + errors.AuthenticationError, + errors.PermissionRequiredError, + ] + + def trigger_error_and_assert_cached(exc, notification_count=0): + with mock.patch(self._WHOIS_GEOIP_CLIENT) as mock_client: + mock_client.return_value.city.side_effect = exc("test") + Device.objects.all().delete() + device = self._create_device(last_ip="172.217.22.14") + cache_key = f"{self._WHOIS_TASK_NAME}_last_operation" + self.assertEqual(cache.get(cache_key), "errored") + self.assertEqual(_notification_qs().count(), notification_count) + _notification_qs().delete() + return device + + # simulate that no matter which permanent error is raised first, + # the rest of the permanent errors should use the cache + for first_error in permanent_errors: + with self.subTest(f"Cache populated by {first_error.__name__}"): + cache.clear() + trigger_error_and_assert_cached(first_error, 1) + for subsequent_error in permanent_errors: + if subsequent_error is first_error: + continue + with self.subTest( + f"Cache reused when {subsequent_error.__name__} occurs " + f"after {first_error.__name__}" + ): + trigger_error_and_assert_cached(subsequent_error, 0) + + with self.subTest("Test cache updated on success"), mock.patch( + self._WHOIS_GEOIP_CLIENT + ) as mock_client: + Device.objects.all().delete() + mocked_response = self._mocked_client_response() + mocked_response.location = None + mock_client.return_value.city.return_value = mocked_response + self._create_device(last_ip="172.217.22.14") + cache_key = f"{self._WHOIS_TASK_NAME}_last_operation" + self.assertEqual(cache.get(cache_key), "success") + self.assertEqual(_notification_qs().count(), 0) + cache.clear() + + @override_settings(CELERY_TASK_EAGER_PROPAGATES=True) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch("openwisp_controller.config.whois.tasks.fetch_whois_details.retry") + def test_whois_task_retry_mechanism(self, mock_retry): + def assert_retry_on_exception(exception, should_retry): + with self.subTest( + f"Test retry mechanism when {exception.__name__} is raised" + ), mock.patch(self._WHOIS_GEOIP_CLIENT) as mock_client: + mock_client.return_value.city.side_effect = exception("test") + Device.objects.all().delete() + mock_retry.reset_mock() + mock_retry.side_effect = exception("test") + with self.assertRaises(exception): + self._create_device(last_ip="172.217.22.14") + if should_retry: + self.assertEqual(mock_retry.call_count, 1) + assert isinstance(mock_retry.call_args.kwargs["exc"], exception) + else: + self.assertEqual(mock_retry.call_count, 0) + + assert_retry_on_exception(errors.HTTPError, should_retry=True) + assert_retry_on_exception(errors.OutOfQueriesError, should_retry=False) + assert_retry_on_exception(errors.AddressNotFoundError, should_retry=False) + assert_retry_on_exception(errors.AuthenticationError, should_retry=False) + assert_retry_on_exception(errors.PermissionRequiredError, should_retry=False) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_TASKS_WARN_LOGGER) + def test_fetch_whois_details_device_not_found(self, mock_warn): + invalid_pk = uuid4() + fetch_whois_details(device_pk=invalid_pk, initial_ip_address="10.0.0.1") + mock_warn.assert_called_once_with( + f"Device {invalid_pk} not found, skipping WHOIS lookup" + ) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_fetch_whois_details_record_already_exists(self, mock_client): + whois_obj = self._create_whois_info() + device = self._create_device(last_ip=whois_obj.ip_address) + mock_client.return_value.city.return_value = self._mocked_client_response() + fetch_whois_details(device_pk=device.pk, initial_ip_address="10.0.0.1") + mock_client.assert_not_called() + + def test_send_whois_task_notification_with_invalid_device_pk(self): + invalid_pk = uuid4() + result = send_whois_task_notification( + device=invalid_pk, notify_type="whois_device_error" + ) + self.assertIsNone(result) + + def test_delete_whois_record_force(self): + whois_obj = self._create_whois_info() + ip_address = whois_obj.ip_address + delete_whois_record(ip_address=ip_address, force=True) + self.assertFalse(WHOISInfo.objects.filter(ip_address=ip_address).exists()) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_get_whois_info_device_whois_disabled(self): + org = self._get_org() + org.config_settings.whois_enabled = False + org.config_settings.save() + device = self._create_device(last_ip="172.217.22.14") + result = get_whois_info(pk=device.pk) + self.assertIsNone(result) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_get_whois_info_with_none_pk(self): + result = get_whois_info(pk=None) + self.assertIsNone(result) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_get_whois_info_device_not_found(self): + result = get_whois_info(pk=uuid4()) + self.assertIsNone(result) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_get_whois_info_not_found(self, mock_client): + mock_client.return_value.city.return_value = self._mocked_client_response() + org = self._get_org() + org.config_settings.whois_enabled = True + org.config_settings.save() + device = self._create_device(last_ip="172.217.22.14") + WHOISInfo.objects.all().delete() + result = get_whois_info(pk=device.pk) + self.assertIsNone(result) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_get_whois_info_returns_data_with_formatted_address(self, mock_client): + mock_client.return_value.city.return_value = self._mocked_client_response() + org = self._get_org() + device = self._create_device(last_ip="172.217.22.14") + WHOISInfo.objects.filter(ip_address=device.last_ip).delete() + org.config_settings.whois_enabled = True + org.config_settings.save() + WHOISInfo.objects.create( + ip_address=device.last_ip, + address={ + "city": "Mountain View", + "country": "United States", + "postal": "94043", + }, + ) + result = get_whois_info(pk=device.pk) + self.assertIsNotNone(result) + self.assertEqual( + result["formatted_address"], "Mountain View, United States, 94043" + ) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", False) + def test_get_whois_info_when_not_configured(self): + device = self._create_device(last_ip="172.217.22.14") + result = get_whois_info(pk=device.pk) + self.assertIsNone(result) + + +@tag("selenium_tests") +class TestWHOISSelenium(CreateWHOISMixin, SeleniumTestMixin, StaticLiveServerTestCase): + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_whois_device_admin(self): + def _assert_no_js_errors(): + browser_logs = [] + for log in self.get_browser_logs(): + if self.browser == "chrome" and log["source"] != "console-api": + continue + elif log["message"] in ["wrong event specified: touchleave"]: + continue + browser_logs.append(log) + self.assertEqual(browser_logs, []) + + whois_obj = self._create_whois_info() + device = self._create_device(last_ip=whois_obj.ip_address) + self.login() + + with self.subTest( + "WHOIS details visible in device admin when WHOIS_CONFIGURED is True" + ): + self.open(reverse("admin:config_device_change", args=[device.pk])) + table = self.find_element(By.CSS_SELECTOR, "table.whois-table") + rows = table.find_elements(By.TAG_NAME, "tr") + for row in rows: + if cells := row.find_elements(By.TAG_NAME, "td"): + self.assertEqual(cells[0].text, whois_obj.isp) + self.assertEqual(cells[1].text, whois_obj.address["country"]) + details = self.find_element(By.CSS_SELECTOR, "details.whois") + self.web_driver.execute_script( + "arguments[0].setAttribute('open','')", details + ) + additional_text = details.find_elements(By.CSS_SELECTOR, ".additional-text") + self.assertIn(whois_obj.asn, additional_text[0].text) + self.assertIn(whois_obj.timezone, additional_text[1].text) + self.assertIn(whois_obj.formatted_address, additional_text[2].text) + self.assertIn(whois_obj.cidr, additional_text[3].text) + _assert_no_js_errors() + + with mock.patch.object(app_settings, "WHOIS_CONFIGURED", False): + with self.subTest( + "WHOIS details not visible in device admin " + + "when WHOIS_CONFIGURED is False" + ): + self.open(reverse("admin:config_device_change", args=[device.pk])) + self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") + self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") + _assert_no_js_errors() + + with self.subTest( + "WHOIS details not visible in device admin when WHOIS is disabled" + ): + org = self._get_org() + org.config_settings.whois_enabled = False + org.config_settings.save(update_fields=["whois_enabled"]) + org.config_settings.refresh_from_db(fields=["whois_enabled"]) + self.open(reverse("admin:config_device_change", args=[device.pk])) + self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") + self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") + _assert_no_js_errors() + + with self.subTest( + "WHOIS details not visible in device admin when WHOIS Info does not exist" + ): + org = self._get_org() + org.config_settings.whois_enabled = True + org.config_settings.save(update_fields=["whois_enabled"]) + org.config_settings.refresh_from_db(fields=["whois_enabled"]) + WHOISInfo.objects.all().delete() + self.open(reverse("admin:config_device_change", args=[device.pk])) + self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") + self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") + _assert_no_js_errors() + + with self.subTest("Check XSS protection in WHOIS details admin view"): + whois_data = { + "ip_address": device.last_ip, + "isp": "", + "timezone": "", + "address": { + "city": "Mountain View", + "country": "", + "continent": "North America", + "postal": "94043", + }, + } + WHOISInfo.objects.all().delete() + self._create_whois_info(**whois_data) + try: + self.open(reverse("admin:config_device_change", args=[device.pk])) + table = self.find_element(By.CSS_SELECTOR, "table.whois-table") + rows = table.find_elements(By.TAG_NAME, "tr") + for row in rows: + if cells := row.find_elements(By.TAG_NAME, "td"): + self.assertIn("onerror", cells[0].text) + self.assertIn("script", cells[1].text) + details = self.find_element(By.CSS_SELECTOR, "details.whois") + self.web_driver.execute_script( + "arguments[0].setAttribute('open','')", details + ) + additional_text = details.find_elements( + By.CSS_SELECTOR, ".additional-text" + ) + self.assertIn("script", additional_text[1].text) + self.assertIn("script", additional_text[2].text) + _assert_no_js_errors() + except UnexpectedAlertPresentException: + self.fail("XSS vulnerability detected in WHOIS details admin view.") diff --git a/openwisp_controller/config/whois/tests/utils.py b/openwisp_controller/config/whois/tests/utils.py new file mode 100644 index 000000000..ce36e5a4d --- /dev/null +++ b/openwisp_controller/config/whois/tests/utils.py @@ -0,0 +1,164 @@ +from unittest import mock + +from django.contrib.gis.geos import Point +from django.urls import reverse +from swapper import load_model + +from ...tests.utils import CreateConfigMixin + +Device = load_model("config", "Device") +WHOISInfo = load_model("config", "WHOISInfo") +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + + +class CreateWHOISMixin(CreateConfigMixin): + def _create_whois_info(self, **kwargs): + options = dict( + ip_address="172.217.22.14", + address={ + "city": "Mountain View", + "country": "United States", + "continent": "North America", + "postal": "94043", + }, + asn="15169", + isp="Google LLC", + timezone="America/Los_Angeles", + cidr="172.217.22.0/24", + coordinates=Point(150, 50, srid=4326), + ) + options.update(kwargs) + w = WHOISInfo(**options) + w.full_clean() + w.save() + return w + + def setUp(self): + super().setUp() + OrganizationConfigSettings.objects.create( + organization=self._get_org(), whois_enabled=True + ) + + +class WHOISTransactionMixin: + @staticmethod + def _mocked_client_response(): + mock_response = mock.MagicMock() + mock_response.city.name = "Mountain View" + mock_response.country.name = "United States" + mock_response.continent.name = "North America" + mock_response.postal.code = "94043" + mock_response.traits.autonomous_system_organization = "Google LLC" + mock_response.traits.autonomous_system_number = 15169 + mock_response.traits.network = "172.217.22.0/24" + mock_response.location.time_zone = "America/Los_Angeles" + mock_response.location.latitude = 50 + mock_response.location.longitude = 150 + return mock_response + + def _task_called(self, mocked_task, task_name="WHOIS lookup"): + org = self._get_org() + device = self._create_device(last_ip="172.217.22.14") + mocked_task.reset_mock() + + with self.subTest(f"{task_name} task not called when last_ip not updated"): + device.name = "default.test.Device2" + device.save() + device.refresh_from_db() + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest(f"{task_name} task not called when last_ip is private"): + device.last_ip = "10.0.0.1" + device.save() + device.refresh_from_db() + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest(f"{task_name} task not called when last_ip is invalid"): + device.last_ip = "invalid_ip" + device.save() + device.refresh_from_db() + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest(f"{task_name} task not called when WHOIS is disabled"): + Device.objects.all().delete() + org.config_settings.whois_enabled = False + # Invalidates old org config settings cache + org.config_settings.save(update_fields=["whois_enabled"]) + org.config_settings.refresh_from_db(fields=["whois_enabled"]) + device = self._create_device(last_ip="172.217.22.14") + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest( + f"{task_name} task called via DeviceChecksumView when WHOIS is enabled" + ): + org.config_settings.whois_enabled = True + # Invalidates old org config settings cache + org.config_settings.save(update_fields=["whois_enabled"]) + org.config_settings.refresh_from_db(fields=["whois_enabled"]) + # config is required for checksum view to work + device.refresh_from_db() + self._create_config(device=device) + # setting remote address field to a public IP to trigger WHOIS task + # since the view uses this header for tracking the device's IP + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR="172.217.22.10", + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_called() + mocked_task.reset_mock() + + with self.subTest( + f"{task_name} task not called via DeviceChecksumView " + "if no WHOIS record and IP unchanged" + ): + WHOISInfo.objects.all().delete() + device.refresh_from_db() + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest( + f"{task_name} task not called via DeviceChecksumView when WHOIS is disabled" + ): + WHOISInfo.objects.all().delete() + device.refresh_from_db() + org.config_settings.whois_enabled = False + org.config_settings.save(update_fields=["whois_enabled"]) + org.config_settings.refresh_from_db(fields=["whois_enabled"]) + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest( + f"{task_name} task not called explicitly via DeviceChecksumView for " + "stale records" + ), mock.patch( + "openwisp_controller.config.whois.service.WHOISService.is_older", + return_value=True, + ): + WHOISInfo.objects.all().delete() + self._create_whois_info(ip_address=device.last_ip) + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_not_called() + mocked_task.reset_mock() diff --git a/openwisp_controller/config/whois/utils.py b/openwisp_controller/config/whois/utils.py new file mode 100644 index 000000000..1a51a6d44 --- /dev/null +++ b/openwisp_controller/config/whois/utils.py @@ -0,0 +1,74 @@ +from django.utils.translation import gettext_lazy as _ +from geoip2 import errors +from openwisp_notifications.signals import notify +from swapper import load_model + +from .. import settings as app_settings + +MESSAGE_MAP = { + "whois_device_error": { + "type": "generic_message", + "level": "error", + "message": _( + "Failed to fetch WHOIS details for device" + " [{notification.target}]({notification.target_link})" + ), + "description": _("WHOIS details could not be fetched for ip: {ip_address}."), + }, +} + +EXCEPTION_MESSAGES = { + errors.AddressNotFoundError: _( + "No WHOIS information found for IP address {ip_address}" + ), + errors.AuthenticationError: _( + "Authentication failed for GeoIP2 service. " + "Check your OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT and " + "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY settings." + ), + errors.OutOfQueriesError: _( + "Your account has run out of queries for the GeoIP2 service." + ), + errors.PermissionRequiredError: _( + "Your account does not have permission to access this service." + ), +} + + +def send_whois_task_notification(device, notify_type, actor=None): + Device = load_model("config", "Device") + if not isinstance(device, Device): + device = Device.objects.filter(pk=device).first() + if not device: + return + notify_details = MESSAGE_MAP[notify_type] + notify.send( + sender=actor or device, + target=device, + action_object=device, + ip_address=device.last_ip, + **notify_details, + ) + + +def get_whois_info(pk): + from .serializers import WHOISSerializer + from .service import WHOISService + + Device = load_model("config", "Device") + + if not app_settings.WHOIS_CONFIGURED or not pk: + return None + device = ( + Device.objects.select_related("organization__config_settings") + .filter(pk=pk) + .first() + ) + if not device or not device._get_organization__config_settings().whois_enabled: + return None + whois_obj = WHOISService(device).get_device_whois_info() + if not whois_obj: + return None + data = WHOISSerializer(whois_obj).data + data["formatted_address"] = getattr(whois_obj, "formatted_address", None) + return data diff --git a/openwisp_controller/geo/admin.py b/openwisp_controller/geo/admin.py index 9064d657a..f08960b8c 100644 --- a/openwisp_controller/geo/admin.py +++ b/openwisp_controller/geo/admin.py @@ -12,6 +12,8 @@ ) from swapper import load_model +from openwisp_controller.config import settings as config_app_settings +from openwisp_controller.config.whois.service import WHOISService from openwisp_users.multitenancy import MultitenantOrgFilter from ..admin import MultitenantAdminMixin @@ -98,6 +100,46 @@ class LocationAdmin(MultitenantAdminMixin, AbstractLocationAdmin): form = LocationForm inlines = [FloorPlanInline] list_select_related = ("organization",) + change_form_template = "admin/geo/location/change_form.html" + + # Adding is_estimated field via 'get_' methods to allow testing in + # isolation as class level insertions are evaluated at import time + # making it unsuitable as per current testing setup. + def get_list_display(self, request): + list_display = list(super().get_list_display(request)) # makes a copy + if config_app_settings.WHOIS_CONFIGURED: + list_display.insert(list_display.index("created"), "is_estimated") + return list_display + + def get_list_filter(self, request): + list_filter = list(super().get_list_filter(request)) # makes a copy + if config_app_settings.WHOIS_CONFIGURED: + list_filter.append("is_estimated") + return list_filter + + def get_fields(self, request, obj=None): + fields = list(super().get_fields(request, obj)) # makes a copy + # do not show the is_estimated field on add pages + # or if the estimated location feature is not enabled for the organization + if not obj or not WHOISService.check_estimated_location_enabled( + obj.organization_id + ): + fields.remove("is_estimated") + return fields + + def get_readonly_fields(self, request, obj=None): + fields = super().get_readonly_fields(request, obj) + if obj and WHOISService.check_estimated_location_enabled(obj.organization_id): + fields = ("is_estimated",) + fields # creates a new tuple + return fields + + def change_view(self, request, object_id, form_url="", extra_context=None): + obj = self.get_object(request, object_id) + org_id = obj.organization_id if obj else None + estimated_enabled = WHOISService.check_estimated_location_enabled(org_id) + extra_context = extra_context or {} + extra_context["estimated_enabled"] = estimated_enabled + return super().change_view(request, object_id, form_url, extra_context) LocationAdmin.list_display.insert(1, "organization") @@ -122,16 +164,39 @@ class DeviceLocationFilter(admin.SimpleListFilter): title = _("has geographic position set?") parameter_name = "with_geo" + def __init__(self, request, params, model, model_admin): + super().__init__(request, params, model, model_admin) + if config_app_settings.WHOIS_CONFIGURED: + self.title = _("geographic position") + def lookups(self, request, model_admin): + if config_app_settings.WHOIS_CONFIGURED: + return ( + ("outdoor", _("Outdoor")), + ("indoor", _("Indoor")), + ("estimated", _("Estimated")), + ("false", _("No Location")), + ) return ( ("true", _("Yes")), ("false", _("No")), ) def queryset(self, request, queryset): - if self.value(): - return queryset.filter(devicelocation__isnull=self.value() == "false") - return queryset + value = self.value() + if not value: + return queryset + if config_app_settings.WHOIS_CONFIGURED: + if value == "estimated": + return queryset.filter(devicelocation__location__is_estimated=True) + elif value in ("indoor", "outdoor"): + # estimated locations are outdoor by default + # so we need to exclude them from the result + return queryset.filter( + devicelocation__location__type=value, + devicelocation__location__is_estimated=False, + ) + return queryset.filter(devicelocation__isnull=self.value() == "false") # Prepend DeviceLocationInline to config.DeviceAdminExportable diff --git a/openwisp_controller/geo/api/filters.py b/openwisp_controller/geo/api/filters.py index 93a7c366e..2be821e58 100644 --- a/openwisp_controller/geo/api/filters.py +++ b/openwisp_controller/geo/api/filters.py @@ -1,6 +1,7 @@ from django.utils.translation import gettext_lazy as _ from django_filters import rest_framework as filters +from openwisp_controller.config import settings as config_app_settings from openwisp_controller.config.api.filters import ( DeviceListFilter as BaseDeviceListFilter, ) @@ -30,6 +31,20 @@ def filter_devicelocation(self, queryset, name, value): # Returns list of device that have devicelocation objects return queryset.exclude(devicelocation__isnull=value) + def filter_is_estimated(self, queryset, name, value): + return queryset.filter(devicelocation__location__is_estimated=value) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if config_app_settings.WHOIS_CONFIGURED: + self.filters["geo_is_estimated"] = filters.BooleanFilter( + field_name="devicelocation__location__is_estimated", + method=self.filter_is_estimated, + ) + self.filters["geo_is_estimated"].label = _( + "Is geographic location estimated?" + ) + class Meta: model = BaseDeviceListFilter.Meta.model geo_fields = [ diff --git a/openwisp_controller/geo/api/serializers.py b/openwisp_controller/geo/api/serializers.py index e4a800d90..3d7cc61e3 100644 --- a/openwisp_controller/geo/api/serializers.py +++ b/openwisp_controller/geo/api/serializers.py @@ -11,6 +11,10 @@ from openwisp_utils.api.serializers import ValidatedModelSerializer from ...serializers import BaseSerializer +from ..estimated_location.mixins import ( + EstimatedLocationGeoJsonMixin, + EstimatedLocationMixin, +) Device = load_model("config", "Device") Location = load_model("geo", "Location") @@ -31,7 +35,9 @@ class Meta: fields = "__all__" -class GeoJsonLocationSerializer(gis_serializers.GeoFeatureModelSerializer): +class GeoJsonLocationSerializer( + EstimatedLocationGeoJsonMixin, gis_serializers.GeoFeatureModelSerializer +): device_count = IntegerField() class Meta: @@ -126,7 +132,7 @@ class Meta: read_only_fields = ("name",) -class LocationSerializer(BaseSerializer): +class LocationSerializer(EstimatedLocationMixin, BaseSerializer): floorplan = FloorPlanLocationSerializer(required=False, allow_null=True) class Meta: @@ -160,7 +166,8 @@ def validate(self, data): def to_representation(self, instance): request = self.context["request"] data = super().to_representation(instance) - floorplans = instance.floorplan_set.all().order_by("-modified") + # floorplan_set is already prefetched and ordered in the view + floorplans = instance.floorplan_set.all() floorplan_list = [] for floorplan in floorplans: dict_ = { @@ -225,7 +232,9 @@ def update(self, instance, validated_data): return super().update(instance, validated_data) -class NestedtLocationSerializer(gis_serializers.GeoFeatureModelSerializer): +class NestedtLocationSerializer( + EstimatedLocationGeoJsonMixin, gis_serializers.GeoFeatureModelSerializer +): class Meta: model = Location geo_field = "geometry" diff --git a/openwisp_controller/geo/api/views.py b/openwisp_controller/geo/api/views.py index b521c3d27..ae87c34df 100644 --- a/openwisp_controller/geo/api/views.py +++ b/openwisp_controller/geo/api/views.py @@ -1,5 +1,5 @@ from django.core.exceptions import ObjectDoesNotExist, ValidationError -from django.db.models import Count +from django.db.models import Count, Prefetch from django.http import Http404 from django.utils.translation import gettext_lazy as _ from django_filters import rest_framework as filters @@ -11,6 +11,7 @@ from rest_framework_gis.pagination import GeoJsonPagination from swapper import load_model +from openwisp_controller.config import settings as config_app_settings from openwisp_controller.config.api.views import DeviceListCreateView from openwisp_users.api.filters import OrganizationManagedFilter from openwisp_users.api.mixins import FilterByOrganizationManaged, FilterByParentManaged @@ -51,6 +52,17 @@ class Meta(OrganizationManagedFilter.Meta): model = Location fields = OrganizationManagedFilter.Meta.fields + ["is_mobile", "type"] + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # This is evaluated at runtime, which makes it suited + # For the automated testing strategy we are using. + # Defining this at class definition does not allow flexible testing. + if config_app_settings.WHOIS_CONFIGURED: + self.filters["is_estimated"] = filters.BooleanFilter( + field_name="is_estimated", + label=_("Is geographic location estimated?"), + ) + class FloorPlanOrganizationFilter(OrganizationManagedFilter): class Meta(OrganizationManagedFilter.Meta): @@ -215,8 +227,10 @@ class GeoJsonLocationList( Shows only locations which are assigned to devices. """ - queryset = Location.objects.filter(devicelocation__isnull=False).annotate( - device_count=Count("devicelocation") + queryset = ( + Location.objects.filter(devicelocation__isnull=False) + .annotate(device_count=Count("devicelocation")) + .order_by("-created") ) serializer_class = GeoJsonLocationSerializer pagination_class = GeoJsonLocationListPagination @@ -311,7 +325,12 @@ class FloorPlanDetailView( class LocationListCreateView(ProtectedAPIMixin, generics.ListCreateAPIView): serializer_class = LocationSerializer - queryset = Location.objects.order_by("-created") + queryset = Location.objects.prefetch_related( + Prefetch( + "floorplan_set", + queryset=FloorPlan.objects.order_by("-created"), + ) + ).order_by("-created") pagination_class = ListViewPagination filter_backends = [filters.DjangoFilterBackend] filterset_class = LocationOrganizationFilter diff --git a/openwisp_controller/geo/apps.py b/openwisp_controller/geo/apps.py index 90a626f8e..0e2cc6264 100644 --- a/openwisp_controller/geo/apps.py +++ b/openwisp_controller/geo/apps.py @@ -14,6 +14,8 @@ from openwisp_utils.admin_theme import register_dashboard_chart from openwisp_utils.admin_theme.menu import register_menu_group +from .estimated_location.handlers import register_estimated_location_notification_types + class GeoConfig(LociConfig): name = "openwisp_controller.geo" @@ -27,6 +29,7 @@ def ready(self): super().ready() self.register_dashboard_charts() self.register_menu_groups() + register_estimated_location_notification_types() if getattr(settings, "TESTING", False): self._add_params_to_test_config() diff --git a/openwisp_controller/geo/base/models.py b/openwisp_controller/geo/base/models.py index c7bc59798..c26aca6cb 100644 --- a/openwisp_controller/geo/base/models.py +++ b/openwisp_controller/geo/base/models.py @@ -1,4 +1,8 @@ +from typing import ClassVar + from django.contrib.gis.db import models +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ from django_loci.base.models import ( AbstractFloorPlan, AbstractLocation, @@ -6,13 +10,96 @@ ) from swapper import get_model_name +from openwisp_controller.config.whois.service import WHOISService from openwisp_users.mixins import OrgMixin, ValidateOrgMixin class BaseLocation(OrgMixin, AbstractLocation): + _changed_checked_fields: ClassVar[list[str]] = [ + "is_estimated", + "address", + "geometry", + ] + + is_estimated = models.BooleanField( + _("Is Estimated?"), + default=False, + help_text=_("Whether the location's coordinates are estimated."), + ) + class Meta(AbstractLocation.Meta): abstract = True + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._set_initial_values_for_changed_checked_fields() + + def _set_initial_values_for_changed_checked_fields(self): + deferred_fields = self.get_deferred_fields() + for field in self._changed_checked_fields: + if field in deferred_fields: + setattr(self, f"_initial_{field}", models.DEFERRED) + else: + setattr(self, f"_initial_{field}", getattr(self, field)) + + def clean(self): + # Raise validation error if `is_estimated` is True but estimated feature is + # disabled. + estimated_status_changed = ( + self._initial_is_estimated is not models.DEFERRED + and self._initial_is_estimated != self.is_estimated + ) + if ( + (self._state.adding or estimated_status_changed) + and self.is_estimated + and not WHOISService.check_estimated_location_enabled(self.organization_id) + ): + raise ValidationError( + { + "is_estimated": _( + "Estimated Location feature required to be configured." + ) + } + ) + return super().clean() + + def save(self, *args, _set_estimated=False, **kwargs): + """ + Save the location object with special handling for estimated locations. + + Parameters: + _set_estimated: Boolean flag to indicate if this save is being performed + by the estimated location system. When False (default), + manual edits will clear the estimated status (only if estimated location + feature is enabled). + *args, **kwargs: Arguments passed to the parent save method. + + Returns: + The result of the parent save method. + """ + changed_fields = set() + if WHOISService.check_estimated_location_enabled(self.organization_id): + address_changed = ( + self._initial_address is not models.DEFERRED + and self._initial_address != self.address + ) + geometry_changed = ( + self._initial_geometry is not models.DEFERRED + and self._initial_geometry != self.geometry + ) + if not _set_estimated and (address_changed or geometry_changed): + self.is_estimated = False + changed_fields = {"is_estimated"} + # Manual changes to is_estimated discarded if feature not enabled + elif self._initial_is_estimated is not models.DEFERRED: + self.is_estimated = self._initial_is_estimated + changed_fields = {"is_estimated"} + if update_fields := kwargs.get("update_fields"): + kwargs["update_fields"] = set(update_fields) | changed_fields + result = super().save(*args, **kwargs) + self._set_initial_values_for_changed_checked_fields() + return result + class BaseFloorPlan(OrgMixin, AbstractFloorPlan): location = models.ForeignKey(get_model_name("geo", "Location"), models.CASCADE) diff --git a/openwisp_controller/geo/estimated_location/__init__.py b/openwisp_controller/geo/estimated_location/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/geo/estimated_location/handlers.py b/openwisp_controller/geo/estimated_location/handlers.py new file mode 100644 index 000000000..2b66a6d94 --- /dev/null +++ b/openwisp_controller/geo/estimated_location/handlers.py @@ -0,0 +1,35 @@ +from django.utils.translation import gettext_lazy as _ +from openwisp_notifications.types import register_notification_type +from swapper import load_model + +from openwisp_controller.config import settings as config_app_settings + +from .utils import MESSAGE_MAP + + +def register_estimated_location_notification_types(): + """ + Register the notification types used by the Estimated Location module. + This is necessary to ensure that the notifications are properly configured. + """ + if not config_app_settings.WHOIS_CONFIGURED: + return + + Device = load_model("config", "Device") + Location = load_model("geo", "Location") + + register_notification_type( + "estimated_location_info", + { + **MESSAGE_MAP["estimated_location_created"], + "verbose_name": _("Estimated Location INFO"), + "verb": _("created"), + "email_subject": _("Estimated location created for {notification.target}"), + "target_link": ( + "openwisp_controller.geo.estimated_location.utils" + ".get_device_location_notification_target_url" + ), + "email_notification": False, + }, + models=[Device, Location], + ) diff --git a/openwisp_controller/geo/estimated_location/mixins.py b/openwisp_controller/geo/estimated_location/mixins.py new file mode 100644 index 000000000..222eaa8b7 --- /dev/null +++ b/openwisp_controller/geo/estimated_location/mixins.py @@ -0,0 +1,33 @@ +from openwisp_controller.config.whois.service import WHOISService + + +# These mixins are required as estimated location is an organization level feature. +# Also, adding it this way makes it read-only as well. +class EstimatedLocationMixin: + """ + Serializer mixin to add estimated location field to the serialized data + if the estimated location feature is configured and enabled for the organization. + """ + + def to_representation(self, obj): + data = super().to_representation(obj) + if WHOISService.check_estimated_location_enabled(obj.organization_id): + data["is_estimated"] = obj.is_estimated + else: + data.pop("is_estimated", None) + return data + + +class EstimatedLocationGeoJsonMixin: + """ + Serializer mixin to add estimated location field to the serialized GeoJSON data + if the estimated location feature is configured and enabled for the organization. + """ + + def to_representation(self, obj): + data = super().to_representation(obj) + if WHOISService.check_estimated_location_enabled(obj.organization_id): + data["properties"]["is_estimated"] = obj.is_estimated + else: + data["properties"].pop("is_estimated", None) + return data diff --git a/openwisp_controller/geo/estimated_location/tasks.py b/openwisp_controller/geo/estimated_location/tasks.py new file mode 100644 index 000000000..58eb863d1 --- /dev/null +++ b/openwisp_controller/geo/estimated_location/tasks.py @@ -0,0 +1,168 @@ +import logging + +from celery import shared_task +from swapper import load_model + +from openwisp_controller.config.whois.utils import send_whois_task_notification + +logger = logging.getLogger(__name__) + + +def _handle_attach_existing_location( + device, device_location, ip_address, existing_device_location +): + """ + Helper function to: + 1. Attach existing device's location (same last_ip) to current device, else + 2. Update current location of device using WHOIS data; if it exists, else + 3. Create a new estimated location for the device using WHOIS data. + """ + Device = load_model("config", "Device") + WHOISInfo = load_model("config", "WHOISInfo") + + current_location = device_location.location + attached_devices_exists = None + if current_location is not None: + attached_devices_exists = ( + Device.objects.filter(devicelocation__location_id=current_location.pk) + .exclude(pk=device.pk) + .exists() + ) + if ( + existing_device_location + and existing_device_location.location != device_location.location + ): + existing_location = existing_device_location.location + device_location.location = existing_location + device_location.full_clean() + device_location.save() + logger.info( + f"Estimated location saved successfully for {device.pk}" + f" for IP: {ip_address}" + ) + # We need to remove existing estimated location of the device + # if it is not shared + if attached_devices_exists is False: + current_location.delete() + send_whois_task_notification( + device=device, + notify_type="estimated_location_updated", + actor=existing_location, + ) + return + # If existing devices with same last_ip do not have any location + # then we create a new location based on WHOIS data. + whois_obj = WHOISInfo.objects.filter(ip_address=ip_address).first() + if not whois_obj or not whois_obj.coordinates: + logger.warning( + f"Coordinates not available for {device.pk} for IP: {ip_address}." + " Estimated location cannot be determined." + ) + return + + location_defaults = { + **whois_obj._get_defaults_for_estimated_location(), + "organization_id": device.organization_id, + } + # Create new location only if location is changed. + if ( + attached_devices_exists + and current_location + and current_location.geometry == location_defaults.get("geometry") + and current_location.name == location_defaults.get("name") + ): + logger.debug( + f"Estimated location unchanged for {device.pk}" + f" for IP: {ip_address}, keeping existing location" + ) + return + # create new location if no location exists for device or the estimated location + # of device is shared. + whois_service = device.whois_service + whois_service._create_or_update_estimated_location( + location_defaults, attached_devices_exists + ) + logger.info( + f"Estimated location saved successfully for {device.pk}" + f" for IP: {ip_address}" + ) + + +@shared_task(name="whois_estimated_location_task") +def manage_estimated_locations(device_pk, ip_address): + """ + Creates/updates estimated location for a device based on the latitude and + longitude or attaches an existing location. + Existing location here means a location of another device whose last_ip matches + the given ip_address. + Does not alters the existing location if it is not estimated. + + - If the current device has no location or location is estimated, either update + to an existing location; if it exists, else + + - A new location is created if current device has no location, or + if it does; it is updated using coords from WHOIS record if it is estimated. + + In case of multiple devices with same last_ip, the task will send a notification + to the user to resolve the conflict manually. + """ + Device = load_model("config", "Device") + DeviceLocation = load_model("geo", "DeviceLocation") + + try: + device = Device.objects.select_related("devicelocation__location").get( + pk=device_pk + ) + except Device.DoesNotExist: + logger.warning( + f"Device {device_pk} not found, skipping manage_estimated_locations" + ) + return + devices_with_location = list( + # "devicelocation" and "devicelocation__location" must be in only() to + # prevent Django from deferring them, which would conflict with + # select_related(). Django raises FieldError if a relation field is + # both deferred and traversed via select_related. + Device.objects.only( + "id", "name", "last_ip", "devicelocation", "devicelocation__location" + ) + .select_related("devicelocation__location") + .filter( + organization_id=device.organization_id, + last_ip=ip_address, + devicelocation__location__isnull=False, + ) + # evaluated to LIMIT query, we need to know if there's more than 1 result + .exclude(pk=device.pk)[:2] + ) + # multiple devices can have same last_ip in cases like usage of proxy + if len(devices_with_location) > 1: + send_whois_task_notification( + device=device, notify_type="estimated_location_error" + ) + logger.error( + "Multiple devices with locations found with same " + f"last_ip {ip_address}. Please resolve the conflict manually." + ) + return + # if device doesn't have a location yet, initialize a draft + if not (device_location := getattr(device, "devicelocation", None)): + device_location = DeviceLocation(content_object=device) + current_location = device_location.location + if not current_location or current_location.is_estimated: + # existing device location + try: + existing_device_location = getattr( + devices_with_location[0], "devicelocation", None + ) + # no existing device location + except IndexError: + existing_device_location = None + _handle_attach_existing_location( + device, device_location, ip_address, existing_device_location + ) + else: + logger.info( + f"Non Estimated location already set for {device_pk}. Update" + f" location manually as per IP: {ip_address}" + ) diff --git a/openwisp_controller/geo/estimated_location/tests/__init__.py b/openwisp_controller/geo/estimated_location/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/geo/estimated_location/tests/tests.py b/openwisp_controller/geo/estimated_location/tests/tests.py new file mode 100644 index 000000000..70794f44c --- /dev/null +++ b/openwisp_controller/geo/estimated_location/tests/tests.py @@ -0,0 +1,1100 @@ +import contextlib +import importlib +from datetime import timedelta +from unittest import mock +from uuid import uuid4 + +from django.contrib.gis.geos import GEOSGeometry +from django.core.exceptions import ImproperlyConfigured, ValidationError +from django.test import TestCase, TransactionTestCase, override_settings +from django.urls import reverse +from django.utils import timezone +from openwisp_notifications.types import unregister_notification_type +from swapper import load_model + +from openwisp_controller.config import settings as config_app_settings +from openwisp_controller.config.whois.handlers import connect_whois_handlers +from openwisp_controller.config.whois.tests.utils import WHOISTransactionMixin +from openwisp_controller.geo import estimated_location + +from ....tests.utils import TestAdminMixin +from ...tests.utils import TestGeoMixin +from ..handlers import register_estimated_location_notification_types +from ..tasks import manage_estimated_locations +from .utils import TestEstimatedLocationMixin + +Config = load_model("config", "Config") +Device = load_model("config", "Device") +Location = load_model("geo", "Location") +DeviceLocation = load_model("geo", "DeviceLocation") +WHOISInfo = load_model("config", "WHOISInfo") +Notification = load_model("openwisp_notifications", "Notification") +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + + +def _notification_qs(): + return Notification.objects.all() + + +class TestEstimatedLocation(TestAdminMixin, TestCase): + @override_settings( + OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT="test_account", + OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY="test_key", + ) + def test_estimated_location_configuration_setting(self): + # reload app_settings to apply the overridden + self.addCleanup(importlib.reload, config_app_settings) + importlib.reload(config_app_settings) + with self.subTest( + "ImproperlyConfigured raised when ESTIMATED_LOCATION_ENABLED is True " + "and WHOIS_ENABLED is False globally" + ): + with override_settings( + OPENWISP_CONTROLLER_WHOIS_ENABLED=False, + OPENWISP_CONTROLLER_ESTIMATED_LOCATION_ENABLED=True, + ): + with self.assertRaises(ImproperlyConfigured): + # reload app_settings to apply the overridden settings + importlib.reload(config_app_settings) + + with self.subTest( + "Test WHOIS not enabled does not allow enabling Estimated Location" + ): + org_settings_obj = OrganizationConfigSettings(organization=self._get_org()) + with self.assertRaises(ValidationError) as context_manager: + org_settings_obj.whois_enabled = False + org_settings_obj.estimated_location_enabled = True + org_settings_obj.full_clean() + self.assertEqual( + context_manager.exception.message_dict["estimated_location_enabled"][0], + "Estimated Location feature requires " + + "WHOIS Lookup feature to be enabled.", + ) + + with self.subTest( + "Test Estimated Location field visible on admin when " + "WHOIS_CONFIGURED is True" + ): + self._login() + org = self._get_org() + url = reverse( + "admin:openwisp_users_organization_change", + args=[org.pk], + ) + response = self.client.get(url) + self.assertContains( + response, 'name="config_settings-0-estimated_location_enabled"' + ) + + with override_settings( + OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT=None, + OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY=None, + ): + importlib.reload(config_app_settings) + with self.subTest( + "Test Estimated Location field hidden on admin when " + "WHOIS_CONFIGURED is False" + ): + self._login() + org = self._get_org() + url = reverse( + "admin:openwisp_users_organization_change", + args=[org.pk], + ) + response = self.client.get(url) + self.assertNotContains( + response, 'name="config_settings-0-estimated_location_enabled"' + ) + + +class TestEstimatedLocationField(TestEstimatedLocationMixin, TestGeoMixin, TestCase): + location_model = Location + + def test_estimated_location_field(self): + org = self._get_org() + org.config_settings.estimated_location_enabled = False + org.config_settings.save() + org.refresh_from_db() + with self.assertRaises(ValidationError) as context_manager: + self._create_location(organization=org, is_estimated=True) + self.assertEqual( + context_manager.exception.message_dict["is_estimated"][0], + "Estimated Location feature required to be configured.", + ) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + def test_estimated_location_admin(self): + connect_whois_handlers() + admin = self._create_admin() + self.client.force_login(admin) + org = self._get_org() + location = self._create_location(organization=org, is_estimated=True) + change_path = reverse("admin:geo_location_change", args=[location.pk]) + add_path = reverse("admin:geo_location_add") + + with self.subTest( + "is-estimated field visible when estimated location is enabled" + ): + response = self.client.get(change_path) + self.assertContains(response, "field-is_estimated") + self.assertContains( + response, "Whether the location's coordinates are estimated." + ) + with self.subTest( + "is-estimated field not visible in add pages because auto-managed" + ): + response = self.client.get(add_path) + self.assertNotContains(response, "field-is_estimated") + + org.config_settings.estimated_location_enabled = False + org.config_settings.save() + org.config_settings.refresh_from_db() + + with self.subTest( + "is-estimated field hidden when estimated location is disabled" + ): + response = self.client.get(change_path) + self.assertNotContains(response, "field-is_estimated") + self.assertNotContains( + response, "Whether the location's coordinates are estimated." + ) + with self.subTest( + "double-check is-estimated field is not " + "leaking if estimated location is disabled" + ): + response = self.client.get(add_path) + self.assertNotContains(response, "field-is_estimated") + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False) + def test_estimated_location_admin_add_whois_disabled(self): + admin = self._create_admin() + self.client.force_login(admin) + path = reverse("admin:geo_location_add") + response = self.client.get(path) + self.assertNotContains(response, "field-is_estimated") + + +class TestEstimatedLocationTransaction( + TestEstimatedLocationMixin, WHOISTransactionMixin, TestGeoMixin, TransactionTestCase +): + location_model = Location + object_location_model = DeviceLocation + + _WHOIS_GEOIP_CLIENT = ( + "openwisp_controller.config.whois.service.geoip2_webservice.Client" + ) + _ESTIMATED_LOCATION_INFO_LOGGER = ( + "openwisp_controller.geo.estimated_location.tasks.logger.info" + ) + _ESTIMATED_LOCATION_WARNING_LOGGER = ( + "openwisp_controller.geo.estimated_location.tasks.logger.warning" + ) + _ESTIMATED_LOCATION_ERROR_LOGGER = ( + "openwisp_controller.geo.estimated_location.tasks.logger.error" + ) + _WHOIS_TASK_NAME = "openwisp_controller.config.whois.tasks.fetch_whois_details" + + def setUp(self): + super().setUp() + self.admin = self._get_admin() + # Unregister the notification type if it was previously registered + with contextlib.suppress(ImproperlyConfigured): + unregister_notification_type("estimated_location_info") + with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True): + register_estimated_location_notification_types() + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch( + "openwisp_controller.config.whois.service.WHOISService.trigger_estimated_location_task" # noqa: E501 + ) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_estimated_location_task_called( + self, mocked_client, mocked_estimated_location_task + ): + connect_whois_handlers() + mocked_response = self._mocked_client_response() + mocked_client.return_value.city.return_value = mocked_response + threshold = config_app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + 1 + + self._task_called( + mocked_estimated_location_task, task_name="Estimated location" + ) + + Device.objects.all().delete() + WHOISInfo.objects.all().delete() + org = self._get_org() + org.config_settings.whois_enabled = True + org.config_settings.estimated_location_enabled = True + org.config_settings.save() + + with self.subTest("Estimated location task called when last_ip is public"): + with mock.patch( + "django.core.cache.cache.get", return_value=None + ) as mocked_get, mock.patch("django.core.cache.cache.set") as mocked_set: + device = self._create_device(last_ip="172.217.22.14") + mocked_estimated_location_task.assert_called() + expected_cache_set_calls = [ + mock.call( + f"organization_config_{org.pk}", + org.config_settings, + timeout=Config._CHECKSUM_CACHE_TIMEOUT, + ), + mock.call( + f"{self._WHOIS_TASK_NAME}_last_operation", "success", None + ), + ] + mocked_set.assert_has_calls(expected_cache_set_calls) + mocked_get.assert_called() + mocked_estimated_location_task.reset_mock() + + with self.subTest( + "Estimated location task called when last_ip is changed and is public" + ): + with mock.patch("django.core.cache.cache.get") as mocked_get, mock.patch( + "django.core.cache.cache.set" + ) as mocked_set: + device.last_ip = "172.217.22.10" + device.save() + device.refresh_from_db() + mocked_estimated_location_task.assert_called() + expected_cache_set_calls = [ + mock.call( + f"{self._WHOIS_TASK_NAME}_last_operation", "success", None + ), + ] + mocked_set.assert_has_calls(expected_cache_set_calls) + mocked_get.assert_called() + mocked_estimated_location_task.reset_mock() + + with self.subTest( + "Estimated location task called when last_ip has related WhoIsInfo" + ): + with mock.patch("django.core.cache.cache.get") as mocked_get, mock.patch( + "django.core.cache.cache.set" + ) as mocked_set: + self._create_config(device=device) + device.last_ip = "172.217.22.14" + self._create_whois_info(ip_address=device.last_ip) + device.save() + device.refresh_from_db() + device.organization.config_settings.refresh_from_db() + mocked_set.assert_not_called() + # The cache `get` is called twice, once for `whois_enabled` and + # once for `estimated_location_enabled` + mocked_get.assert_called() + mocked_estimated_location_task.assert_called() + mocked_estimated_location_task.reset_mock() + + with self.subTest( + "Estimated location task not called via DeviceChecksumView when " + "last_ip already has related WhoIsInfo" + ): + WHOISInfo.objects.all().delete() + self._create_whois_info(ip_address=device.last_ip) + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_estimated_location_task.assert_not_called() + mocked_estimated_location_task.reset_mock() + + with self.subTest( + "Estimate location task not called when address/coordinates not updated" + ): + WHOISInfo.objects.all().delete() + whois_obj = self._create_whois_info(ip_address=device.last_ip) + WHOISInfo.objects.filter(pk=whois_obj.pk).update( + modified=timezone.now() - timedelta(days=threshold) + ) + self._create_object_location(content_object=device) + device.save() + device.refresh_from_db() + mocked_estimated_location_task.assert_not_called() + mocked_estimated_location_task.reset_mock() + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_estimated_location_task.assert_not_called() + mocked_estimated_location_task.reset_mock() + + with self.subTest( + "Estimate location task called when address/coordinates updated" + ): + WHOISInfo.objects.all().delete() + whois_obj = self._create_whois_info(ip_address=device.last_ip) + WHOISInfo.objects.filter(pk=whois_obj.pk).update( + modified=timezone.now() - timedelta(days=threshold) + ) + mocked_response.city.name = "New city" + mocked_client.return_value.city.return_value = mocked_response + device.save() + device.refresh_from_db() + mocked_estimated_location_task.assert_called() + mocked_estimated_location_task.reset_mock() + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch("openwisp_controller.config.whois.service.send_whois_task_notification") + @mock.patch( + "openwisp_controller.geo.estimated_location.tasks.send_whois_task_notification" + ) + @mock.patch( + "openwisp_controller.config.whois.service.WHOISService.trigger_estimated_location_task" # noqa: E501 + ) + @mock.patch(_ESTIMATED_LOCATION_INFO_LOGGER) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_estimated_location_creation_and_update( + self, mock_client, mock_info, _mocked_task, _mocked_notify, _mocked_notify2 + ): + connect_whois_handlers() + + def _verify_location_details(device, mocked_response): + location = device.devicelocation.location + mocked_location = mocked_response.location + mock_address = { + "city": mocked_response.city.name, + "country": mocked_response.country.name, + "continent": mocked_response.continent.name, + "postal": mocked_response.postal.code, + } + address = WHOISInfo(address=mock_address).formatted_address + ip_address = mocked_response.ip_address or device.last_ip + location_name = WHOISInfo( + address=mock_address, ip_address=ip_address + )._location_name + self.assertEqual(location.name, location_name) + self.assertEqual(location.address, address) + self.assertEqual( + location.geometry, + GEOSGeometry( + f"POINT({mocked_location.longitude} {mocked_location.latitude})", + srid=4326, + ), + ) + + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + + with self.subTest("Test Estimated location created when device is created"): + device = self._create_device(last_ip="172.217.22.14") + with self.assertNumQueries(13): + manage_estimated_locations(device.pk, device.last_ip) + location = device.devicelocation.location + mocked_response.ip_address = device.last_ip + self.assertEqual(location.is_estimated, True) + self.assertEqual(location.is_mobile, False) + self.assertEqual(location.type, "outdoor") + _verify_location_details(device, mocked_response) + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device.pk}" + f" for IP: {device.last_ip}" + ) + mock_info.reset_mock() + + with self.subTest("Test Estimated location updated when last ip is updated"): + device.last_ip = "172.217.22.10" + mocked_response.location.latitude = 50 + mocked_response.location.longitude = 150 + mocked_response.city.name = "New City" + mock_client.return_value.city.return_value = mocked_response + device.save() + device.refresh_from_db() + with self.assertNumQueries(7): + manage_estimated_locations(device.pk, device.last_ip) + + location = device.devicelocation.location + mocked_response.ip_address = device.last_ip + self.assertEqual(location.is_estimated, True) + self.assertEqual(location.is_mobile, False) + self.assertEqual(location.type, "outdoor") + _verify_location_details(device, mocked_response) + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device.pk}" + f" for IP: {device.last_ip}" + ) + mock_info.reset_mock() + + with self.subTest("Test Estimated location Name when address not available"): + device.last_ip = "172.217.22.11" + mocked_response.city.name = "" + mocked_response.country.name = "" + mocked_response.continent.name = "" + mocked_response.postal.code = "" + mock_client.return_value.city.return_value = mocked_response + device.save() + device.refresh_from_db() + with self.assertNumQueries(7): + manage_estimated_locations(device.pk, device.last_ip) + + location = device.devicelocation.location + mocked_response.ip_address = device.last_ip + self.assertEqual(location.is_estimated, True) + self.assertEqual(location.is_mobile, False) + _verify_location_details(device, mocked_response) + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device.pk}" + f" for IP: {device.last_ip}" + ) + mock_info.reset_mock() + + with self.subTest( + "Test Non Estimated Location not updated when last ip is updated" + ): + mocked_response.ip_address = device.last_ip + device.last_ip = "172.217.22.12" + device.devicelocation.location.is_estimated = False + mock_client.return_value.city.return_value = self._mocked_client_response() + device.devicelocation.location.save(_set_estimated=True) + device.save() + device.refresh_from_db() + with self.assertNumQueries(2): + manage_estimated_locations(device.pk, device.last_ip) + + location = device.devicelocation.location + self.assertEqual(location.is_estimated, False) + self.assertEqual(location.is_mobile, False) + self.assertEqual(location.type, "outdoor") + _verify_location_details(device, mocked_response) + mock_info.assert_called_once_with( + f"Non Estimated location already set for {device.pk}. Update" + f" location manually as per IP: {device.last_ip}" + ) + mock_info.reset_mock() + + with self.subTest( + "Test location shared for same IP when new device's location does not exist" + ): + Device.objects.all().delete() + device1 = self._create_device(last_ip="172.217.22.10") + manage_estimated_locations(device1.pk, device1.last_ip) + mock_info.reset_mock() + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.10", + ) + with self.assertNumQueries(7): + manage_estimated_locations(device2.pk, device2.last_ip) + + self.assertEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device2.pk}" + f" for IP: {device2.last_ip}" + ) + mock_info.reset_mock() + + with self.subTest( + "Test location shared for same IP when new device's location is estimated" + ): + Device.objects.all().delete() + device1 = self._create_device(last_ip="172.217.22.10") + manage_estimated_locations(device1.pk, device1.last_ip) + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.11", + ) + manage_estimated_locations(device2.pk, device2.last_ip) + mock_info.reset_mock() + old_location = device2.devicelocation.location + device2.last_ip = "172.217.22.10" + device2.save() + # 3 queries related to notifications cleanup + device2.refresh_from_db() + with self.assertNumQueries(15): + manage_estimated_locations(device2.pk, device2.last_ip) + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device2.pk}" + f" for IP: {device2.last_ip}" + ) + + self.assertEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + self.assertEqual(Location.objects.filter(pk=old_location.pk).count(), 0) + mock_info.reset_mock() + + with self.subTest( + "Test location not shared for same IP when new " + "device's location is not estimated" + ): + Device.objects.all().delete() + device1 = self._create_device(last_ip="172.217.22.10") + manage_estimated_locations(device1.pk, device1.last_ip) + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.11", + ) + manage_estimated_locations(device2.pk, device2.last_ip) + mock_info.reset_mock() + old_location = device2.devicelocation.location + old_location.is_estimated = False + old_location.save() + device2.last_ip = "172.217.22.10" + device2.save() + device2.refresh_from_db() + with self.assertNumQueries(2): + manage_estimated_locations(device2.pk, device2.last_ip) + mock_info.assert_called_once_with( + f"Non Estimated location already set for {device2.pk}. Update" + f" location manually as per IP: {device2.last_ip}" + ) + + self.assertNotEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + self.assertEqual(Location.objects.filter(pk=old_location.pk).count(), 1) + mock_info.reset_mock() + + with self.subTest( + "Shared location not updated when either device's last_ip changes. " + "New location created for device with updated last_ip" + ): + Device.objects.all().delete() + device1 = self._create_device(last_ip="172.217.22.10") + manage_estimated_locations(device1.pk, device1.last_ip) + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.10", + ) + manage_estimated_locations(device2.pk, device2.last_ip) + mock_info.reset_mock() + self.assertEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + device2.last_ip = "172.217.22.11" + device2.save() + device2.refresh_from_db() + with self.assertNumQueries(13): + manage_estimated_locations(device2.pk, device2.last_ip) + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device2.pk}" + f" for IP: {device2.last_ip}" + ) + self.assertNotEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + mock_info.reset_mock() + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_ESTIMATED_LOCATION_WARNING_LOGGER) + def test_manage_estimated_location_device_not_found(self, mock_warn): + invalid_pk = uuid4() + manage_estimated_locations(device_pk=invalid_pk, ip_address="10.0.0.1") + mock_warn.assert_called_once_with( + f"Device {invalid_pk} not found, skipping manage_estimated_locations" + ) + + @mock.patch( + "openwisp_controller.config.whois.service.current_app.send_task", + side_effect=TestEstimatedLocationMixin.run_task, + ) + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_estimated_location_handling_on_whois_update( + self, mock_client, mock_send_task + ): + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + threshold = config_app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + 1 + new_time = timezone.now() - timedelta(days=threshold) + org = self._get_org() + org.config_settings.estimated_location_enabled = False + org.config_settings.save() + device = self._create_device(last_ip="172.217.22.10") + with self.assertRaises(Device.devicelocation.RelatedObjectDoesNotExist): + # Accessing devicelocation to verify it doesn't exist (raises if not) + device.devicelocation + org.config_settings.estimated_location_enabled = True + org.config_settings.save() + whois_obj = device.whois_service.get_device_whois_info() + WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) + device.name = "test.new.name" + device.save() + device.refresh_from_db() + # location created so can safely access devicelocation + # Accessing devicelocation to verify it exists (raises if not) + device.devicelocation + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch( + "openwisp_controller.config.whois.service.current_app.send_task", + side_effect=TestEstimatedLocationMixin.run_task, + ) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_unchanged_whois_data_no_location_recreation(self, mock_client, _): + """Ensure identical WHOIS results do not recreate a shared Location when + devices reuse the same IP.""" + connect_whois_handlers() + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + shared_ip = "20.49.19.19" + device1 = self._create_device( + name="device-a", + mac_address="00:11:22:33:44:55", + last_ip=shared_ip, + ) + device2 = self._create_device( + name="device-b", + mac_address="00:11:22:33:44:66", + last_ip=shared_ip, + ) + original_location = device1.devicelocation.location + self.assertEqual(original_location.pk, device2.devicelocation.location.pk) + location_count = Location.objects.count() + notification_count = _notification_qs().count() + # Clear the last ip for both devices, so setting them again + # will trigger the WHOIS lookup flow. + for device in (device1, device2): + device.last_ip = "" + device.save(update_fields=["last_ip"]) + device.refresh_from_db() + # We set the same shared IP again. This simulates device fetching checksum. + for device in (device1, device2): + device.last_ip = shared_ip + device.save(update_fields=["last_ip"]) + device.refresh_from_db() + # The location object should remain unchanged since the WHOIS data is the same. + self.assertEqual(original_location.pk, device1.devicelocation.location.pk) + self.assertEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + self.assertEqual(Location.objects.count(), location_count) + self.assertTrue(Location.objects.filter(pk=original_location.pk).exists()) + self.assertEqual(_notification_qs().count(), notification_count) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch( + "openwisp_controller.config.whois.service.current_app.send_task", + side_effect=TestEstimatedLocationMixin.run_task, + ) + @mock.patch(_ESTIMATED_LOCATION_INFO_LOGGER) + @mock.patch(_ESTIMATED_LOCATION_ERROR_LOGGER) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_estimated_location_notification( + self, mock_client, mock_error, mock_info, _ + ): + def _verify_notification(device, messages, notify_level="info"): + notification_qs = _notification_qs() + self.assertEqual(notification_qs.count(), 1) + notification = notification_qs.first() + device_location = getattr(device, "devicelocation", None) + actor = device + if device_location: + actor = device_location.location + self.assertEqual(notification.actor, actor) + self.assertEqual(notification.target, device) + self.assertEqual(notification.type, "estimated_location_info") + self.assertEqual(notification.level, notify_level) + for message in messages: + self.assertIn(message, notification.message) + self.assertIn(device.last_ip, notification.rendered_description) + self.assertIn("#devicelocation-group", notification.target_url) + + with self.subTest("Test Notification for location create"): + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + device1 = self._create_device(last_ip="172.217.22.10") + messages = ["Estimated location", "created successfully"] + _verify_notification(device1, messages) + + with self.subTest("Test Notification for location update"): + _notification_qs().delete() + # will have same location as first device + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.10", + ) + messages = ["Estimated location", "updated successfully"] + _verify_notification(device2, messages) + + with self.subTest("Test Error Notification for conflicting locations"): + _notification_qs().delete() + mock_info.reset_mock() + mock_error.reset_mock() + device3 = self._create_device( + name="11:22:33:44:55:77", + mac_address="11:22:33:44:55:77", + last_ip=device2.last_ip, + ) + mock_info.assert_not_called() + mock_error.assert_called_once_with( + f"Multiple devices with locations found with same " + f"last_ip {device3.last_ip}. Please resolve the conflict manually." + ) + messages = [ + "Unable to create estimated location for device", + "Please assign/create a location manually.", + ] + _verify_notification(device3, messages, "error") + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch("openwisp_controller.config.whois.service.send_whois_task_notification") + @mock.patch( + "openwisp_controller.geo.estimated_location.tasks.send_whois_task_notification" + ) + @mock.patch( + "openwisp_controller.config.whois.service.WHOISService.trigger_estimated_location_task" # noqa: E501 + ) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_manage_estimated_locations_no_coordinates_warning( + self, mock_client, _mocked_task, _mocked_notify, _mocked_notify2 + ): + with mock.patch.object(estimated_location.tasks.logger, "warning") as mock_warn: + connect_whois_handlers() + mock_client.return_value.city.return_value = self._mocked_client_response() + device = self._create_device(last_ip="172.217.22.14") + WHOISInfo.objects.filter(ip_address=device.last_ip).delete() + WHOISInfo.objects.create( + ip_address=device.last_ip, + address={ + "city": "Mountain View", + "country": "United States", + "postal": "94043", + }, + ) + manage_estimated_locations(device.pk, device.last_ip) + mock_warn.assert_called_once_with( + f"Coordinates not available for {device.pk} for IP: {device.last_ip}. " + "Estimated location cannot be determined." + ) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch( + "openwisp_controller.config.whois.service.current_app.send_task", + side_effect=TestEstimatedLocationMixin.run_task, + ) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_estimate_location_status_remove(self, mock_client, _): + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + device = self._create_device(last_ip="172.217.22.10") + location = device.devicelocation.location + self.assertTrue(location.is_estimated) + org = self._get_org() + connect_whois_handlers() + + with self.subTest( + "Test Estimated Status unchanged if Estimated feature is disabled" + ): + org.config_settings.estimated_location_enabled = False + org.config_settings.save() + org.config_settings.refresh_from_db() + location.geometry = GEOSGeometry("POINT(12.512124 41.898903)", srid=4326) + location.save() + location.refresh_from_db() + self.assertTrue(location.is_estimated) + self.assertIn(f": {device.last_ip}", location.name) + + with self.subTest( + "Test Estimated Status unchanged if Estimated feature is enabled" + " and desired fields not changed" + ): + org.config_settings.estimated_location_enabled = True + org.config_settings.save() + org.config_settings.refresh_from_db() + location._set_initial_values_for_changed_checked_fields() + location.type = "outdoor" + location.is_mobile = True + location.save() + location.refresh_from_db() + self.assertTrue(location.is_estimated) + + with self.subTest( + "Test Estimated Status changed if Estimated feature is enabled" + " and desired fields changed" + ): + location.geometry = GEOSGeometry("POINT(15.512124 45.898903)", srid=4326) + location.save(update_fields=["geometry"]) + location.refresh_from_db() + self.assertFalse(location.is_estimated) + # Note: Name is no longer automatically cleaned up when + # is_estimated becomes False. Users must update the name manually + # if desired + + +class TestEstimatedLocationFieldFilters( + TestEstimatedLocationMixin, TestGeoMixin, TestCase +): + location_model = Location + object_location_model = DeviceLocation + + def setUp(self): + super().setUp() + admin = self._create_admin() + self.client.force_login(admin) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + def test_estimated_location_api_status_configured(self): + org1 = self._get_org() + org2 = self._create_org(name="org2") + OrganizationConfigSettings.objects.create( + organization=org2, + whois_enabled=False, + estimated_location_enabled=False, + ) + org1_location = self._create_location( + name="org1-location", organization=org1, is_estimated=True + ) + org2_location = self._create_location(name="org2-location", organization=org2) + org1_device = self._create_device(organization=org1) + org2_device = self._create_device(organization=org2) + self._create_object_location(content_object=org1_device, location=org1_location) + self._create_object_location(content_object=org2_device, location=org2_location) + + with self.subTest("Test Estimated Location in Locations List"): + path = reverse("geo_api:list_location") + with self.assertNumQueries(4): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 2) + self.assertContains(response, org1_location.id) + self.assertContains(response, org2_location.id) + results_by_id = {item["id"]: item for item in response.data["results"]} + self.assertIn("is_estimated", results_by_id[str(org1_location.id)]) + self.assertNotIn("is_estimated", results_by_id[str(org2_location.id)]) + + with self.subTest("Test Estimated Location in Device Locations List"): + path = reverse("geo_api:device_location", args=[org1_device.pk]) + with self.assertNumQueries(4): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertIn("is_estimated", response.data["location"]["properties"]) + path = reverse("geo_api:device_location", args=[org2_device.pk]) + with self.assertNumQueries(4): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertNotIn("is_estimated", response.data["location"]["properties"]) + + with self.subTest("Test Estimated Location in GeoJSON List"): + path = reverse("geo_api:location_geojson") + with self.assertNumQueries(3): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 2) + results_by_id = {item["id"]: item for item in response.data["features"]} + location1_result = results_by_id[str(org1_location.id)] + location2_result = results_by_id[str(org2_location.id)] + self.assertIn("is_estimated", location1_result["properties"]) + self.assertTrue(location1_result["properties"]["is_estimated"]) + self.assertNotIn("is_estimated", location2_result["properties"]) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False) + def test_estimated_location_api_status_not_configured(self): + org = self._get_org() + location = self._create_location(name="org1-location", organization=org) + device = self._create_device(organization=org) + self._create_object_location(content_object=device, location=location) + + with self.subTest("Test Estimated status not in Locations List"): + path = reverse("geo_api:list_location") + with self.assertNumQueries(4): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 1) + self.assertContains(response, location.id) + api_location = response.data["results"][0] + self.assertNotIn("is_estimated", api_location) + + with self.subTest("Test Estimated status not in Device Locations List"): + path = reverse("geo_api:device_location", args=[device.pk]) + with self.assertNumQueries(4): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertNotIn("is_estimated", response.data["location"]["properties"]) + + with self.subTest("Test Estimated status not in GeoJSON Location List"): + path = reverse("geo_api:location_geojson") + with self.assertNumQueries(3): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 1) + location_features = response.data["features"][0] + self.assertNotIn("is_estimated", location_features["properties"]) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + def test_estimated_location_filter_list_api(self): + org = self._get_org() + location1 = self._create_location( + name="location1", is_estimated=True, organization=org + ) + location2 = self._create_location( + name="location2", is_estimated=False, organization=org + ) + location3 = self._create_location( + name="location3", is_estimated=False, organization=org + ) + device1 = self._create_device() + device2 = self._create_device( + name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66" + ) + self._create_object_location(content_object=device1, location=location1) + self._create_object_location(content_object=device2, location=location2) + path = reverse("geo_api:list_location") + + with self.subTest( + "Test Estimated Location filter available in location list " + "when WHOIS is configured" + ): + with self.assertNumQueries(4): + response = self.client.get(path, {"is_estimated": True}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 1) + self.assertContains(response, location1.id) + self.assertNotContains(response, location2.id) + self.assertNotContains(response, location3.id) + + with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False): + with self.subTest( + "Test Estimated Location filter not available in location list " + "when WHOIS not configured" + ): + with self.assertNumQueries(4): + response = self.client.get(path, {"is_estimated": True}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 3) + self.assertContains(response, location1.id) + self.assertContains(response, location2.id) + + path = reverse("config_api:device_list") + + with self.subTest( + "Test Estimated Location filter available in device list " + "when WHOIS is configured" + ): + with self.assertNumQueries(3): + response = self.client.get(path, {"geo_is_estimated": True}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 1) + self.assertContains(response, device1.id) + self.assertNotContains(response, device2.id) + + with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False): + with self.subTest( + "Test Estimated Location filter not available in device list " + "when WHOIS not configured" + ): + with self.assertNumQueries(3): + response = self.client.get(path, {"geo_is_estimated": True}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 2) + self.assertContains(response, device1.id) + self.assertContains(response, device2.id) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + def test_location_admin_estimated_field(self): + org = self._get_org() + estimated_location = self._create_location( + name="location1", is_estimated=True, organization=org + ) + estimated_device = self._create_device() + self._create_object_location( + content_object=estimated_device, location=estimated_location + ) + path = reverse("admin:geo_location_changelist") + response = self.client.get(path) + + with self.subTest("Test location Admin estimated field displayed"): + self.assertContains(response, "column-is_estimated") + + with self.subTest("Test location admin estimated field sorting enabled"): + self.assertContains(response, "sortable column-is_estimated") + + with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False): + response = self.client.get(path) + with self.subTest( + "Test location admin estimated field not displayed " + "when WHOIS not configured" + ): + self.assertNotContains(response, "column-is_estimated") + + with self.subTest( + "Test location admin estimated field sorting not enabled " + "when WHOIS not configured" + ): + self.assertNotContains(response, "sortable column-is_estimated") + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + def test_estimated_location_filter_admin(self): + org = self._get_org() + estimated_location = self._create_location( + name="location1", is_estimated=True, organization=org + ) + outdoor_location = self._create_location(name="location2", organization=org) + indoor_location = self._create_location( + name="location3", organization=org, type="indoor" + ) + estimated_device = self._create_device() + outdoor_device = self._create_device( + name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66" + ) + indoor_device = self._create_device( + name="11:22:33:44:55:77", mac_address="11:22:33:44:55:77" + ) + self._create_object_location( + content_object=estimated_device, location=estimated_location + ) + self._create_object_location( + content_object=outdoor_device, location=outdoor_location + ) + self._create_object_location( + content_object=indoor_device, location=indoor_location + ) + path = reverse("admin:config_device_changelist") + + with self.subTest("Test All Locations Filter"): + response = self.client.get(path) + self.assertContains(response, estimated_device.id) + self.assertContains(response, outdoor_device.id) + self.assertContains(response, indoor_device.id) + self.assertContains(response, "3 Devices") + + with self.subTest("Test Estimated Location Filter"): + response = self.client.get(path, {"with_geo": "estimated"}) + self.assertContains(response, estimated_device.id) + self.assertNotContains(response, outdoor_device.id) + self.assertNotContains(response, indoor_device.id) + self.assertContains(response, "1 Device") + + with self.subTest("Test Outdoor Location Filter"): + response = self.client.get(path, {"with_geo": "outdoor"}) + self.assertContains(response, outdoor_device.id) + self.assertNotContains(response, estimated_device.id) + self.assertNotContains(response, indoor_device.id) + self.assertContains(response, "1 Device") + + with self.subTest("Test Indoor Location Filter"): + response = self.client.get(path, {"with_geo": "indoor"}) + self.assertContains(response, indoor_device.id) + self.assertNotContains(response, outdoor_device.id) + self.assertNotContains(response, estimated_device.id) + self.assertContains(response, "1 Device") + + with self.subTest("Test No Location Filter"): + response = self.client.get(path, {"with_geo": "false"}) + self.assertNotContains(response, indoor_device.id) + self.assertNotContains(response, outdoor_device.id) + self.assertNotContains(response, estimated_device.id) + self.assertContains(response, "0 Devices") + + with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False): + with self.subTest( + "Test Estimated Location Admin specific filters not available" + " when WHOIS not configured" + ): + for filter_value in ["estimated", "outdoor", "indoor"]: + with self.subTest(filter_value=filter_value): + response = self.client.get(path, {"with_geo": filter_value}) + self.assertContains(response, estimated_device.id) + self.assertContains(response, outdoor_device.id) + self.assertContains(response, indoor_device.id) + self.assertContains(response, "3 Devices") diff --git a/openwisp_controller/geo/estimated_location/tests/utils.py b/openwisp_controller/geo/estimated_location/tests/utils.py new file mode 100644 index 000000000..0ca26f335 --- /dev/null +++ b/openwisp_controller/geo/estimated_location/tests/utils.py @@ -0,0 +1,34 @@ +from swapper import load_model + +from openwisp_controller.config.whois.tasks import fetch_whois_details +from openwisp_controller.config.whois.tests.utils import CreateWHOISMixin + +from ..tasks import manage_estimated_locations + +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + + +class TestEstimatedLocationMixin(CreateWHOISMixin): + def setUp(self): + # skip the org config settings creation from the parent mixin + super(CreateWHOISMixin, self).setUp() + OrganizationConfigSettings.objects.create( + organization=self._get_org(), + whois_enabled=True, + estimated_location_enabled=True, + ) + + # helper to mock send_task to directly call the task function synchronously + @staticmethod + def run_task(name, args=None, kwargs=None, **_): + # This mock intercepts all Celery send_task calls and executes them + # synchronously for testing purposes. We need to handle both task names + # because when a device is created/saved, the WHOIS lookup task may be + # triggered first, which then triggers the estimated location task. + # Without handling both, the assertion would fail and break the test flow. + if name == "whois_estimated_location_task": + return manage_estimated_locations(*args or [], **kwargs or {}) + elif name == "openwisp_controller.config.whois.tasks.fetch_whois_details": + return fetch_whois_details(*args or [], **kwargs or {}) + # Let other tasks pass through (they may be mocked elsewhere) + return None diff --git a/openwisp_controller/geo/estimated_location/utils.py b/openwisp_controller/geo/estimated_location/utils.py new file mode 100644 index 000000000..f7516afdc --- /dev/null +++ b/openwisp_controller/geo/estimated_location/utils.py @@ -0,0 +1,49 @@ +from django.utils.translation import gettext_lazy as _ + +from openwisp_controller.config.whois.utils import MESSAGE_MAP + +# Mutating the existing MESSAGE_MAP to include estimated location messages +MESSAGE_MAP.update( + { + "estimated_location_error": { + "level": "error", + "type": "estimated_location_info", + "message": _( + "Unable to create estimated location for device " + "[{notification.target}]({notification.target_link}). " + "Please assign/create a location manually." + ), + "description": _("Multiple devices found for IP: {ip_address}"), + }, + "estimated_location_created": { + "type": "estimated_location_info", + "level": "info", + "message": _( + "Estimated location [{notification.actor}]({notification.actor_link})" + " for device" + " [{notification.target}]({notification.target_link})" + " {notification.verb} successfully." + ), + "description": _("Geographic coordinates inferred from IP: {ip_address}"), + }, + "estimated_location_updated": { + "type": "estimated_location_info", + "level": "info", + "message": _( + "Estimated location [{notification.actor}]({notification.actor_link})" + " for device" + " [{notification.target}]({notification.target_link})" + " updated successfully." + ), + "description": _("Geographic coordinates updated for IP: {ip_address}"), + }, + } +) + + +def get_device_location_notification_target_url(obj, field, absolute_url=True): + # importing here to avoid "AppRegistryNotReady" + from openwisp_notifications.utils import _get_object_link + + url = _get_object_link(obj._related_object(field), absolute_url) + return f"{url}#devicelocation-group" diff --git a/openwisp_controller/geo/migrations/0004_location_is_estimated.py b/openwisp_controller/geo/migrations/0004_location_is_estimated.py new file mode 100644 index 000000000..45ac400da --- /dev/null +++ b/openwisp_controller/geo/migrations/0004_location_is_estimated.py @@ -0,0 +1,22 @@ +# Generated by Django 5.2.1 on 2025-06-25 19:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("geo", "0003_alter_devicelocation_floorplan_location"), + ] + + operations = [ + migrations.AddField( + model_name="location", + name="is_estimated", + field=models.BooleanField( + default=False, + help_text=("Whether the location's coordinates are estimated."), + verbose_name="Is Estimated?", + ), + ), + ] diff --git a/openwisp_controller/geo/templates/admin/geo/location/change_form.html b/openwisp_controller/geo/templates/admin/geo/location/change_form.html new file mode 100644 index 000000000..5c1bd85a4 --- /dev/null +++ b/openwisp_controller/geo/templates/admin/geo/location/change_form.html @@ -0,0 +1,11 @@ +{% extends "admin/django_loci/location_change_form.html" %} +{% load i18n %} + +{% block messages %} + {{ block.super }} + {% if original and estimated_enabled and original.is_estimated %} +
    +
  • {% trans "This location is estimated based on the device's last IP address. Please refine it for greater accuracy." %}
    • +
+ {% endif %} +{% endblock messages %} diff --git a/openwisp_controller/geo/tests/test_admin.py b/openwisp_controller/geo/tests/test_admin.py index 8d914dc8d..06a498cfe 100644 --- a/openwisp_controller/geo/tests/test_admin.py +++ b/openwisp_controller/geo/tests/test_admin.py @@ -137,6 +137,16 @@ def test_admin_menu_groups(self): html=True, ) + def test_location_readonly_fields(self): + location = self._create_location( + name="location1org", type="indoor", organization=self._get_org() + ) + self._create_admin() + self._login() + url = reverse(f"admin:{self.app_label}_location_change", args=[location.pk]) + response = self.client.get(url) + self.assertNotContains(response, '=3.1.0,<4.0.0 +geoip2>=5.1.0,<6.0.0 diff --git a/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py b/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py new file mode 100644 index 000000000..d9125877c --- /dev/null +++ b/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py @@ -0,0 +1,77 @@ +# Generated by Django 5.2.1 on 2025-06-26 17:48 + +import django.utils.timezone +import model_utils.fields +from django.db import migrations, models + +import openwisp_utils.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("sample_config", "0007_alter_config_status"), + ] + + operations = [ + migrations.CreateModel( + name="WHOISInfo", + fields=[ + ( + "created", + model_utils.fields.AutoCreatedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="created", + ), + ), + ( + "modified", + model_utils.fields.AutoLastModifiedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="modified", + ), + ), + ( + "ip_address", + models.GenericIPAddressField( + db_index=True, primary_key=True, serialize=False + ), + ), + ( + "isp", + models.CharField( + blank=True, help_text="Organization for ASN", max_length=100 + ), + ), + ("asn", models.CharField(blank=True, help_text="ASN", max_length=10)), + ( + "timezone", + models.CharField(blank=True, help_text="Time zone", max_length=35), + ), + ( + "address", + models.JSONField(blank=True, default=dict, help_text="Address"), + ), + ("cidr", models.CharField(blank=True, help_text="CIDR", max_length=49)), + ("details", models.CharField(blank=True, max_length=64, null=True)), + ], + options={ + "abstract": False, + "swappable": "CONFIG_WHOISINFO_MODEL", + }, + ), + migrations.AddField( + model_name="organizationconfigsettings", + name="whois_enabled", + field=openwisp_utils.fields.FallbackBooleanChoiceField( + blank=True, + default=None, + fallback=False, + help_text="Whether the WHOIS lookup feature is enabled", + null=True, + verbose_name="WHOIS Enabled", + ), + ), + ] diff --git a/tests/openwisp2/sample_config/migrations/0009_organizationconfigsettings_approximate_location_enabled_and_more.py b/tests/openwisp2/sample_config/migrations/0009_organizationconfigsettings_approximate_location_enabled_and_more.py new file mode 100644 index 000000000..f94921b0c --- /dev/null +++ b/tests/openwisp2/sample_config/migrations/0009_organizationconfigsettings_approximate_location_enabled_and_more.py @@ -0,0 +1,35 @@ +# Generated by Django 5.2.1 on 2025-07-11 16:28 + +import django.contrib.gis.db.models.fields +from django.db import migrations + +import openwisp_utils.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("sample_config", "0008_whoisinfo_organizationconfigsettings_whois_enabled"), + ] + + operations = [ + migrations.AddField( + model_name="organizationconfigsettings", + name="estimated_location_enabled", + field=openwisp_utils.fields.FallbackBooleanChoiceField( + blank=True, + default=None, + fallback=False, + help_text="Whether the estimated location feature is enabled", + null=True, + verbose_name="Estimated Location Enabled", + ), + ), + migrations.AddField( + model_name="whoisinfo", + name="coordinates", + field=django.contrib.gis.db.models.fields.PointField( + blank=True, help_text="Coordinates", null=True, srid=4326 + ), + ), + ] diff --git a/tests/openwisp2/sample_config/models.py b/tests/openwisp2/sample_config/models.py index 8fec929db..e0e6448a5 100644 --- a/tests/openwisp2/sample_config/models.py +++ b/tests/openwisp2/sample_config/models.py @@ -13,6 +13,7 @@ ) from openwisp_controller.config.base.template import AbstractTemplate from openwisp_controller.config.base.vpn import AbstractVpn, AbstractVpnClient +from openwisp_controller.config.base.whois import AbstractWHOISInfo class DetailsModel(models.Model): @@ -111,3 +112,12 @@ class OrganizationLimits(DetailsModel, AbstractOrganizationLimits): class Meta(AbstractOrganizationLimits.Meta): abstract = False + + +class WHOISInfo(DetailsModel, AbstractWHOISInfo): + """ + Stores WHOIS information for devices. + """ + + class Meta(AbstractWHOISInfo.Meta): + abstract = False diff --git a/tests/openwisp2/sample_geo/migrations/0004_location_is_estimated.py b/tests/openwisp2/sample_geo/migrations/0004_location_is_estimated.py new file mode 100644 index 000000000..1c2e65e19 --- /dev/null +++ b/tests/openwisp2/sample_geo/migrations/0004_location_is_estimated.py @@ -0,0 +1,22 @@ +# Generated by Django 5.2.1 on 2025-07-01 19:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("sample_geo", "0003_alter_devicelocation_floorplan_location"), + ] + + operations = [ + migrations.AddField( + model_name="location", + name="is_estimated", + field=models.BooleanField( + default=False, + help_text="Whether the location's coordinates are estimated.", + verbose_name="Is Estimated?", + ), + ), + ] diff --git a/tests/openwisp2/settings.py b/tests/openwisp2/settings.py index c46e3e50a..ea1a79a56 100644 --- a/tests/openwisp2/settings.py +++ b/tests/openwisp2/settings.py @@ -208,6 +208,14 @@ OPENWISP_CONTROLLER_CONTEXT = {"vpnserver1": "vpn.testdomain.com"} OPENWISP_USERS_AUTH_API = True +# GEOIP Related Settings +OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT = os.getenv( + "OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT", "" +) +OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY = os.getenv( + "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY", "" +) + TEST_RUNNER = "openwisp_utils.tests.TimeLoggingTestRunner" if os.environ.get("SAMPLE_APP", False): @@ -266,6 +274,7 @@ CONFIG_VPNCLIENT_MODEL = "sample_config.VpnClient" CONFIG_ORGANIZATIONCONFIGSETTINGS_MODEL = "sample_config.OrganizationConfigSettings" CONFIG_ORGANIZATIONLIMITS_MODEL = "sample_config.OrganizationLimits" + CONFIG_WHOISINFO_MODEL = "sample_config.WHOISInfo" DJANGO_X509_CA_MODEL = "sample_pki.Ca" DJANGO_X509_CERT_MODEL = "sample_pki.Cert" GEO_LOCATION_MODEL = "sample_geo.Location"