Morango upgrade and sync queue tweaks

kolibri/core/auth/models.py

@@ -1463,6 +1463,7 @@ class Facility(Collection):
     _KIND = collection_kinds.FACILITY
 
     objects = CollectionProxyManager()
+    syncing_objects = CollectionProxyManager()
 
     class Meta:
         proxy = True
@@ -1584,6 +1585,7 @@ class Classroom(Collection):
     _KIND = collection_kinds.CLASSROOM
 
     objects = CollectionProxyManager()
+    syncing_objects = CollectionProxyManager()
 
     class Meta:
         proxy = True
@@ -1654,6 +1656,7 @@ class LearnerGroup(Collection):
     _KIND = collection_kinds.LEARNERGROUP
 
     objects = CollectionProxyManager()
+    syncing_objects = CollectionProxyManager()
 
     class Meta:
         proxy = True
@@ -1701,6 +1704,7 @@ class AdHocGroup(Collection):
     _KIND = collection_kinds.ADHOCLEARNERSGROUP
 
     objects = CollectionProxyManager()
+    syncing_objects = CollectionProxyManager()
 
     class Meta:
         proxy = True

kolibri/core/auth/test/test_morango_integration.py

@@ -705,6 +705,18 @@ def test_single_user_sync(self, servers):
             2,
         )
 
+        # Mark learner1 as soft deleted on s0
+        learner1.date_deleted = timezone.now()
+        learner1.save(using=s0.db_alias)
+
+        # Sync from s1 to s0 with the now soft-deleted learner1
+        s1.sync(s0, facility, user=learner1)
+
+        # Assert that learner1 on s1 is now also marked as soft deleted
+        self.assertIsNotNone(
+            FacilityUser.all_objects.using(s1.db_alias).get(id=learner1.id).date_deleted
+        )
+
     @multiple_kolibri_servers(2)
     def test_single_user_sync_resumption(self, servers):
         self.maxDiff = None

kolibri/core/device/signals.py

@@ -1,3 +1,4 @@
+from django.db import transaction
 from django.db.models.signals import post_delete
 from django.db.models.signals import post_save
 from django.dispatch import receiver
@@ -12,7 +13,11 @@ def sync_queue_delete_update_user_sync_status(sender, instance=None, *args, **kw
     When a sync queue object is deleted, we update the user sync status, since it's an aggregate
     of all the sync queue objects for a given user.
     """
-    UserSyncStatus.update_status(instance.user_id)
+
+    def update_status_after_commit():
+        UserSyncStatus.update_status(instance.user_id)
+
+    transaction.on_commit(update_status_after_commit)
 
 
 @receiver(post_save, sender=SyncQueue)
@@ -21,4 +26,8 @@ def sync_queue_save_update_user_sync_status(sender, instance=None, *args, **kwar
     When a sync queue object is saved, we update the user sync status, since it's an aggregate
     of all the sync queue objects for a given user.
     """
-    UserSyncStatus.update_status(instance.user_id)
+
+    def update_status_after_commit():
+        UserSyncStatus.update_status(instance.user_id)
+
+    transaction.on_commit(update_status_after_commit)

kolibri/core/device/soud.py

@@ -165,9 +165,11 @@ def get_all_user_ids():
         if is_full_facility_import(dataset_id):
             continue
 
-        for user_id in FacilityUser.objects.filter(dataset_id=dataset_id).values_list(
-            "id", flat=True
-        ):
+        # Include users that are soft-deleted, to allow for either propagation of deletions
+        # or restoration of users on the server to be synced to the client.
+        for user_id in FacilityUser.all_objects.filter(
+            dataset_id=dataset_id
+        ).values_list("id", flat=True):
             yield user_id
 
 

kolibri/core/public/api.py

@@ -15,6 +15,9 @@
 from django.views.decorators.csrf import csrf_exempt
 from django.views.decorators.gzip import gzip_page
 from django_filters.rest_framework import DjangoFilterBackend
+from morango.models import DeletedModels
+from morango.models import HardDeletedModels
+from morango.models import Store
 from rest_framework import exceptions
 from rest_framework import filters
 from rest_framework import serializers
@@ -300,7 +303,14 @@ def create_or_update(self, request):
         user_id = serializer.validated_data["user"]
         instance_id = serializer.validated_data["instance"]
 
-        if not FacilityUser.objects.filter(id=user_id).exists():
+        if (
+            not FacilityUser.all_objects.filter(id=user_id).exists()
+            and not DeletedModels.objects.filter(id=user_id).exists()
+            and not HardDeletedModels.objects.filter(id=user_id).exists()
+            and not Store.objects.filter(id=user_id)
+            .filter(Q(deleted=True) | Q(hard_deleted=True))
+            .exists()
+        ):
             content = "This user is not registered in any of this server facilities"
             raise exceptions.NotFound(content)
 

kolibri/core/public/test/test_api.py

@@ -10,12 +10,16 @@
 from morango.constants import transfer_stages
 from morango.constants import transfer_statuses
 from morango.models import InstanceIDModel
+from morango.models import ScopeDefinition
 from morango.models import SyncSession
 from morango.models import TransferSession
+from morango.sync.controller import MorangoProfileController
 from rest_framework import status
 from rest_framework.test import APITestCase
+from rest_framework.test import APITransactionTestCase
 
 import kolibri
+from kolibri.core.auth.constants.morango_sync import ScopeDefinitions
 from kolibri.core.auth.models import Facility
 from kolibri.core.auth.models import FacilityUser
 from kolibri.core.auth.test.helpers import provision_device
@@ -276,6 +280,26 @@ def test_public_filter_unlisted(self):
         self.assertEqual(len(data), 2)
 
 
+def get_single_user_sync_filter(user):
+    """
+    Helper function to programmatically generate single user sync filter partitions.
+    This uses the Morango ScopeDefinition to create proper partitions for single user syncing.
+
+    :param user: FacilityUser instance
+    :return: List of partition strings for single user syncing
+    """
+    scope_params = {
+        "dataset_id": user.dataset_id,
+        "user_id": user.id,
+    }
+    scope_def = ScopeDefinition.objects.get(id=ScopeDefinitions.SINGLE_USER)
+    scope = scope_def.get_scope(scope_params)
+    # Return the read_filter, as the tests below are for serialization of a
+    # deletion of a single user synced user on a full facility device to be synced
+    # to a single user device.
+    return scope.read_filter
+
+
 class SyncQueueViewSetTestCase(APITestCase):
     """
     IMPORTANT: These tests are to never be changed. They are enforcing a
@@ -557,3 +581,157 @@ def test_update_full_queue_should_scale_keep_alive(self):
         data = response.json()
         self.assertEqual(response.status_code, 200)
         self.assertEqual(data["keep_alive"], 3 * HANDSHAKING_TIME)
+
+
+class SyncQueueViewSetDeletedUsersTestCase(APITransactionTestCase):
+    """
+    IMPORTANT: These tests are to never be changed. They are enforcing a
+    public API contract. If the tests fail, then the implementation needs
+    to be changed, and not the tests themselves.
+    """
+
+    databases = "__all__"
+
+    def setUp(self):
+        setup_device()
+        self.facility = Facility.get_default_facility()
+        self.learner = FacilityUser.objects.create(
+            username="test",
+            password="***",
+            facility=self.facility,
+        )
+        self.instance_id = uuid.uuid4().hex
+
+    def test_create_soft_deleted_user_should_sync(self):
+        soft_deleted_user = FacilityUser.objects.create(
+            username="soft_deleted",
+            password="***",
+            facility=self.facility,
+        )
+        user_filter = get_single_user_sync_filter(soft_deleted_user)
+        controller = MorangoProfileController(FacilityUser.morango_profile)
+        controller.serialize_into_store(user_filter)
+        # Soft delete the user by setting date_deleted
+        soft_deleted_user.date_deleted = timezone.now()
+        soft_deleted_user.save()
+
+        response = self.client.post(
+            reverse("kolibri:core:syncqueue"),
+            data={
+                "user": soft_deleted_user.id,
+                "instance": self.instance_id,
+            },
+            format="json",
+        )
+        data = response.json()
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(data["status"], SyncQueueStatus.Ready)
+
+    def test_create_deleted_user_should_sync(self):
+        deleted_user = FacilityUser.objects.create(
+            username="deleted",
+            password="***",
+            facility=self.facility,
+        )
+        user_id = deleted_user.id
+        user_filter = get_single_user_sync_filter(deleted_user)
+        controller = MorangoProfileController(FacilityUser.morango_profile)
+        controller.serialize_into_store(user_filter)
+        # Delete the user
+        deleted_user.delete()
+
+        response = self.client.post(
+            reverse("kolibri:core:syncqueue"),
+            data={
+                "user": user_id,
+                "instance": self.instance_id,
+            },
+            format="json",
+        )
+        data = response.json()
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(data["status"], SyncQueueStatus.Ready)
+
+    def test_create_morango_hard_deleted_user_should_sync(self):
+        morango_deleted_user = FacilityUser.objects.create(
+            username="morango_deleted",
+            password="***",
+            facility=self.facility,
+        )
+        user_id = morango_deleted_user.id
+        user_filter = get_single_user_sync_filter(morango_deleted_user)
+        controller = MorangoProfileController(FacilityUser.morango_profile)
+        controller.serialize_into_store(user_filter)
+
+        # Hard delete the user using Morango's hard_delete flag
+        morango_deleted_user.delete(hard_delete=True)
+
+        response = self.client.post(
+            reverse("kolibri:core:syncqueue"),
+            data={
+                "user": user_id,
+                "instance": self.instance_id,
+            },
+            format="json",
+        )
+        data = response.json()
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(data["status"], SyncQueueStatus.Ready)
+
+    def test_create_store_deleted_user_should_sync(self):
+        controller = MorangoProfileController(FacilityUser.morango_profile)
+        store_deleted_user = FacilityUser.objects.create(
+            username="store_deleted",
+            password="***",
+            facility=self.facility,
+        )
+        user_id = store_deleted_user.id
+        user_filter = get_single_user_sync_filter(store_deleted_user)
+        controller.serialize_into_store(user_filter)
+
+        # Delete the user first, then serialize into store
+        store_deleted_user.delete()
+
+        # Serialize deleted models into Store
+        controller.serialize_into_store(user_filter)
+
+        response = self.client.post(
+            reverse("kolibri:core:syncqueue"),
+            data={
+                "user": user_id,
+                "instance": self.instance_id,
+            },
+            format="json",
+        )
+        data = response.json()
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(data["status"], SyncQueueStatus.Ready)
+
+    def test_create_store_hard_deleted_user_should_sync(self):
+        controller = MorangoProfileController(FacilityUser.morango_profile)
+        store_hard_deleted_user = FacilityUser.objects.create(
+            username="store_hard_deleted",
+            password="***",
+            facility=self.facility,
+        )
+        user_id = store_hard_deleted_user.id
+        user_filter = get_single_user_sync_filter(store_hard_deleted_user)
+        controller.serialize_into_store(user_filter)
+
+        # Hard delete the user first, then serialize into store
+        store_hard_deleted_user.delete(hard_delete=True)
+
+        # Serialize deleted models into Store
+        controller.serialize_into_store(user_filter)
+
+        response = self.client.post(
+            reverse("kolibri:core:syncqueue"),
+            data={
+                "user": user_id,
+                "instance": self.instance_id,
+            },
+            format="json",
+        )
+        data = response.json()
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(data["status"], SyncQueueStatus.Ready)

requirements/base.txt

@@ -12,7 +12,7 @@ cheroot==10.0.1
 magicbus==4.1.2
 le-utils==0.2.10
 jsonfield==3.1.0
-morango==0.8.3
+morango==0.8.4
 tzlocal==4.2
 pytz==2024.1
 python-dateutil==2.9.0.post0