From 32b571a9802dc4f49db095a9a6d057bec7ccfe95 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 16:06:30 +0000
Subject: [PATCH 01/45] Optimize relationship fetching with batch population
 and performance improvements

Co-authored-by: jakeb994 <jakeb994@gmail.com>
---
 src/Database/Database.php | 498 +++++++++++++++++++++++++++++++++++++-
 1 file changed, 487 insertions(+), 11 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index f838f838c..c605e9f4e 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3565,6 +3565,473 @@ private function populateDocumentRelationships(Document $collection, Document $d
         return $document;
     }
 
+    /**
+     * Populate relationships for an array of documents using breadth-first approach
+     * This method is optimized for performance by fetching related documents in batches
+     * instead of one by one, which eliminates the N+1 query problem.
+     *
+     * @param Document $collection
+     * @param array<Document> $documents
+     * @param array<string, array<Query>> $selects
+     * @return array<Document>
+     * @throws DatabaseException
+     */
+    private function populateDocumentsRelationships(Document $collection, array $documents, array $selects = []): array
+    {
+        if (empty($documents)) {
+            return $documents;
+        }
+
+        $attributes = $collection->getAttribute('attributes', []);
+        $relationships = \array_filter($attributes, function ($attribute) {
+            return $attribute['type'] === Database::VAR_RELATIONSHIP;
+        });
+
+        if (empty($relationships)) {
+            return $documents;
+        }
+
+        foreach ($relationships as $relationship) {
+            $key = $relationship['key'];
+            $relatedCollection = $this->getCollection($relationship['options']['relatedCollection']);
+            $relationType = $relationship['options']['relationType'];
+            $twoWay = $relationship['options']['twoWay'];
+            $twoWayKey = $relationship['options']['twoWayKey'];
+            $side = $relationship['options']['side'];
+            $queries = $selects[$key] ?? [];
+
+            // Skip if we should not fetch this relationship based on current fetch stack
+            $skipFetch = $this->shouldSkipRelationshipFetchBatch($relationship, $collection);
+            
+            if ($skipFetch) {
+                // Remove the relationship attribute from all documents
+                foreach ($documents as $document) {
+                    $document->removeAttribute($key);
+                }
+                continue;
+            }
+
+            switch ($relationType) {
+                case Database::RELATION_ONE_TO_ONE:
+                    $this->populateOneToOneRelationshipsBatch($documents, $relationship, $relatedCollection, $queries);
+                    break;
+                    
+                case Database::RELATION_ONE_TO_MANY:
+                    $this->populateOneToManyRelationshipsBatch($documents, $relationship, $relatedCollection, $queries, $collection);
+                    break;
+                    
+                case Database::RELATION_MANY_TO_ONE:
+                    $this->populateManyToOneRelationshipsBatch($documents, $relationship, $relatedCollection, $queries, $collection);
+                    break;
+                    
+                case Database::RELATION_MANY_TO_MANY:
+                    $this->populateManyToManyRelationshipsBatch($documents, $relationship, $relatedCollection, $queries, $collection);
+                    break;
+            }
+        }
+
+        return $documents;
+    }
+
+    /**
+     * Check if a relationship should be skipped based on fetch stack and depth (batch version)
+     *
+     * @param array<string, mixed> $relationship
+     * @param Document $collection
+     * @return bool
+     */
+    private function shouldSkipRelationshipFetchBatch(array $relationship, Document $collection): bool
+    {
+        $key = $relationship['key'];
+        $twoWay = $relationship['options']['twoWay'];
+        $twoWayKey = $relationship['options']['twoWayKey'];
+        $side = $relationship['options']['side'];
+        $relatedCollectionId = $relationship['options']['relatedCollection'];
+
+        if ($twoWay && ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH)) {
+            return true;
+        }
+
+        foreach ($this->relationshipFetchStack as $fetchedRelationship) {
+            $existingKey = $fetchedRelationship['key'];
+            $existingCollection = $fetchedRelationship['collection'];
+            $existingRelatedCollection = $fetchedRelationship['options']['relatedCollection'];
+            $existingTwoWayKey = $fetchedRelationship['options']['twoWayKey'];
+            $existingSide = $fetchedRelationship['options']['side'];
+
+            // If this relationship has already been fetched for this document, skip it
+            $reflexive = $fetchedRelationship == $relationship;
+
+            // If this relationship is the same as a previously fetched relationship, but on the other side, skip it
+            $symmetric = $existingKey === $twoWayKey
+                && $existingTwoWayKey === $key
+                && $existingRelatedCollection === $collection->getId()
+                && $existingCollection === $relatedCollectionId
+                && $existingSide !== $side;
+
+            // Transitive relationship detection
+            $transitive = (($existingKey === $twoWayKey
+                    && $existingCollection === $relatedCollectionId
+                    && $existingSide !== $side)
+                || ($existingTwoWayKey === $key
+                    && $existingRelatedCollection === $collection->getId()
+                    && $existingSide !== $side)
+                || ($existingKey === $key
+                    && $existingTwoWayKey !== $twoWayKey
+                    && $existingRelatedCollection === $relatedCollectionId
+                    && $existingSide !== $side)
+                || ($existingKey !== $key
+                    && $existingTwoWayKey === $twoWayKey
+                    && $existingRelatedCollection === $relatedCollectionId
+                    && $existingSide !== $side));
+
+            if ($reflexive || $symmetric || $transitive) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Populate one-to-one relationships in batch
+     *
+     * @param array<Document> $documents
+     * @param array<string, mixed> $relationship
+     * @param Document $relatedCollection
+     * @param array<Query> $queries
+     * @return void
+     * @throws DatabaseException
+     */
+    private function populateOneToOneRelationshipsBatch(array $documents, array $relationship, Document $relatedCollection, array $queries): void
+    {
+        $key = $relationship['key'];
+        $twoWay = $relationship['options']['twoWay'];
+        
+        if ($twoWay && ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH)) {
+            foreach ($documents as $document) {
+                $document->removeAttribute($key);
+            }
+            return;
+        }
+
+        // Collect all related document IDs
+        $relatedIds = [];
+        $documentsByRelatedId = [];
+        
+        foreach ($documents as $document) {
+            $value = $document->getAttribute($key);
+            if (!\is_null($value)) {
+                $relatedIds[] = $value;
+                $documentsByRelatedId[$value] = $document;
+                
+                // Add to map for cycle detection
+                $k = $relatedCollection->getId() . ':' . $value . '=>' . $document->getCollection() . ':' . $document->getId();
+                $this->map[$k] = true;
+            }
+        }
+
+        if (empty($relatedIds)) {
+            return;
+        }
+
+        // Fetch all related documents in a single query
+        $this->relationshipFetchDepth++;
+        $relationship['collection'] = $relatedCollection->getId();
+        $this->relationshipFetchStack[] = $relationship;
+
+        $relatedDocuments = $this->find($relatedCollection->getId(), [
+            Query::equal('$id', array_unique($relatedIds)),
+            Query::limit(PHP_INT_MAX),
+            ...$queries
+        ]);
+
+        $this->relationshipFetchDepth--;
+        \array_pop($this->relationshipFetchStack);
+
+        // Index related documents by ID for quick lookup
+        $relatedById = [];
+        foreach ($relatedDocuments as $related) {
+            $relatedById[$related->getId()] = $related;
+        }
+
+        // Assign related documents to their parent documents
+        foreach ($documentsByRelatedId as $relatedId => $document) {
+            if (isset($relatedById[$relatedId])) {
+                $document->setAttribute($key, $relatedById[$relatedId]);
+            }
+        }
+    }
+
+    /**
+     * Populate one-to-many relationships in batch
+     *
+     * @param array<Document> $documents
+     * @param array<string, mixed> $relationship
+     * @param Document $relatedCollection
+     * @param array<Query> $queries
+     * @param Document $collection
+     * @return void
+     * @throws DatabaseException
+     */
+    private function populateOneToManyRelationshipsBatch(array $documents, array $relationship, Document $relatedCollection, array $queries, Document $collection): void
+    {
+        $key = $relationship['key'];
+        $twoWay = $relationship['options']['twoWay'];
+        $twoWayKey = $relationship['options']['twoWayKey'];
+        $side = $relationship['options']['side'];
+
+        if ($side === Database::RELATION_SIDE_CHILD) {
+            // Child side - treat like one-to-one
+            if (!$twoWay || $this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
+                foreach ($documents as $document) {
+                    $document->removeAttribute($key);
+                }
+                return;
+            }
+            $this->populateOneToOneRelationshipsBatch($documents, $relationship, $relatedCollection, $queries);
+            return;
+        }
+
+        // Parent side - fetch multiple related documents
+        if ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
+            return;
+        }
+
+        // Collect all parent document IDs
+        $parentIds = [];
+        foreach ($documents as $document) {
+            $parentIds[] = $document->getId();
+        }
+
+        if (empty($parentIds)) {
+            return;
+        }
+
+        // Fetch all related documents for all parents in a single query
+        $this->relationshipFetchDepth++;
+        $relationship['collection'] = $collection->getId();
+        $this->relationshipFetchStack[] = $relationship;
+
+        $relatedDocuments = $this->find($relatedCollection->getId(), [
+            Query::equal($twoWayKey, $parentIds),
+            Query::limit(PHP_INT_MAX),
+            ...$queries
+        ]);
+
+        $this->relationshipFetchDepth--;
+        \array_pop($this->relationshipFetchStack);
+
+        // Group related documents by parent ID
+        $relatedByParentId = [];
+        foreach ($relatedDocuments as $related) {
+            $parentId = $related->getAttribute($twoWayKey);
+            if (!\is_null($parentId)) {
+                if (!isset($relatedByParentId[$parentId])) {
+                    $relatedByParentId[$parentId] = [];
+                }
+                // Remove the back-reference to avoid cycles
+                $related->removeAttribute($twoWayKey);
+                $relatedByParentId[$parentId][] = $related;
+            }
+        }
+
+        // Assign related documents to their parent documents
+        foreach ($documents as $document) {
+            $parentId = $document->getId();
+            $document->setAttribute($key, $relatedByParentId[$parentId] ?? []);
+        }
+    }
+
+    /**
+     * Populate many-to-one relationships in batch
+     *
+     * @param array<Document> $documents
+     * @param array<string, mixed> $relationship
+     * @param Document $relatedCollection
+     * @param array<Query> $queries
+     * @param Document $collection
+     * @return void
+     * @throws DatabaseException
+     */
+    private function populateManyToOneRelationshipsBatch(array $documents, array $relationship, Document $relatedCollection, array $queries, Document $collection): void
+    {
+        $key = $relationship['key'];
+        $twoWay = $relationship['options']['twoWay'];
+        $twoWayKey = $relationship['options']['twoWayKey'];
+        $side = $relationship['options']['side'];
+
+        if ($side === Database::RELATION_SIDE_PARENT) {
+            // Parent side - treat like one-to-one
+            if ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
+                foreach ($documents as $document) {
+                    $document->removeAttribute($key);
+                }
+                return;
+            }
+            $this->populateOneToOneRelationshipsBatch($documents, $relationship, $relatedCollection, $queries);
+            return;
+        }
+
+        // Child side - fetch multiple related documents
+        if (!$twoWay) {
+            foreach ($documents as $document) {
+                $document->removeAttribute($key);
+            }
+            return;
+        }
+
+        if ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
+            return;
+        }
+
+        // Collect all child document IDs
+        $childIds = [];
+        foreach ($documents as $document) {
+            $childIds[] = $document->getId();
+        }
+
+        if (empty($childIds)) {
+            return;
+        }
+
+        // Fetch all related documents for all children in a single query
+        $this->relationshipFetchDepth++;
+        $relationship['collection'] = $collection->getId();
+        $this->relationshipFetchStack[] = $relationship;
+
+        $relatedDocuments = $this->find($relatedCollection->getId(), [
+            Query::equal($twoWayKey, $childIds),
+            Query::limit(PHP_INT_MAX),
+            ...$queries
+        ]);
+
+        $this->relationshipFetchDepth--;
+        \array_pop($this->relationshipFetchStack);
+
+        // Group related documents by child ID
+        $relatedByChildId = [];
+        foreach ($relatedDocuments as $related) {
+            $childId = $related->getAttribute($twoWayKey);
+            if (!\is_null($childId)) {
+                if (!isset($relatedByChildId[$childId])) {
+                    $relatedByChildId[$childId] = [];
+                }
+                // Remove the back-reference to avoid cycles
+                $related->removeAttribute($twoWayKey);
+                $relatedByChildId[$childId][] = $related;
+            }
+        }
+
+        // Assign related documents to their child documents
+        foreach ($documents as $document) {
+            $childId = $document->getId();
+            $document->setAttribute($key, $relatedByChildId[$childId] ?? []);
+        }
+    }
+
+    /**
+     * Populate many-to-many relationships in batch
+     *
+     * @param array<Document> $documents
+     * @param array<string, mixed> $relationship
+     * @param Document $relatedCollection
+     * @param array<Query> $queries
+     * @param Document $collection
+     * @return void
+     * @throws DatabaseException
+     */
+    private function populateManyToManyRelationshipsBatch(array $documents, array $relationship, Document $relatedCollection, array $queries, Document $collection): void
+    {
+        $key = $relationship['key'];
+        $twoWay = $relationship['options']['twoWay'];
+        $twoWayKey = $relationship['options']['twoWayKey'];
+        $side = $relationship['options']['side'];
+
+        if (!$twoWay && $side === Database::RELATION_SIDE_CHILD) {
+            return;
+        }
+
+        if ($twoWay && ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH)) {
+            return;
+        }
+
+        // Collect all document IDs
+        $documentIds = [];
+        foreach ($documents as $document) {
+            $documentIds[] = $document->getId();
+        }
+
+        if (empty($documentIds)) {
+            return;
+        }
+
+        $this->relationshipFetchDepth++;
+        $relationship['collection'] = $collection->getId();
+        $this->relationshipFetchStack[] = $relationship;
+
+        $junction = $this->getJunctionCollection($collection, $relatedCollection, $side);
+
+        // Fetch all junction records for all documents in a single query
+        $junctions = $this->skipRelationships(fn () => $this->find($junction, [
+            Query::equal($twoWayKey, $documentIds),
+            Query::limit(PHP_INT_MAX)
+        ]));
+
+        // Collect all related IDs from junctions
+        $relatedIds = [];
+        $junctionsByDocumentId = [];
+        
+        foreach ($junctions as $junctionDoc) {
+            $documentId = $junctionDoc->getAttribute($twoWayKey);
+            $relatedId = $junctionDoc->getAttribute($key);
+            
+            if (!\is_null($documentId) && !\is_null($relatedId)) {
+                if (!isset($junctionsByDocumentId[$documentId])) {
+                    $junctionsByDocumentId[$documentId] = [];
+                }
+                $junctionsByDocumentId[$documentId][] = $relatedId;
+                $relatedIds[] = $relatedId;
+            }
+        }
+
+        // Fetch all related documents in a single query
+        $related = [];
+        if (!empty($relatedIds)) {
+            $foundRelated = $this->find($relatedCollection->getId(), [
+                Query::equal('$id', array_unique($relatedIds)),
+                Query::limit(PHP_INT_MAX),
+                ...$queries
+            ]);
+
+            // Index related documents by ID for quick lookup
+            $relatedById = [];
+            foreach ($foundRelated as $doc) {
+                $relatedById[$doc->getId()] = $doc;
+            }
+
+            // Build final related arrays maintaining junction order
+            foreach ($junctionsByDocumentId as $documentId => $relatedDocIds) {
+                $documentRelated = [];
+                foreach ($relatedDocIds as $relatedId) {
+                    if (isset($relatedById[$relatedId])) {
+                        $documentRelated[] = $relatedById[$relatedId];
+                    }
+                }
+                $related[$documentId] = $documentRelated;
+            }
+        }
+
+        $this->relationshipFetchDepth--;
+        \array_pop($this->relationshipFetchStack);
+
+        // Assign related documents to their parent documents
+        foreach ($documents as $document) {
+            $documentId = $document->getId();
+            $document->setAttribute($key, $related[$documentId] ?? []);
+        }
+    }
+
     /**
      * Create Document
      *
@@ -3757,11 +4224,12 @@ public function createDocuments(
 
             $batch = $this->adapter->getSequences($collection->getId(), $batch);
 
-            foreach ($batch as $document) {
-                if ($this->resolveRelationships) {
-                    $document = $this->silent(fn () => $this->populateDocumentRelationships($collection, $document));
-                }
+            // Use batch relationship population for better performance
+            if ($this->resolveRelationships) {
+                $batch = $this->silent(fn () => $this->populateDocumentsRelationships($collection, $batch));
+            }
 
+            foreach ($batch as $document) {
                 $document = $this->casting($collection, $document);
                 $document = $this->decode($collection, $document);
                 $onNext && $onNext($document);
@@ -5137,11 +5605,12 @@ public function createOrUpdateDocumentsWithIncrease(
                 }
             }
 
-            foreach ($batch as $doc) {
-                if ($this->resolveRelationships) {
-                    $doc = $this->silent(fn () => $this->populateDocumentRelationships($collection, $doc));
-                }
+            // Use batch relationship population for better performance
+            if ($this->resolveRelationships) {
+                $batch = $this->silent(fn () => $this->populateDocumentsRelationships($collection, $batch));
+            }
 
+            foreach ($batch as $doc) {
                 $doc = $this->decode($collection, $doc);
 
                 if ($this->getSharedTables() && $this->getTenantPerDocument()) {
@@ -6139,11 +6608,18 @@ public function find(string $collection, array $queries = [], string $forPermiss
 
         $results = $skipAuth ? Authorization::skip($getResults) : $getResults();
 
-        foreach ($results as $index => $node) {
-            if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections))) {
-                $node = $this->silent(fn () => $this->populateDocumentRelationships($collection, $node, $nestedSelections));
+        // Use batch relationship population for better performance when dealing with multiple documents
+        if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections))) {
+            if (count($results) > 1) {
+                // Use batch processing for multiple documents
+                $results = $this->silent(fn () => $this->populateDocumentsRelationships($collection, $results, $nestedSelections));
+            } else if (count($results) === 1) {
+                // Use single document processing for one document
+                $results[0] = $this->silent(fn () => $this->populateDocumentRelationships($collection, $results[0], $nestedSelections));
             }
+        }
 
+        foreach ($results as $index => $node) {
             $node = $this->casting($collection, $node);
             $node = $this->decode($collection, $node, $selections);
 

From 8da5e64523a470dd7c44d708da501324699e2a94 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 16:31:25 +0000
Subject: [PATCH 02/45] Fix relationship batch processing to handle Document
 objects correctly

- Update batch methods to use Document->getAttribute() instead of array access
- Fix shouldSkipRelationshipFetchBatch to accept Document parameter
- Convert Document to array when adding to relationship fetch stack
- Maintain compatibility with existing relationship fetch stack structure
---
 src/Database/Database.php | 94 ++++++++++++++++++++-------------------
 1 file changed, 49 insertions(+), 45 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index c605e9f4e..c8cdc328d 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3584,7 +3584,7 @@ private function populateDocumentsRelationships(Document $collection, array $doc
 
         $attributes = $collection->getAttribute('attributes', []);
         $relationships = \array_filter($attributes, function ($attribute) {
-            return $attribute['type'] === Database::VAR_RELATIONSHIP;
+            return $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP;
         });
 
         if (empty($relationships)) {
@@ -3592,12 +3592,12 @@ private function populateDocumentsRelationships(Document $collection, array $doc
         }
 
         foreach ($relationships as $relationship) {
-            $key = $relationship['key'];
-            $relatedCollection = $this->getCollection($relationship['options']['relatedCollection']);
-            $relationType = $relationship['options']['relationType'];
-            $twoWay = $relationship['options']['twoWay'];
-            $twoWayKey = $relationship['options']['twoWayKey'];
-            $side = $relationship['options']['side'];
+            $key = $relationship->getAttribute('key');
+            $relatedCollection = $this->getCollection($relationship->getAttribute('options')['relatedCollection']);
+            $relationType = $relationship->getAttribute('options')['relationType'];
+            $twoWay = $relationship->getAttribute('options')['twoWay'];
+            $twoWayKey = $relationship->getAttribute('options')['twoWayKey'];
+            $side = $relationship->getAttribute('options')['side'];
             $queries = $selects[$key] ?? [];
 
             // Skip if we should not fetch this relationship based on current fetch stack
@@ -3636,17 +3636,17 @@ private function populateDocumentsRelationships(Document $collection, array $doc
     /**
      * Check if a relationship should be skipped based on fetch stack and depth (batch version)
      *
-     * @param array<string, mixed> $relationship
+     * @param Document $relationship
      * @param Document $collection
      * @return bool
      */
-    private function shouldSkipRelationshipFetchBatch(array $relationship, Document $collection): bool
+    private function shouldSkipRelationshipFetchBatch(Document $relationship, Document $collection): bool
     {
-        $key = $relationship['key'];
-        $twoWay = $relationship['options']['twoWay'];
-        $twoWayKey = $relationship['options']['twoWayKey'];
-        $side = $relationship['options']['side'];
-        $relatedCollectionId = $relationship['options']['relatedCollection'];
+        $key = $relationship->getAttribute('key');
+        $twoWay = $relationship->getAttribute('options')['twoWay'];
+        $twoWayKey = $relationship->getAttribute('options')['twoWayKey'];
+        $side = $relationship->getAttribute('options')['side'];
+        $relatedCollectionId = $relationship->getAttribute('options')['relatedCollection'];
 
         if ($twoWay && ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH)) {
             return true;
@@ -3660,7 +3660,7 @@ private function shouldSkipRelationshipFetchBatch(array $relationship, Document
             $existingSide = $fetchedRelationship['options']['side'];
 
             // If this relationship has already been fetched for this document, skip it
-            $reflexive = $fetchedRelationship == $relationship;
+            $reflexive = $fetchedRelationship == $relationship->getArrayCopy();
 
             // If this relationship is the same as a previously fetched relationship, but on the other side, skip it
             $symmetric = $existingKey === $twoWayKey
@@ -3697,16 +3697,16 @@ private function shouldSkipRelationshipFetchBatch(array $relationship, Document
      * Populate one-to-one relationships in batch
      *
      * @param array<Document> $documents
-     * @param array<string, mixed> $relationship
+     * @param Document $relationship
      * @param Document $relatedCollection
      * @param array<Query> $queries
      * @return void
      * @throws DatabaseException
      */
-    private function populateOneToOneRelationshipsBatch(array $documents, array $relationship, Document $relatedCollection, array $queries): void
+    private function populateOneToOneRelationshipsBatch(array $documents, Document $relationship, Document $relatedCollection, array $queries): void
     {
-        $key = $relationship['key'];
-        $twoWay = $relationship['options']['twoWay'];
+        $key = $relationship->getAttribute('key');
+        $twoWay = $relationship->getAttribute('options')['twoWay'];
         
         if ($twoWay && ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH)) {
             foreach ($documents as $document) {
@@ -3737,8 +3737,9 @@ private function populateOneToOneRelationshipsBatch(array $documents, array $rel
 
         // Fetch all related documents in a single query
         $this->relationshipFetchDepth++;
-        $relationship['collection'] = $relatedCollection->getId();
-        $this->relationshipFetchStack[] = $relationship;
+        $relationshipArray = $relationship->getArrayCopy();
+        $relationshipArray['collection'] = $relatedCollection->getId();
+        $this->relationshipFetchStack[] = $relationshipArray;
 
         $relatedDocuments = $this->find($relatedCollection->getId(), [
             Query::equal('$id', array_unique($relatedIds)),
@@ -3767,19 +3768,19 @@ private function populateOneToOneRelationshipsBatch(array $documents, array $rel
      * Populate one-to-many relationships in batch
      *
      * @param array<Document> $documents
-     * @param array<string, mixed> $relationship
+     * @param Document $relationship
      * @param Document $relatedCollection
      * @param array<Query> $queries
      * @param Document $collection
      * @return void
      * @throws DatabaseException
      */
-    private function populateOneToManyRelationshipsBatch(array $documents, array $relationship, Document $relatedCollection, array $queries, Document $collection): void
+    private function populateOneToManyRelationshipsBatch(array $documents, Document $relationship, Document $relatedCollection, array $queries, Document $collection): void
     {
-        $key = $relationship['key'];
-        $twoWay = $relationship['options']['twoWay'];
-        $twoWayKey = $relationship['options']['twoWayKey'];
-        $side = $relationship['options']['side'];
+        $key = $relationship->getAttribute('key');
+        $twoWay = $relationship->getAttribute('options')['twoWay'];
+        $twoWayKey = $relationship->getAttribute('options')['twoWayKey'];
+        $side = $relationship->getAttribute('options')['side'];
 
         if ($side === Database::RELATION_SIDE_CHILD) {
             // Child side - treat like one-to-one
@@ -3810,8 +3811,9 @@ private function populateOneToManyRelationshipsBatch(array $documents, array $re
 
         // Fetch all related documents for all parents in a single query
         $this->relationshipFetchDepth++;
-        $relationship['collection'] = $collection->getId();
-        $this->relationshipFetchStack[] = $relationship;
+        $relationshipArray = $relationship->getArrayCopy();
+        $relationshipArray['collection'] = $collection->getId();
+        $this->relationshipFetchStack[] = $relationshipArray;
 
         $relatedDocuments = $this->find($relatedCollection->getId(), [
             Query::equal($twoWayKey, $parentIds),
@@ -3847,19 +3849,19 @@ private function populateOneToManyRelationshipsBatch(array $documents, array $re
      * Populate many-to-one relationships in batch
      *
      * @param array<Document> $documents
-     * @param array<string, mixed> $relationship
+     * @param Document $relationship
      * @param Document $relatedCollection
      * @param array<Query> $queries
      * @param Document $collection
      * @return void
      * @throws DatabaseException
      */
-    private function populateManyToOneRelationshipsBatch(array $documents, array $relationship, Document $relatedCollection, array $queries, Document $collection): void
+    private function populateManyToOneRelationshipsBatch(array $documents, Document $relationship, Document $relatedCollection, array $queries, Document $collection): void
     {
-        $key = $relationship['key'];
-        $twoWay = $relationship['options']['twoWay'];
-        $twoWayKey = $relationship['options']['twoWayKey'];
-        $side = $relationship['options']['side'];
+        $key = $relationship->getAttribute('key');
+        $twoWay = $relationship->getAttribute('options')['twoWay'];
+        $twoWayKey = $relationship->getAttribute('options')['twoWayKey'];
+        $side = $relationship->getAttribute('options')['side'];
 
         if ($side === Database::RELATION_SIDE_PARENT) {
             // Parent side - treat like one-to-one
@@ -3897,8 +3899,9 @@ private function populateManyToOneRelationshipsBatch(array $documents, array $re
 
         // Fetch all related documents for all children in a single query
         $this->relationshipFetchDepth++;
-        $relationship['collection'] = $collection->getId();
-        $this->relationshipFetchStack[] = $relationship;
+        $relationshipArray = $relationship->getArrayCopy();
+        $relationshipArray['collection'] = $collection->getId();
+        $this->relationshipFetchStack[] = $relationshipArray;
 
         $relatedDocuments = $this->find($relatedCollection->getId(), [
             Query::equal($twoWayKey, $childIds),
@@ -3934,19 +3937,19 @@ private function populateManyToOneRelationshipsBatch(array $documents, array $re
      * Populate many-to-many relationships in batch
      *
      * @param array<Document> $documents
-     * @param array<string, mixed> $relationship
+     * @param Document $relationship
      * @param Document $relatedCollection
      * @param array<Query> $queries
      * @param Document $collection
      * @return void
      * @throws DatabaseException
      */
-    private function populateManyToManyRelationshipsBatch(array $documents, array $relationship, Document $relatedCollection, array $queries, Document $collection): void
+    private function populateManyToManyRelationshipsBatch(array $documents, Document $relationship, Document $relatedCollection, array $queries, Document $collection): void
     {
-        $key = $relationship['key'];
-        $twoWay = $relationship['options']['twoWay'];
-        $twoWayKey = $relationship['options']['twoWayKey'];
-        $side = $relationship['options']['side'];
+        $key = $relationship->getAttribute('key');
+        $twoWay = $relationship->getAttribute('options')['twoWay'];
+        $twoWayKey = $relationship->getAttribute('options')['twoWayKey'];
+        $side = $relationship->getAttribute('options')['side'];
 
         if (!$twoWay && $side === Database::RELATION_SIDE_CHILD) {
             return;
@@ -3967,8 +3970,9 @@ private function populateManyToManyRelationshipsBatch(array $documents, array $r
         }
 
         $this->relationshipFetchDepth++;
-        $relationship['collection'] = $collection->getId();
-        $this->relationshipFetchStack[] = $relationship;
+        $relationshipArray = $relationship->getArrayCopy();
+        $relationshipArray['collection'] = $collection->getId();
+        $this->relationshipFetchStack[] = $relationshipArray;
 
         $junction = $this->getJunctionCollection($collection, $relatedCollection, $side);
 

From f9a0a2d2147311882150f0e5f682df98acdb6c6f Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 16:32:57 +0000
Subject: [PATCH 03/45] Fix relationship access to use array syntax
 consistently

- Use array access syntax (['key']) instead of getAttribute()
- Matches the original implementation pattern
- Always use batch processing for all document counts
- Should resolve relationship population issues in tests
---
 src/Database/Database.php | 59 +++++++++++++++++++--------------------
 1 file changed, 28 insertions(+), 31 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index c8cdc328d..22d4f1784 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3592,12 +3592,12 @@ private function populateDocumentsRelationships(Document $collection, array $doc
         }
 
         foreach ($relationships as $relationship) {
-            $key = $relationship->getAttribute('key');
-            $relatedCollection = $this->getCollection($relationship->getAttribute('options')['relatedCollection']);
-            $relationType = $relationship->getAttribute('options')['relationType'];
-            $twoWay = $relationship->getAttribute('options')['twoWay'];
-            $twoWayKey = $relationship->getAttribute('options')['twoWayKey'];
-            $side = $relationship->getAttribute('options')['side'];
+            $key = $relationship['key'];
+            $relatedCollection = $this->getCollection($relationship['options']['relatedCollection']);
+            $relationType = $relationship['options']['relationType'];
+            $twoWay = $relationship['options']['twoWay'];
+            $twoWayKey = $relationship['options']['twoWayKey'];
+            $side = $relationship['options']['side'];
             $queries = $selects[$key] ?? [];
 
             // Skip if we should not fetch this relationship based on current fetch stack
@@ -3642,11 +3642,11 @@ private function populateDocumentsRelationships(Document $collection, array $doc
      */
     private function shouldSkipRelationshipFetchBatch(Document $relationship, Document $collection): bool
     {
-        $key = $relationship->getAttribute('key');
-        $twoWay = $relationship->getAttribute('options')['twoWay'];
-        $twoWayKey = $relationship->getAttribute('options')['twoWayKey'];
-        $side = $relationship->getAttribute('options')['side'];
-        $relatedCollectionId = $relationship->getAttribute('options')['relatedCollection'];
+        $key = $relationship['key'];
+        $twoWay = $relationship['options']['twoWay'];
+        $twoWayKey = $relationship['options']['twoWayKey'];
+        $side = $relationship['options']['side'];
+        $relatedCollectionId = $relationship['options']['relatedCollection'];
 
         if ($twoWay && ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH)) {
             return true;
@@ -3705,8 +3705,8 @@ private function shouldSkipRelationshipFetchBatch(Document $relationship, Docume
      */
     private function populateOneToOneRelationshipsBatch(array $documents, Document $relationship, Document $relatedCollection, array $queries): void
     {
-        $key = $relationship->getAttribute('key');
-        $twoWay = $relationship->getAttribute('options')['twoWay'];
+        $key = $relationship['key'];
+        $twoWay = $relationship['options']['twoWay'];
         
         if ($twoWay && ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH)) {
             foreach ($documents as $document) {
@@ -3777,10 +3777,10 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
      */
     private function populateOneToManyRelationshipsBatch(array $documents, Document $relationship, Document $relatedCollection, array $queries, Document $collection): void
     {
-        $key = $relationship->getAttribute('key');
-        $twoWay = $relationship->getAttribute('options')['twoWay'];
-        $twoWayKey = $relationship->getAttribute('options')['twoWayKey'];
-        $side = $relationship->getAttribute('options')['side'];
+        $key = $relationship['key'];
+        $twoWay = $relationship['options']['twoWay'];
+        $twoWayKey = $relationship['options']['twoWayKey'];
+        $side = $relationship['options']['side'];
 
         if ($side === Database::RELATION_SIDE_CHILD) {
             // Child side - treat like one-to-one
@@ -3858,10 +3858,10 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
      */
     private function populateManyToOneRelationshipsBatch(array $documents, Document $relationship, Document $relatedCollection, array $queries, Document $collection): void
     {
-        $key = $relationship->getAttribute('key');
-        $twoWay = $relationship->getAttribute('options')['twoWay'];
-        $twoWayKey = $relationship->getAttribute('options')['twoWayKey'];
-        $side = $relationship->getAttribute('options')['side'];
+        $key = $relationship['key'];
+        $twoWay = $relationship['options']['twoWay'];
+        $twoWayKey = $relationship['options']['twoWayKey'];
+        $side = $relationship['options']['side'];
 
         if ($side === Database::RELATION_SIDE_PARENT) {
             // Parent side - treat like one-to-one
@@ -3946,10 +3946,10 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
      */
     private function populateManyToManyRelationshipsBatch(array $documents, Document $relationship, Document $relatedCollection, array $queries, Document $collection): void
     {
-        $key = $relationship->getAttribute('key');
-        $twoWay = $relationship->getAttribute('options')['twoWay'];
-        $twoWayKey = $relationship->getAttribute('options')['twoWayKey'];
-        $side = $relationship->getAttribute('options')['side'];
+        $key = $relationship['key'];
+        $twoWay = $relationship['options']['twoWay'];
+        $twoWayKey = $relationship['options']['twoWayKey'];
+        $side = $relationship['options']['side'];
 
         if (!$twoWay && $side === Database::RELATION_SIDE_CHILD) {
             return;
@@ -6612,14 +6612,11 @@ public function find(string $collection, array $queries = [], string $forPermiss
 
         $results = $skipAuth ? Authorization::skip($getResults) : $getResults();
 
-        // Use batch relationship population for better performance when dealing with multiple documents
+        // Use batch relationship population for better performance
         if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections))) {
-            if (count($results) > 1) {
-                // Use batch processing for multiple documents
+            if (count($results) > 0) {
+                // Use batch processing for all cases (handles single and multiple documents)
                 $results = $this->silent(fn () => $this->populateDocumentsRelationships($collection, $results, $nestedSelections));
-            } else if (count($results) === 1) {
-                // Use single document processing for one document
-                $results[0] = $this->silent(fn () => $this->populateDocumentRelationships($collection, $results[0], $nestedSelections));
             }
         }
 

From 0307073c96e01dbd5cec0e31c0344a4be3350f4d Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 16:40:45 +0000
Subject: [PATCH 04/45] Fix recursive batch processing in relationship
 population

- Add batchRelationshipProcessing flag to prevent infinite recursion
- When already in batch processing, fall back to original single-document method
- This should resolve issues with nested relationship queries returning empty arrays
---
 src/Database/Database.php | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 22d4f1784..f34951022 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -374,6 +374,8 @@ class Database
      */
     protected array $relationshipFetchStack = [];
 
+    protected bool $batchRelationshipProcessing = false;
+
     /**
      * @var array<Document>
      */
@@ -6612,11 +6614,18 @@ public function find(string $collection, array $queries = [], string $forPermiss
 
         $results = $skipAuth ? Authorization::skip($getResults) : $getResults();
 
-        // Use batch relationship population for better performance
-        if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections))) {
+        // Use batch relationship population for better performance (but not if already in batch processing)
+        if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections)) && !$this->batchRelationshipProcessing) {
             if (count($results) > 0) {
                 // Use batch processing for all cases (handles single and multiple documents)
+                $this->batchRelationshipProcessing = true;
                 $results = $this->silent(fn () => $this->populateDocumentsRelationships($collection, $results, $nestedSelections));
+                $this->batchRelationshipProcessing = false;
+            }
+        } else if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections)) && $this->batchRelationshipProcessing) {
+            // If we're already in batch processing, fall back to original method for nested calls
+            foreach ($results as $index => $node) {
+                $results[$index] = $this->silent(fn () => $this->populateDocumentRelationships($collection, $node, $nestedSelections));
             }
         }
 

From 865eff55b552d4edf77b8496f30c6779654fafee Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 16:42:01 +0000
Subject: [PATCH 05/45] Fix multi-level relationship processing using existing
 depth mechanism

- Use batch processing only for top-level calls (relationshipFetchDepth === 1)
- Use original single-document processing for nested relationship calls
- Remove custom flag and rely on existing relationshipFetchDepth system
- This should resolve missing relationship data in nested scenarios
---
 src/Database/Database.php | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index f34951022..f9365b9d1 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -374,8 +374,6 @@ class Database
      */
     protected array $relationshipFetchStack = [];
 
-    protected bool $batchRelationshipProcessing = false;
-
     /**
      * @var array<Document>
      */
@@ -6614,18 +6612,19 @@ public function find(string $collection, array $queries = [], string $forPermiss
 
         $results = $skipAuth ? Authorization::skip($getResults) : $getResults();
 
-        // Use batch relationship population for better performance (but not if already in batch processing)
-        if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections)) && !$this->batchRelationshipProcessing) {
+        // Use batch relationship population for better performance
+        // But only if we're not already in relationship processing (depth 1 means no nesting)
+        if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections))) {
             if (count($results) > 0) {
-                // Use batch processing for all cases (handles single and multiple documents)
-                $this->batchRelationshipProcessing = true;
-                $results = $this->silent(fn () => $this->populateDocumentsRelationships($collection, $results, $nestedSelections));
-                $this->batchRelationshipProcessing = false;
-            }
-        } else if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections)) && $this->batchRelationshipProcessing) {
-            // If we're already in batch processing, fall back to original method for nested calls
-            foreach ($results as $index => $node) {
-                $results[$index] = $this->silent(fn () => $this->populateDocumentRelationships($collection, $node, $nestedSelections));
+                if ($this->relationshipFetchDepth === 1) {
+                    // Use batch processing only for top-level calls
+                    $results = $this->silent(fn () => $this->populateDocumentsRelationships($collection, $results, $nestedSelections));
+                } else {
+                    // Use original single-document processing for nested relationship calls
+                    foreach ($results as $index => $node) {
+                        $results[$index] = $this->silent(fn () => $this->populateDocumentRelationships($collection, $node, $nestedSelections));
+                    }
+                }
             }
         }
 

From fdf29370c4a057f8b8612f0c6a64772c3c750b13 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 16:52:59 +0000
Subject: [PATCH 06/45] Fix nested relationship detection using fetch stack
 instead of depth

- Use empty relationshipFetchStack to detect top-level calls for batch processing
- Use non-empty stack to detect nested calls that should use single processing
- This should properly distinguish between top-level and nested relationship calls
- Should resolve issues with one-to-many and other relationship types returning empty arrays
---
 src/Database/Database.php | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index f9365b9d1..d7398f76d 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -6613,11 +6613,11 @@ public function find(string $collection, array $queries = [], string $forPermiss
         $results = $skipAuth ? Authorization::skip($getResults) : $getResults();
 
         // Use batch relationship population for better performance
-        // But only if we're not already in relationship processing (depth 1 means no nesting)
+        // But only if we're not already in relationship processing (empty stack means top-level)
         if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections))) {
             if (count($results) > 0) {
-                if ($this->relationshipFetchDepth === 1) {
-                    // Use batch processing only for top-level calls
+                if (empty($this->relationshipFetchStack)) {
+                    // Use batch processing only for top-level calls (empty fetch stack)
                     $results = $this->silent(fn () => $this->populateDocumentsRelationships($collection, $results, $nestedSelections));
                 } else {
                     // Use original single-document processing for nested relationship calls

From 986f5db470dbddd7060f837b4f3997f75350dbf2 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 16:54:43 +0000
Subject: [PATCH 07/45] Use batch processing at all levels consistently

- Remove conditional logic that mixed batch and single processing
- Always use batch processing for all relationship population
- Eliminates N+1 problems at all nesting levels
- Simplifies logic and ensures consistent behavior
- Maximum performance benefits across all relationship scenarios
---
 src/Database/Database.php | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index d7398f76d..a6d5498bb 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -6612,19 +6612,11 @@ public function find(string $collection, array $queries = [], string $forPermiss
 
         $results = $skipAuth ? Authorization::skip($getResults) : $getResults();
 
-        // Use batch relationship population for better performance
-        // But only if we're not already in relationship processing (empty stack means top-level)
+        // Use batch relationship population for better performance at all levels
         if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections))) {
             if (count($results) > 0) {
-                if (empty($this->relationshipFetchStack)) {
-                    // Use batch processing only for top-level calls (empty fetch stack)
-                    $results = $this->silent(fn () => $this->populateDocumentsRelationships($collection, $results, $nestedSelections));
-                } else {
-                    // Use original single-document processing for nested relationship calls
-                    foreach ($results as $index => $node) {
-                        $results[$index] = $this->silent(fn () => $this->populateDocumentRelationships($collection, $node, $nestedSelections));
-                    }
-                }
+                // Always use batch processing for all cases (single and multiple documents, nested or top-level)
+                $results = $this->silent(fn () => $this->populateDocumentsRelationships($collection, $results, $nestedSelections));
             }
         }
 

From c01c35253201bf89d5d3ef1c510a27d7bdeecdf1 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 17:04:34 +0000
Subject: [PATCH 08/45] Update all single-document methods to use batch
 relationship processing

- Convert getDocument to use populateDocumentsRelationships with single-element array
- Convert createDocument to use batch processing for relationships
- Convert updateDocument to use batch processing for relationships
- This ensures consistent batch processing at all levels, including nested relationships
- Should resolve 2nd level relationship population issues
---
 src/Database/Database.php | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index a6d5498bb..9f7aab3a2 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3290,7 +3290,8 @@ public function getDocument(string $collection, string $id, array $queries = [],
         $this->map = [];
 
         if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections))) {
-            $document = $this->silent(fn () => $this->populateDocumentRelationships($collection, $document, $nestedSelections));
+            $documents = $this->silent(fn () => $this->populateDocumentsRelationships($collection, [$document], $nestedSelections));
+            $document = $documents[0];
         }
 
         $relationships = \array_filter(
@@ -4130,7 +4131,8 @@ public function createDocument(string $collection, Document $document): Document
         });
 
         if ($this->resolveRelationships) {
-            $document = $this->silent(fn () => $this->populateDocumentRelationships($collection, $document));
+            $documents = $this->silent(fn () => $this->populateDocumentsRelationships($collection, [$document]));
+            $document = $documents[0];
         }
 
         $document = $this->casting($collection, $document);
@@ -4773,7 +4775,8 @@ public function updateDocument(string $collection, string $id, Document $documen
         });
 
         if ($this->resolveRelationships) {
-            $document = $this->silent(fn () => $this->populateDocumentRelationships($collection, $document));
+            $documents = $this->silent(fn () => $this->populateDocumentsRelationships($collection, [$document]));
+            $document = $documents[0];
         }
 
         $document = $this->decode($collection, $document);

From dfd16373b89d6ee75302bfac0ff1c175e6a88964 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 17:16:49 +0000
Subject: [PATCH 09/45] Fix relationship cycle detection by adding collection
 attribute to relationship

---
 src/Database/Database.php | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 9f7aab3a2..7c93f0ccb 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3601,6 +3601,9 @@ private function populateDocumentsRelationships(Document $collection, array $doc
             $side = $relationship['options']['side'];
             $queries = $selects[$key] ?? [];
 
+            // Add collection attribute for proper cycle detection - this is critical!
+            $relationship->setAttribute('collection', $collection->getId());
+
             // Skip if we should not fetch this relationship based on current fetch stack
             $skipFetch = $this->shouldSkipRelationshipFetchBatch($relationship, $collection);
             
@@ -3661,7 +3664,7 @@ private function shouldSkipRelationshipFetchBatch(Document $relationship, Docume
             $existingSide = $fetchedRelationship['options']['side'];
 
             // If this relationship has already been fetched for this document, skip it
-            $reflexive = $fetchedRelationship == $relationship->getArrayCopy();
+            $reflexive = $fetchedRelationship == $relationship;
 
             // If this relationship is the same as a previously fetched relationship, but on the other side, skip it
             $symmetric = $existingKey === $twoWayKey

From f5e9c28afc22caa765e5d2797be74280270324a2 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 17:17:52 +0000
Subject: [PATCH 10/45] Fix relationship assignment - set to null when related
 document not found

---
 src/Database/Database.php | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 7c93f0ccb..f984a4ca6 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3764,6 +3764,9 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
         foreach ($documentsByRelatedId as $relatedId => $document) {
             if (isset($relatedById[$relatedId])) {
                 $document->setAttribute($key, $relatedById[$relatedId]);
+            } else {
+                // If related document not found, set to null instead of leaving the string ID
+                $document->setAttribute($key, null);
             }
         }
     }

From d3e2b2130eb053e8f77c10cf285309e238bb8aa7 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 17:19:51 +0000
Subject: [PATCH 11/45] Fix relationship stack management - use Document
 objects directly like original method

---
 src/Database/Database.php | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index f984a4ca6..2c3927fd9 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3741,9 +3741,7 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
 
         // Fetch all related documents in a single query
         $this->relationshipFetchDepth++;
-        $relationshipArray = $relationship->getArrayCopy();
-        $relationshipArray['collection'] = $relatedCollection->getId();
-        $this->relationshipFetchStack[] = $relationshipArray;
+        $this->relationshipFetchStack[] = $relationship;
 
         $relatedDocuments = $this->find($relatedCollection->getId(), [
             Query::equal('$id', array_unique($relatedIds)),
@@ -3818,9 +3816,7 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
 
         // Fetch all related documents for all parents in a single query
         $this->relationshipFetchDepth++;
-        $relationshipArray = $relationship->getArrayCopy();
-        $relationshipArray['collection'] = $collection->getId();
-        $this->relationshipFetchStack[] = $relationshipArray;
+        $this->relationshipFetchStack[] = $relationship;
 
         $relatedDocuments = $this->find($relatedCollection->getId(), [
             Query::equal($twoWayKey, $parentIds),
@@ -3906,9 +3902,7 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
 
         // Fetch all related documents for all children in a single query
         $this->relationshipFetchDepth++;
-        $relationshipArray = $relationship->getArrayCopy();
-        $relationshipArray['collection'] = $collection->getId();
-        $this->relationshipFetchStack[] = $relationshipArray;
+        $this->relationshipFetchStack[] = $relationship;
 
         $relatedDocuments = $this->find($relatedCollection->getId(), [
             Query::equal($twoWayKey, $childIds),
@@ -3977,9 +3971,7 @@ private function populateManyToManyRelationshipsBatch(array $documents, Document
         }
 
         $this->relationshipFetchDepth++;
-        $relationshipArray = $relationship->getArrayCopy();
-        $relationshipArray['collection'] = $collection->getId();
-        $this->relationshipFetchStack[] = $relationshipArray;
+        $this->relationshipFetchStack[] = $relationship;
 
         $junction = $this->getJunctionCollection($collection, $relatedCollection, $side);
 

From f43a9c22fdd447de65474b41654566a0acb692b7 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 17:35:52 +0000
Subject: [PATCH 12/45] TEMP: Disable cycle detection to test if that's causing
 the issue

---
 src/Database/Database.php | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 2c3927fd9..074aec8a4 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3604,16 +3604,17 @@ private function populateDocumentsRelationships(Document $collection, array $doc
             // Add collection attribute for proper cycle detection - this is critical!
             $relationship->setAttribute('collection', $collection->getId());
 
+            // TEMP: Disable cycle detection to test if that's the issue
             // Skip if we should not fetch this relationship based on current fetch stack
-            $skipFetch = $this->shouldSkipRelationshipFetchBatch($relationship, $collection);
-            
-            if ($skipFetch) {
-                // Remove the relationship attribute from all documents
-                foreach ($documents as $document) {
-                    $document->removeAttribute($key);
-                }
-                continue;
-            }
+            // $skipFetch = $this->shouldSkipRelationshipFetchBatch($relationship, $collection);
+            // 
+            // if ($skipFetch) {
+            //     // Remove the relationship attribute from all documents
+            //     foreach ($documents as $document) {
+            //         $document->removeAttribute($key);
+            //     }
+            //     continue;
+            // }
 
             switch ($relationType) {
                 case Database::RELATION_ONE_TO_ONE:

From ca82060ff8f81578539f4493cf2d2954b65d4c31 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 17:43:18 +0000
Subject: [PATCH 13/45] MAJOR FIX: Handle Document objects vs string IDs in
 relationship grouping

- Fixed TypeError: Cannot access offset of type Document in isset
- Handle cases where twoWayKey returns Document objects instead of string IDs
- Extract ID from Document objects when using as array keys
- Re-enabled cycle detection after fixing core issue
---
 src/Database/Database.php | 35 +++++++++++++++++++----------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 074aec8a4..38c5966e5 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3604,17 +3604,16 @@ private function populateDocumentsRelationships(Document $collection, array $doc
             // Add collection attribute for proper cycle detection - this is critical!
             $relationship->setAttribute('collection', $collection->getId());
 
-            // TEMP: Disable cycle detection to test if that's the issue
             // Skip if we should not fetch this relationship based on current fetch stack
-            // $skipFetch = $this->shouldSkipRelationshipFetchBatch($relationship, $collection);
-            // 
-            // if ($skipFetch) {
-            //     // Remove the relationship attribute from all documents
-            //     foreach ($documents as $document) {
-            //         $document->removeAttribute($key);
-            //     }
-            //     continue;
-            // }
+            $skipFetch = $this->shouldSkipRelationshipFetchBatch($relationship, $collection);
+            
+            if ($skipFetch) {
+                // Remove the relationship attribute from all documents
+                foreach ($documents as $document) {
+                    $document->removeAttribute($key);
+                }
+                continue;
+            }
 
             switch ($relationType) {
                 case Database::RELATION_ONE_TO_ONE:
@@ -3833,12 +3832,14 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
         foreach ($relatedDocuments as $related) {
             $parentId = $related->getAttribute($twoWayKey);
             if (!\is_null($parentId)) {
-                if (!isset($relatedByParentId[$parentId])) {
-                    $relatedByParentId[$parentId] = [];
+                // Handle case where parentId might be a Document object instead of string
+                $parentKey = $parentId instanceof Document ? $parentId->getId() : $parentId;
+                if (!isset($relatedByParentId[$parentKey])) {
+                    $relatedByParentId[$parentKey] = [];
                 }
                 // Remove the back-reference to avoid cycles
                 $related->removeAttribute($twoWayKey);
-                $relatedByParentId[$parentId][] = $related;
+                $relatedByParentId[$parentKey][] = $related;
             }
         }
 
@@ -3919,12 +3920,14 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
         foreach ($relatedDocuments as $related) {
             $childId = $related->getAttribute($twoWayKey);
             if (!\is_null($childId)) {
-                if (!isset($relatedByChildId[$childId])) {
-                    $relatedByChildId[$childId] = [];
+                // Handle case where childId might be a Document object instead of string
+                $childKey = $childId instanceof Document ? $childId->getId() : $childId;
+                if (!isset($relatedByChildId[$childKey])) {
+                    $relatedByChildId[$childKey] = [];
                 }
                 // Remove the back-reference to avoid cycles
                 $related->removeAttribute($twoWayKey);
-                $relatedByChildId[$childId][] = $related;
+                $relatedByChildId[$childKey][] = $related;
             }
         }
 

From a41b7cec6bc76ce9655deb7b7cc89ddf82bde7a0 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 17:49:42 +0000
Subject: [PATCH 14/45] TEMP: Simplify cycle detection to only check max depth
 to test if complex cycle detection is causing issues

---
 src/Database/Database.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 38c5966e5..e8dfd23e2 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3604,8 +3604,8 @@ private function populateDocumentsRelationships(Document $collection, array $doc
             // Add collection attribute for proper cycle detection - this is critical!
             $relationship->setAttribute('collection', $collection->getId());
 
-            // Skip if we should not fetch this relationship based on current fetch stack
-            $skipFetch = $this->shouldSkipRelationshipFetchBatch($relationship, $collection);
+            // TEMP: Simplify cycle detection to only check max depth
+            $skipFetch = $twoWay && ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH);
             
             if ($skipFetch) {
                 // Remove the relationship attribute from all documents

From 3d227ed471860190e733a64b654caba07c1f5b62 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 17:57:11 +0000
Subject: [PATCH 15/45] Improve cycle detection: Use property-based comparison
 instead of Document object equality

- Replace Document == comparison with specific property matching
- Compare key, collection, and relatedCollection for reflexive detection
- Should fix cycle detection issues while preventing infinite recursion
---
 src/Database/Database.php | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index e8dfd23e2..ce08f2b7d 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3604,8 +3604,8 @@ private function populateDocumentsRelationships(Document $collection, array $doc
             // Add collection attribute for proper cycle detection - this is critical!
             $relationship->setAttribute('collection', $collection->getId());
 
-            // TEMP: Simplify cycle detection to only check max depth
-            $skipFetch = $twoWay && ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH);
+            // Skip if we should not fetch this relationship based on current fetch stack
+            $skipFetch = $this->shouldSkipRelationshipFetchBatch($relationship, $collection);
             
             if ($skipFetch) {
                 // Remove the relationship attribute from all documents
@@ -3664,7 +3664,10 @@ private function shouldSkipRelationshipFetchBatch(Document $relationship, Docume
             $existingSide = $fetchedRelationship['options']['side'];
 
             // If this relationship has already been fetched for this document, skip it
-            $reflexive = $fetchedRelationship == $relationship;
+            // Compare by key identifying properties instead of full Document comparison
+            $reflexive = ($fetchedRelationship['key'] === $relationship['key'] &&
+                         $fetchedRelationship['collection'] === $relationship['collection'] &&
+                         $fetchedRelationship['options']['relatedCollection'] === $relationship['options']['relatedCollection']);
 
             // If this relationship is the same as a previously fetched relationship, but on the other side, skip it
             $symmetric = $existingKey === $twoWayKey

From f6cecd6d4b0c2acca144e6341bde7fb49349a9a2 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 18:06:11 +0000
Subject: [PATCH 16/45] Revert to simpler cycle detection - only check max
 depth

- Complex cycle detection was over-blocking legitimate relationships
- Simpler approach showed better results (8 errors vs 15 errors)
- Focus on max depth limit for two-way relationships only
- Keep back-reference removal as in original implementation
---
 src/Database/Database.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index ce08f2b7d..d06ed1c2a 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3604,8 +3604,8 @@ private function populateDocumentsRelationships(Document $collection, array $doc
             // Add collection attribute for proper cycle detection - this is critical!
             $relationship->setAttribute('collection', $collection->getId());
 
-            // Skip if we should not fetch this relationship based on current fetch stack
-            $skipFetch = $this->shouldSkipRelationshipFetchBatch($relationship, $collection);
+            // Use simpler cycle detection - only check max depth for two-way relationships
+            $skipFetch = $twoWay && ($this->relationshipFetchDepth >= Database::RELATION_MAX_DEPTH);
             
             if ($skipFetch) {
                 // Remove the relationship attribute from all documents

From 8b58a58384700b6fe6f7ffeeed0507895d4035e5 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 18:11:53 +0000
Subject: [PATCH 17/45] REDESIGN: Implement breadth-first cycle detection for
 batch processing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Replace depth-first cycle detection with collection-level cycle detection
- Track collection→collection paths instead of document-level relationships
- Detect direct cycles (A→B, B→A) and indirect cycles (A→B→A)
- Designed specifically for batch/breadth-first relationship traversal
- Should prevent infinite loops while allowing legitimate nested relationships
---
 src/Database/Database.php | 80 ++++++++++++++++++---------------------
 1 file changed, 36 insertions(+), 44 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index d06ed1c2a..a12141f3a 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3604,8 +3604,8 @@ private function populateDocumentsRelationships(Document $collection, array $doc
             // Add collection attribute for proper cycle detection - this is critical!
             $relationship->setAttribute('collection', $collection->getId());
 
-            // Use simpler cycle detection - only check max depth for two-way relationships
-            $skipFetch = $twoWay && ($this->relationshipFetchDepth >= Database::RELATION_MAX_DEPTH);
+            // Use batch-aware cycle detection designed for breadth-first traversal
+            $skipFetch = $this->shouldSkipRelationshipFetchBatch($relationship, $collection);
             
             if ($skipFetch) {
                 // Remove the relationship attribute from all documents
@@ -3638,7 +3638,12 @@ private function populateDocumentsRelationships(Document $collection, array $doc
     }
 
     /**
-     * Check if a relationship should be skipped based on fetch stack and depth (batch version)
+     * Check if a relationship should be skipped based on batch cycle detection
+     * 
+     * Batch processing uses breadth-first traversal, so we need different cycle detection:
+     * - Track collection-to-collection relationships to prevent collection loops
+     * - Use depth limits to prevent infinite nesting
+     * - Detect direct collection cycles (A→B→A)
      *
      * @param Document $relationship
      * @param Document $collection
@@ -3646,53 +3651,40 @@ private function populateDocumentsRelationships(Document $collection, array $doc
      */
     private function shouldSkipRelationshipFetchBatch(Document $relationship, Document $collection): bool
     {
-        $key = $relationship['key'];
-        $twoWay = $relationship['options']['twoWay'];
-        $twoWayKey = $relationship['options']['twoWayKey'];
-        $side = $relationship['options']['side'];
+        $currentCollectionId = $collection->getId();
         $relatedCollectionId = $relationship['options']['relatedCollection'];
+        $twoWay = $relationship['options']['twoWay'];
+        $key = $relationship['key'];
 
-        if ($twoWay && ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH)) {
+        // Always respect max depth limit for two-way relationships
+        if ($twoWay && ($this->relationshipFetchDepth >= Database::RELATION_MAX_DEPTH)) {
             return true;
         }
 
+        // For batch processing, detect collection-level cycles
+        // Track the path of collections being processed
+        $collectionPath = [];
         foreach ($this->relationshipFetchStack as $fetchedRelationship) {
-            $existingKey = $fetchedRelationship['key'];
-            $existingCollection = $fetchedRelationship['collection'];
-            $existingRelatedCollection = $fetchedRelationship['options']['relatedCollection'];
-            $existingTwoWayKey = $fetchedRelationship['options']['twoWayKey'];
-            $existingSide = $fetchedRelationship['options']['side'];
-
-            // If this relationship has already been fetched for this document, skip it
-            // Compare by key identifying properties instead of full Document comparison
-            $reflexive = ($fetchedRelationship['key'] === $relationship['key'] &&
-                         $fetchedRelationship['collection'] === $relationship['collection'] &&
-                         $fetchedRelationship['options']['relatedCollection'] === $relationship['options']['relatedCollection']);
-
-            // If this relationship is the same as a previously fetched relationship, but on the other side, skip it
-            $symmetric = $existingKey === $twoWayKey
-                && $existingTwoWayKey === $key
-                && $existingRelatedCollection === $collection->getId()
-                && $existingCollection === $relatedCollectionId
-                && $existingSide !== $side;
-
-            // Transitive relationship detection
-            $transitive = (($existingKey === $twoWayKey
-                    && $existingCollection === $relatedCollectionId
-                    && $existingSide !== $side)
-                || ($existingTwoWayKey === $key
-                    && $existingRelatedCollection === $collection->getId()
-                    && $existingSide !== $side)
-                || ($existingKey === $key
-                    && $existingTwoWayKey !== $twoWayKey
-                    && $existingRelatedCollection === $relatedCollectionId
-                    && $existingSide !== $side)
-                || ($existingKey !== $key
-                    && $existingTwoWayKey === $twoWayKey
-                    && $existingRelatedCollection === $relatedCollectionId
-                    && $existingSide !== $side));
-
-            if ($reflexive || $symmetric || $transitive) {
+            $fromCollection = $fetchedRelationship['collection'] ?? null;
+            $toCollection = $fetchedRelationship['options']['relatedCollection'] ?? null;
+            if ($fromCollection && $toCollection) {
+                $collectionPath[] = $fromCollection . '→' . $toCollection;
+            }
+        }
+
+        // Check if adding this relationship would create a collection cycle
+        $newPath = $currentCollectionId . '→' . $relatedCollectionId;
+        
+        // Direct cycle: if we're going from A→B and we already have B→A
+        $reversePath = $relatedCollectionId . '→' . $currentCollectionId;
+        if (in_array($reversePath, $collectionPath)) {
+            return true;
+        }
+
+        // Indirect cycle: if we're going to a collection we've already processed from
+        foreach ($collectionPath as $existingPath) {
+            if (str_starts_with($existingPath, $relatedCollectionId . '→')) {
+                // We're about to fetch from a collection we've already fetched from
                 return true;
             }
         }

From a4d022260d71c232bc997fff2af53979789dcf79 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 18:20:39 +0000
Subject: [PATCH 18/45] FIX: Make cycle detection less restrictive for
 breadth-first processing

- Only block exact same relationship (same key + collections) being processed
- Only block direct back-references for two-way relationships
- Remove overly broad collection-level blocking that prevented legitimate relationships
- Should allow multiple different relationships to same collection
---
 src/Database/Database.php | 38 ++++++++++++++++----------------------
 1 file changed, 16 insertions(+), 22 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index a12141f3a..4d6d6b3da 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3661,30 +3661,24 @@ private function shouldSkipRelationshipFetchBatch(Document $relationship, Docume
             return true;
         }
 
-        // For batch processing, detect collection-level cycles
-        // Track the path of collections being processed
-        $collectionPath = [];
+        // For batch processing, only block true cycles that would cause infinite recursion
+        // Check if this exact relationship is already being processed
         foreach ($this->relationshipFetchStack as $fetchedRelationship) {
-            $fromCollection = $fetchedRelationship['collection'] ?? null;
-            $toCollection = $fetchedRelationship['options']['relatedCollection'] ?? null;
-            if ($fromCollection && $toCollection) {
-                $collectionPath[] = $fromCollection . '→' . $toCollection;
+            $existingCollection = $fetchedRelationship['collection'] ?? null;
+            $existingRelated = $fetchedRelationship['options']['relatedCollection'] ?? null;
+            $existingKey = $fetchedRelationship['key'] ?? null;
+            
+            // Only block if it's the EXACT same relationship (same key, same collections)
+            if ($existingCollection === $currentCollectionId && 
+                $existingRelated === $relatedCollectionId && 
+                $existingKey === $key) {
+                return true;
             }
-        }
-
-        // Check if adding this relationship would create a collection cycle
-        $newPath = $currentCollectionId . '→' . $relatedCollectionId;
-        
-        // Direct cycle: if we're going from A→B and we already have B→A
-        $reversePath = $relatedCollectionId . '→' . $currentCollectionId;
-        if (in_array($reversePath, $collectionPath)) {
-            return true;
-        }
-
-        // Indirect cycle: if we're going to a collection we've already processed from
-        foreach ($collectionPath as $existingPath) {
-            if (str_starts_with($existingPath, $relatedCollectionId . '→')) {
-                // We're about to fetch from a collection we've already fetched from
+            
+            // Block direct back-reference cycles only for two-way relationships
+            if ($twoWay && 
+                $existingCollection === $relatedCollectionId && 
+                $existingRelated === $currentCollectionId) {
                 return true;
             }
         }

From 31873c7484c657274d52d0130c1cfa2d5e541cf6 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 18:24:54 +0000
Subject: [PATCH 19/45] SIMPLIFY: Much more permissive cycle detection for
 breadth-first processing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Remove complex collection-level cycle tracking that was over-blocking relationships
- Use simple depth-based limits only: max depth for all, max-1 for two-way
- Should allow legitimate nested relationships like veterinarians→presidents→animals
- Still prevents infinite recursion through depth limits
---
 src/Database/Database.php | 41 +++++++++++----------------------------
 1 file changed, 11 insertions(+), 30 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 4d6d6b3da..62e5824b2 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3640,10 +3640,10 @@ private function populateDocumentsRelationships(Document $collection, array $doc
     /**
      * Check if a relationship should be skipped based on batch cycle detection
      * 
-     * Batch processing uses breadth-first traversal, so we need different cycle detection:
-     * - Track collection-to-collection relationships to prevent collection loops
-     * - Use depth limits to prevent infinite nesting
-     * - Detect direct collection cycles (A→B→A)
+     * Simplified cycle detection for batch processing:
+     * - Respect max depth for any relationship type
+     * - Block exact duplicate relationships only
+     * - Less restrictive than depth-first approach to allow legitimate nested relationships
      *
      * @param Document $relationship
      * @param Document $collection
@@ -3651,36 +3651,17 @@ private function populateDocumentsRelationships(Document $collection, array $doc
      */
     private function shouldSkipRelationshipFetchBatch(Document $relationship, Document $collection): bool
     {
-        $currentCollectionId = $collection->getId();
-        $relatedCollectionId = $relationship['options']['relatedCollection'];
-        $twoWay = $relationship['options']['twoWay'];
-        $key = $relationship['key'];
+        $twoWay = $relationship['options']['twoWay'] ?? false;
 
-        // Always respect max depth limit for two-way relationships
-        if ($twoWay && ($this->relationshipFetchDepth >= Database::RELATION_MAX_DEPTH)) {
+        // Respect max depth limit to prevent infinite recursion
+        if ($this->relationshipFetchDepth >= Database::RELATION_MAX_DEPTH) {
             return true;
         }
 
-        // For batch processing, only block true cycles that would cause infinite recursion
-        // Check if this exact relationship is already being processed
-        foreach ($this->relationshipFetchStack as $fetchedRelationship) {
-            $existingCollection = $fetchedRelationship['collection'] ?? null;
-            $existingRelated = $fetchedRelationship['options']['relatedCollection'] ?? null;
-            $existingKey = $fetchedRelationship['key'] ?? null;
-            
-            // Only block if it's the EXACT same relationship (same key, same collections)
-            if ($existingCollection === $currentCollectionId && 
-                $existingRelated === $relatedCollectionId && 
-                $existingKey === $key) {
-                return true;
-            }
-            
-            // Block direct back-reference cycles only for two-way relationships
-            if ($twoWay && 
-                $existingCollection === $relatedCollectionId && 
-                $existingRelated === $currentCollectionId) {
-                return true;
-            }
+        // For breadth-first batch processing, be more permissive
+        // Only block if we're at the very edge of max depth for two-way relationships
+        if ($twoWay && ($this->relationshipFetchDepth >= (Database::RELATION_MAX_DEPTH - 1))) {
+            return true;
         }
 
         return false;

From dedc4889839d1f3e15b1fa8d8b6b34ac24a72d4b Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 19:00:02 +0000
Subject: [PATCH 20/45] EXPERIMENT: Remove cycle detection entirely for
 breadth-first processing

Per user insight: In breadth-first batch processing:
- Depth control prevents infinite recursion (process by levels)
- Map prevents duplicate fetches (no need to track cycles)
- Level-by-level processing can't create infinite loops

Testing if cycle detection is unnecessary in batch scenario.
---
 src/Database/Database.php | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 62e5824b2..e3d626687 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3604,8 +3604,11 @@ private function populateDocumentsRelationships(Document $collection, array $doc
             // Add collection attribute for proper cycle detection - this is critical!
             $relationship->setAttribute('collection', $collection->getId());
 
-            // Use batch-aware cycle detection designed for breadth-first traversal
-            $skipFetch = $this->shouldSkipRelationshipFetchBatch($relationship, $collection);
+            // In breadth-first batch processing, cycle detection is not needed:
+            // - Depth control prevents infinite recursion 
+            // - Map prevents duplicate fetches
+            // - Level-by-level processing can't create cycles
+            $skipFetch = false;
             
             if ($skipFetch) {
                 // Remove the relationship attribute from all documents

From 0c68374645eb0a5c1e9ef1859aa272c4af909cd8 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 19:01:09 +0000
Subject: [PATCH 21/45] EXPERIMENT: Pure breadth-first processing with minimal
 cycle prevention

- Remove all cycle detection logic, rely only on depth control
- Add map tracking to all batch methods to prevent duplicate fetches
- Test theory: level-by-level processing + depth control = no infinite recursion possible
- Should allow maximum relationship population with minimal restrictions
---
 src/Database/Database.php | 35 +++++++++++++++++++++++++----------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index e3d626687..0aac9025e 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3604,10 +3604,10 @@ private function populateDocumentsRelationships(Document $collection, array $doc
             // Add collection attribute for proper cycle detection - this is critical!
             $relationship->setAttribute('collection', $collection->getId());
 
-            // In breadth-first batch processing, cycle detection is not needed:
-            // - Depth control prevents infinite recursion 
-            // - Map prevents duplicate fetches
-            // - Level-by-level processing can't create cycles
+            // In breadth-first batch processing, rely only on depth control:
+            // - Level-by-level processing prevents infinite cycles naturally
+            // - Depth limit prevents infinite recursion
+            // - Map tracking in individual methods prevents duplicate fetches  
             $skipFetch = false;
             
             if ($skipFetch) {
@@ -3777,10 +3777,15 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
             return;
         }
 
-        // Collect all parent document IDs
+        // Collect all parent document IDs and track in map
         $parentIds = [];
         foreach ($documents as $document) {
-            $parentIds[] = $document->getId();
+            $parentId = $document->getId();
+            $parentIds[] = $parentId;
+            
+            // Add to map for duplicate prevention (one-to-many: parent -> children)
+            $k = $collection->getId() . ':' . $parentId . '=>' . $relatedCollection->getId() . ':*';
+            $this->map[$k] = true;
         }
 
         if (empty($parentIds)) {
@@ -3865,10 +3870,15 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
             return;
         }
 
-        // Collect all child document IDs
+        // Collect all child document IDs and track in map
         $childIds = [];
         foreach ($documents as $document) {
-            $childIds[] = $document->getId();
+            $childId = $document->getId();
+            $childIds[] = $childId;
+            
+            // Add to map for duplicate prevention (many-to-one: children -> parents)
+            $k = $collection->getId() . ':' . $childId . '=>' . $relatedCollection->getId() . ':*';
+            $this->map[$k] = true;
         }
 
         if (empty($childIds)) {
@@ -3937,10 +3947,15 @@ private function populateManyToManyRelationshipsBatch(array $documents, Document
             return;
         }
 
-        // Collect all document IDs
+        // Collect all document IDs and track in map
         $documentIds = [];
         foreach ($documents as $document) {
-            $documentIds[] = $document->getId();
+            $documentId = $document->getId();
+            $documentIds[] = $documentId;
+            
+            // Add to map for duplicate prevention (many-to-many: documents -> related)
+            $k = $collection->getId() . ':' . $documentId . '=>' . $relatedCollection->getId() . ':*';
+            $this->map[$k] = true;
         }
 
         if (empty($documentIds)) {

From db23d7683936d1c065acb247c4b4eda21c3e4b7c Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 19:12:50 +0000
Subject: [PATCH 22/45] OPTIMIZE: Map-based duplicate prevention replaces cycle
 detection

Your insight implemented:
- Use map to check if relationships already processed BEFORE collecting IDs
- Skip duplicate relationship processing automatically
- No cycle detection needed - map + depth control is sufficient
- Should eliminate redundant database queries and improve performance significantly
---
 src/Database/Database.php | 61 +++++++++++++++++++--------------------
 1 file changed, 29 insertions(+), 32 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 0aac9025e..101d36528 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3601,22 +3601,12 @@ private function populateDocumentsRelationships(Document $collection, array $doc
             $side = $relationship['options']['side'];
             $queries = $selects[$key] ?? [];
 
-            // Add collection attribute for proper cycle detection - this is critical!
+            // Add collection attribute for relationship context
             $relationship->setAttribute('collection', $collection->getId());
 
-            // In breadth-first batch processing, rely only on depth control:
-            // - Level-by-level processing prevents infinite cycles naturally
-            // - Depth limit prevents infinite recursion
-            // - Map tracking in individual methods prevents duplicate fetches  
-            $skipFetch = false;
-            
-            if ($skipFetch) {
-                // Remove the relationship attribute from all documents
-                foreach ($documents as $document) {
-                    $document->removeAttribute($key);
-                }
-                continue;
-            }
+            // EXPERIMENT: Pure breadth-first processing with NO cycle detection
+            // Theory: Level-by-level processing + depth control prevents infinite recursion
+            // Map tracking prevents duplicate fetches, reverse mapping handles result assignment
 
             switch ($relationType) {
                 case Database::RELATION_ONE_TO_ONE:
@@ -3692,19 +3682,20 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
             return;
         }
 
-        // Collect all related document IDs
+        // Collect all related document IDs, checking map to avoid duplicates
         $relatedIds = [];
         $documentsByRelatedId = [];
         
         foreach ($documents as $document) {
             $value = $document->getAttribute($key);
             if (!\is_null($value)) {
-                $relatedIds[] = $value;
-                $documentsByRelatedId[$value] = $document;
-                
-                // Add to map for cycle detection
+                // Check if this relationship has already been processed
                 $k = $relatedCollection->getId() . ':' . $value . '=>' . $document->getCollection() . ':' . $document->getId();
-                $this->map[$k] = true;
+                if (!isset($this->map[$k])) {
+                    $relatedIds[] = $value;
+                    $documentsByRelatedId[$value] = $document;
+                    $this->map[$k] = true;
+                }
             }
         }
 
@@ -3777,15 +3768,17 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
             return;
         }
 
-        // Collect all parent document IDs and track in map
+        // Collect all parent document IDs, checking map to avoid duplicates
         $parentIds = [];
         foreach ($documents as $document) {
             $parentId = $document->getId();
-            $parentIds[] = $parentId;
             
-            // Add to map for duplicate prevention (one-to-many: parent -> children)
+            // Check if this parent->children relationship has already been processed
             $k = $collection->getId() . ':' . $parentId . '=>' . $relatedCollection->getId() . ':*';
-            $this->map[$k] = true;
+            if (!isset($this->map[$k])) {
+                $parentIds[] = $parentId;
+                $this->map[$k] = true;
+            }
         }
 
         if (empty($parentIds)) {
@@ -3870,15 +3863,17 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
             return;
         }
 
-        // Collect all child document IDs and track in map
+        // Collect all child document IDs, checking map to avoid duplicates
         $childIds = [];
         foreach ($documents as $document) {
             $childId = $document->getId();
-            $childIds[] = $childId;
             
-            // Add to map for duplicate prevention (many-to-one: children -> parents)
+            // Check if this child->parent relationship has already been processed
             $k = $collection->getId() . ':' . $childId . '=>' . $relatedCollection->getId() . ':*';
-            $this->map[$k] = true;
+            if (!isset($this->map[$k])) {
+                $childIds[] = $childId;
+                $this->map[$k] = true;
+            }
         }
 
         if (empty($childIds)) {
@@ -3947,15 +3942,17 @@ private function populateManyToManyRelationshipsBatch(array $documents, Document
             return;
         }
 
-        // Collect all document IDs and track in map
+        // Collect all document IDs, checking map to avoid duplicates
         $documentIds = [];
         foreach ($documents as $document) {
             $documentId = $document->getId();
-            $documentIds[] = $documentId;
             
-            // Add to map for duplicate prevention (many-to-many: documents -> related)
+            // Check if this document->related relationship has already been processed
             $k = $collection->getId() . ':' . $documentId . '=>' . $relatedCollection->getId() . ':*';
-            $this->map[$k] = true;
+            if (!isset($this->map[$k])) {
+                $documentIds[] = $documentId;
+                $this->map[$k] = true;
+            }
         }
 
         if (empty($documentIds)) {

From 0a3ef8c1cfabecc4053a1ea3e0f6e966ff26d4d7 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 6 Aug 2025 19:13:34 +0000
Subject: [PATCH 23/45] ULTIMATE OPTIMIZATION: Skip already-populated
 relationships + map-based deduplication

- Skip one-to-one relationships already populated (value is Document object)
- Skip array relationships already populated (array contains Document objects)
- Map prevents duplicate processing across calls
- No cycle detection needed - breadth-first + depth control + smart skipping
- Should achieve maximum efficiency with minimal redundant work
---
 src/Database/Database.php | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 101d36528..858d4504e 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3689,6 +3689,11 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
         foreach ($documents as $document) {
             $value = $document->getAttribute($key);
             if (!\is_null($value)) {
+                // Skip if value is already a Document object (already populated)
+                if ($value instanceof Document) {
+                    continue;
+                }
+                
                 // Check if this relationship has already been processed
                 $k = $relatedCollection->getId() . ':' . $value . '=>' . $document->getCollection() . ':' . $document->getId();
                 if (!isset($this->map[$k])) {
@@ -3771,6 +3776,12 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
         // Collect all parent document IDs, checking map to avoid duplicates
         $parentIds = [];
         foreach ($documents as $document) {
+            // Skip if relationship is already populated (array of Documents)
+            $existingValue = $document->getAttribute($key);
+            if (!empty($existingValue) && is_array($existingValue) && !empty($existingValue[0]) && $existingValue[0] instanceof Document) {
+                continue;
+            }
+            
             $parentId = $document->getId();
             
             // Check if this parent->children relationship has already been processed
@@ -3866,6 +3877,12 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
         // Collect all child document IDs, checking map to avoid duplicates
         $childIds = [];
         foreach ($documents as $document) {
+            // Skip if relationship is already populated (array of Documents)
+            $existingValue = $document->getAttribute($key);
+            if (!empty($existingValue) && is_array($existingValue) && !empty($existingValue[0]) && $existingValue[0] instanceof Document) {
+                continue;
+            }
+            
             $childId = $document->getId();
             
             // Check if this child->parent relationship has already been processed
@@ -3945,6 +3962,12 @@ private function populateManyToManyRelationshipsBatch(array $documents, Document
         // Collect all document IDs, checking map to avoid duplicates
         $documentIds = [];
         foreach ($documents as $document) {
+            // Skip if relationship is already populated (array of Documents)
+            $existingValue = $document->getAttribute($key);
+            if (!empty($existingValue) && is_array($existingValue) && !empty($existingValue[0]) && $existingValue[0] instanceof Document) {
+                continue;
+            }
+            
             $documentId = $document->getId();
             
             // Check if this document->related relationship has already been processed

From 34511fe1400f17f042d0f85b143285e0d6d0c523 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Thu, 7 Aug 2025 04:17:19 +0000
Subject: [PATCH 24/45] DEBUG: Add extensive logging to track relationship
 population issues

- Add debug output in testZoo to see presidents directly
- Add logging in one-to-one batch method to track ID collection and fetching
- Add logging to main relationship processing loop
- Should help identify why some relationships populate and others don't
---
 src/Database/Database.php                      |  9 +++++++++
 tests/e2e/Adapter/Scopes/RelationshipTests.php | 10 +++++++++-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 858d4504e..27cd5151f 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3607,6 +3607,8 @@ private function populateDocumentsRelationships(Document $collection, array $doc
             // EXPERIMENT: Pure breadth-first processing with NO cycle detection
             // Theory: Level-by-level processing + depth control prevents infinite recursion
             // Map tracking prevents duplicate fetches, reverse mapping handles result assignment
+            
+            error_log("Processing relationship '$key' of type '$relationType' for collection '" . $collection->getId() . "' with " . count($documents) . " documents at depth " . $this->relationshipFetchDepth);
 
             switch ($relationType) {
                 case Database::RELATION_ONE_TO_ONE:
@@ -3700,13 +3702,18 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
                     $relatedIds[] = $value;
                     $documentsByRelatedId[$value] = $document;
                     $this->map[$k] = true;
+                } else {
+                    error_log("OneToOne: Skipping already processed relationship '$k'");
                 }
             }
         }
 
         if (empty($relatedIds)) {
+            error_log("OneToOne: No related IDs to fetch for key '$key'");
             return;
         }
+        
+        error_log("OneToOne: Fetching " . count($relatedIds) . " related docs for key '$key': " . implode(', ', $relatedIds));
 
         // Fetch all related documents in a single query
         $this->relationshipFetchDepth++;
@@ -3717,6 +3724,8 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
             Query::limit(PHP_INT_MAX),
             ...$queries
         ]);
+        
+        error_log("OneToOne: Found " . count($relatedDocuments) . " related docs for key '$key'");
 
         $this->relationshipFetchDepth--;
         \array_pop($this->relationshipFetchStack);
diff --git a/tests/e2e/Adapter/Scopes/RelationshipTests.php b/tests/e2e/Adapter/Scopes/RelationshipTests.php
index edc2f3e0b..98a32badf 100644
--- a/tests/e2e/Adapter/Scopes/RelationshipTests.php
+++ b/tests/e2e/Adapter/Scopes/RelationshipTests.php
@@ -228,12 +228,20 @@ public function testZoo(): void
                     '*',
                     'animals.*',
                     'animals.zoo.*',
-                    //'animals.president.*',
+                    'animals.president.*',
+                    'presidents.*',
+                    'presidents.animal.*',
                 ])
             ]
         );
 
+        var_dump('=== VETERINARIANS RESULT ===');
         var_dump($docs);
+        
+        // Let's also check what presidents look like directly
+        $presidents = $database->find('presidents');
+        var_dump('=== PRESIDENTS DIRECTLY ===');
+        var_dump($presidents);
 
         //$this->assertEquals('shmuel', 'fogel');
     }

From 52723c2e44a8a82745aef056ef5272d05614a373 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Thu, 7 Aug 2025 12:05:36 +0000
Subject: [PATCH 25/45] DEBUG: Add var_dump for animal relationships
 specifically

- Add targeted debugging for 'animal' relationship key
- Should show in test output when presidents->animal relationships are processed
- Will help identify if IDs are collected and if animals are found
---
 src/Database/Database.php | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 27cd5151f..ddfa6522a 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3714,6 +3714,9 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
         }
         
         error_log("OneToOne: Fetching " . count($relatedIds) . " related docs for key '$key': " . implode(', ', $relatedIds));
+        if ($key === 'animal') {
+            var_dump("DEBUG OneToOne: About to fetch animals with IDs: " . implode(', ', $relatedIds));
+        }
 
         // Fetch all related documents in a single query
         $this->relationshipFetchDepth++;
@@ -3726,6 +3729,10 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
         ]);
         
         error_log("OneToOne: Found " . count($relatedDocuments) . " related docs for key '$key'");
+        if ($key === 'animal') {
+            var_dump("DEBUG OneToOne: Found " . count($relatedDocuments) . " animals");
+            var_dump($relatedDocuments);
+        }
 
         $this->relationshipFetchDepth--;
         \array_pop($this->relationshipFetchStack);

From bffb3be1962c8cb0c31735a47a2120aa215d50d6 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Thu, 7 Aug 2025 12:07:35 +0000
Subject: [PATCH 26/45] MAJOR FIX: Handle multiple documents referencing same
 related ID in one-to-one

Root cause found: When multiple animals reference same president (biden),
the second animal was overwriting the first in documentsByRelatedId.

Changes:
- Track arrays of documents per related ID instead of single document
- Set relationship for ALL documents that reference the same related ID
- Should fix inconsistent relationship population (iguana vs tiger)
---
 src/Database/Database.php | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index ddfa6522a..323c0a2bd 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3696,15 +3696,17 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
                     continue;
                 }
                 
-                // Check if this relationship has already been processed
-                $k = $relatedCollection->getId() . ':' . $value . '=>' . $document->getCollection() . ':' . $document->getId();
-                if (!isset($this->map[$k])) {
-                    $relatedIds[] = $value;
-                    $documentsByRelatedId[$value] = $document;
-                    $this->map[$k] = true;
-                } else {
-                    error_log("OneToOne: Skipping already processed relationship '$k'");
+                // For one-to-one, multiple documents can reference the same related ID
+                // We need to track ALL documents that reference each related ID
+                $relatedIds[] = $value;
+                if (!isset($documentsByRelatedId[$value])) {
+                    $documentsByRelatedId[$value] = [];
                 }
+                $documentsByRelatedId[$value][] = $document;
+                
+                // Map tracks that we're processing this specific document->related relationship
+                $k = $relatedCollection->getId() . ':' . $value . '=>' . $document->getCollection() . ':' . $document->getId();
+                $this->map[$k] = true;
             }
         }
 
@@ -3744,12 +3746,17 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
         }
 
         // Assign related documents to their parent documents
-        foreach ($documentsByRelatedId as $relatedId => $document) {
+        foreach ($documentsByRelatedId as $relatedId => $documents) {
             if (isset($relatedById[$relatedId])) {
-                $document->setAttribute($key, $relatedById[$relatedId]);
+                // Set the relationship for all documents that reference this related ID
+                foreach ($documents as $document) {
+                    $document->setAttribute($key, $relatedById[$relatedId]);
+                }
             } else {
                 // If related document not found, set to null instead of leaving the string ID
-                $document->setAttribute($key, null);
+                foreach ($documents as $document) {
+                    $document->setAttribute($key, null);
+                }
             }
         }
     }

From f6a05fc14af317b581b6817c9d49a11de7ed2514 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Thu, 7 Aug 2025 12:18:24 +0000
Subject: [PATCH 27/45] CLEANUP: Remove excessive animal-specific debug output

- Keep general error logging but remove targeted var_dumps
- One-to-one fix applied, ready to tackle remaining issues
- Errors reduced from 13 to 10, making progress
---
 src/Database/Database.php | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 323c0a2bd..3021d943f 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3716,9 +3716,6 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
         }
         
         error_log("OneToOne: Fetching " . count($relatedIds) . " related docs for key '$key': " . implode(', ', $relatedIds));
-        if ($key === 'animal') {
-            var_dump("DEBUG OneToOne: About to fetch animals with IDs: " . implode(', ', $relatedIds));
-        }
 
         // Fetch all related documents in a single query
         $this->relationshipFetchDepth++;
@@ -3731,10 +3728,6 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
         ]);
         
         error_log("OneToOne: Found " . count($relatedDocuments) . " related docs for key '$key'");
-        if ($key === 'animal') {
-            var_dump("DEBUG OneToOne: Found " . count($relatedDocuments) . " animals");
-            var_dump($relatedDocuments);
-        }
 
         $this->relationshipFetchDepth--;
         \array_pop($this->relationshipFetchStack);

From 52628e29e8cee3c418d93dc2f21e4e91f5fe548a Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Thu, 7 Aug 2025 12:37:59 +0000
Subject: [PATCH 28/45] Add detailed debugging to relationship population

- Add error_log statements to track query processing
- Debug nested selections flow
- Track queries passed to batch methods
- Monitor find() calls in relationship population
---
 src/Database/Database.php                     | 30 ++++++++++++++++---
 .../e2e/Adapter/Scopes/RelationshipTests.php  |  8 +++++
 2 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 3021d943f..6bcc2b8a6 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3609,21 +3609,26 @@ private function populateDocumentsRelationships(Document $collection, array $doc
             // Map tracking prevents duplicate fetches, reverse mapping handles result assignment
             
             error_log("Processing relationship '$key' of type '$relationType' for collection '" . $collection->getId() . "' with " . count($documents) . " documents at depth " . $this->relationshipFetchDepth);
+        error_log("Select queries for '$key': " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
 
             switch ($relationType) {
                 case Database::RELATION_ONE_TO_ONE:
+                    error_log("Calling populateOneToOneRelationshipsBatch for '$key' with queries: " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
                     $this->populateOneToOneRelationshipsBatch($documents, $relationship, $relatedCollection, $queries);
                     break;
                     
                 case Database::RELATION_ONE_TO_MANY:
+                    error_log("Calling populateOneToManyRelationshipsBatch for '$key' with queries: " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
                     $this->populateOneToManyRelationshipsBatch($documents, $relationship, $relatedCollection, $queries, $collection);
                     break;
                     
                 case Database::RELATION_MANY_TO_ONE:
+                    error_log("Calling populateManyToOneRelationshipsBatch for '$key' with queries: " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
                     $this->populateManyToOneRelationshipsBatch($documents, $relationship, $relatedCollection, $queries, $collection);
                     break;
                     
                 case Database::RELATION_MANY_TO_MANY:
+                    error_log("Calling populateManyToManyRelationshipsBatch for '$key' with queries: " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
                     $this->populateManyToManyRelationshipsBatch($documents, $relationship, $relatedCollection, $queries, $collection);
                     break;
             }
@@ -3721,13 +3726,24 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
         $this->relationshipFetchDepth++;
         $this->relationshipFetchStack[] = $relationship;
 
-        $relatedDocuments = $this->find($relatedCollection->getId(), [
+        $queryParams = [
             Query::equal('$id', array_unique($relatedIds)),
             Query::limit(PHP_INT_MAX),
             ...$queries
-        ]);
+        ];
+        
+        error_log("OneToOne: Executing find with queries: " . json_encode(array_map(fn($q) => $q->toString(), $queryParams)));
+        
+        $relatedDocuments = $this->find($relatedCollection->getId(), $queryParams);
         
         error_log("OneToOne: Found " . count($relatedDocuments) . " related docs for key '$key'");
+        
+        // Debug: check if the related documents have their relationships populated
+        if (!empty($relatedDocuments)) {
+            foreach ($relatedDocuments as $idx => $doc) {
+                error_log("OneToOne: Related doc $idx ID=" . $doc->getId() . " attrs=" . json_encode($doc->getArrayCopy()));
+            }
+        }
 
         $this->relationshipFetchDepth--;
         \array_pop($this->relationshipFetchStack);
@@ -3816,11 +3832,15 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
         $this->relationshipFetchDepth++;
         $this->relationshipFetchStack[] = $relationship;
 
-        $relatedDocuments = $this->find($relatedCollection->getId(), [
+        $queryParams = [
             Query::equal($twoWayKey, $parentIds),
             Query::limit(PHP_INT_MAX),
             ...$queries
-        ]);
+        ];
+        
+        error_log("OneToMany: Executing find with queries: " . json_encode(array_map(fn($q) => $q->toString(), $queryParams)));
+        
+        $relatedDocuments = $this->find($relatedCollection->getId(), $queryParams);
 
         $this->relationshipFetchDepth--;
         \array_pop($this->relationshipFetchStack);
@@ -6644,6 +6664,8 @@ public function find(string $collection, array $queries = [], string $forPermiss
         // Use batch relationship population for better performance at all levels
         if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections))) {
             if (count($results) > 0) {
+                error_log("find(): About to populate relationships for collection '" . $collection->getId() . "' with " . count($results) . " results at depth " . $this->relationshipFetchDepth);
+                error_log("find(): nestedSelections = " . json_encode($nestedSelections));
                 // Always use batch processing for all cases (single and multiple documents, nested or top-level)
                 $results = $this->silent(fn () => $this->populateDocumentsRelationships($collection, $results, $nestedSelections));
             }
diff --git a/tests/e2e/Adapter/Scopes/RelationshipTests.php b/tests/e2e/Adapter/Scopes/RelationshipTests.php
index 98a32badf..f50534a00 100644
--- a/tests/e2e/Adapter/Scopes/RelationshipTests.php
+++ b/tests/e2e/Adapter/Scopes/RelationshipTests.php
@@ -220,6 +220,10 @@ public function testZoo(): void
         ]));
 
         var_dump('=== start === === start === === start === === start === === start === === start === === start === === start === === start ===');
+        
+        // Clear error log and start fresh
+        file_put_contents('/tmp/debug.log', '');
+        ini_set('error_log', '/tmp/debug.log');
 
         $docs = $database->find(
             'veterinarians',
@@ -242,6 +246,10 @@ public function testZoo(): void
         $presidents = $database->find('presidents');
         var_dump('=== PRESIDENTS DIRECTLY ===');
         var_dump($presidents);
+        
+        // Dump error log contents
+        var_dump('=== ERROR LOG ===');
+        var_dump(file_get_contents('/tmp/debug.log'));
 
         //$this->assertEquals('shmuel', 'fogel');
     }

From cd6b022179f578c9926ce8f71039c0027f418a8f Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Thu, 7 Aug 2025 12:39:40 +0000
Subject: [PATCH 29/45] Add simple relationship test to debug basic
 functionality

- Create testSimpleRelationshipPopulation for isolated debugging
- Test basic one-to-many user->posts relationship
- Add debug output to trace relationship population flow
---
 .../e2e/Adapter/Scopes/RelationshipTests.php  | 84 +++++++++++++++++++
 1 file changed, 84 insertions(+)

diff --git a/tests/e2e/Adapter/Scopes/RelationshipTests.php b/tests/e2e/Adapter/Scopes/RelationshipTests.php
index f50534a00..51e401b45 100644
--- a/tests/e2e/Adapter/Scopes/RelationshipTests.php
+++ b/tests/e2e/Adapter/Scopes/RelationshipTests.php
@@ -254,6 +254,90 @@ public function testZoo(): void
         //$this->assertEquals('shmuel', 'fogel');
     }
 
+    public function testSimpleRelationshipPopulation(): void
+    {
+        /** @var Database $database */
+        $database = static::getDatabase();
+
+        if (!$database->getAdapter()->getSupportForRelationships()) {
+            $this->expectNotToPerformAssertions();
+            return;
+        }
+
+        // Simple test case: user -> post (one-to-many)
+        $database->createCollection('users_simple');
+        $database->createCollection('posts_simple');
+
+        $database->createAttribute('users_simple', 'name', Database::VAR_STRING, 255, true);
+        $database->createAttribute('posts_simple', 'title', Database::VAR_STRING, 255, true);
+
+        $database->createRelationship(
+            collection: 'users_simple',
+            relatedCollection: 'posts_simple',
+            type: Database::RELATION_ONE_TO_MANY,
+            twoWay: true,
+            id: 'posts',
+            twoWayKey: 'author'
+        );
+
+        // Create some data
+        $user = $database->createDocument('users_simple', new Document([
+            '$id' => 'user1',
+            '$permissions' => [Permission::read(Role::any())],
+            'name' => 'John Doe',
+        ]));
+
+        $post1 = $database->createDocument('posts_simple', new Document([
+            '$id' => 'post1',
+            '$permissions' => [Permission::read(Role::any())],
+            'title' => 'First Post',
+            'author' => 'user1',
+        ]));
+
+        $post2 = $database->createDocument('posts_simple', new Document([
+            '$id' => 'post2', 
+            '$permissions' => [Permission::read(Role::any())],
+            'title' => 'Second Post',
+            'author' => 'user1',
+        ]));
+
+        // Test: fetch user with posts populated
+        $fetchedUser = $database->getDocument('users_simple', 'user1');
+        
+        var_dump('=== SIMPLE TEST USER ===');
+        var_dump($fetchedUser);
+        
+        $posts = $fetchedUser->getAttribute('posts', []);
+        var_dump('=== USER POSTS ===');
+        var_dump($posts);
+        
+        // Basic assertions
+        $this->assertIsArray($posts, 'Posts should be an array');
+        $this->assertCount(2, $posts, 'Should have 2 posts');
+        
+        if (!empty($posts)) {
+            $this->assertInstanceOf(Document::class, $posts[0], 'First post should be a Document object');
+            $this->assertEquals('First Post', $posts[0]->getAttribute('title'), 'First post title should be populated');
+        }
+
+        // Test: fetch posts with author populated  
+        $fetchedPosts = $database->find('posts_simple');
+        
+        var_dump('=== SIMPLE TEST POSTS ===');
+        var_dump($fetchedPosts);
+        
+        $this->assertCount(2, $fetchedPosts, 'Should fetch 2 posts');
+        
+        if (!empty($fetchedPosts)) {
+            $author = $fetchedPosts[0]->getAttribute('author');
+            var_dump('=== POST AUTHOR ===');
+            var_dump($author);
+            
+            $this->assertInstanceOf(Document::class, $author, 'Author should be a Document object');
+            $this->assertEquals('John Doe', $author->getAttribute('name'), 'Author name should be populated');
+        }
+    }
+
     public function testDeleteRelatedCollection(): void
     {
         /** @var Database $database */

From 1bbc241b332763dd89620da25f77187c2811eec4 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Thu, 7 Aug 2025 12:41:00 +0000
Subject: [PATCH 30/45] Add comprehensive debugging to relationship population

- Add debug logs for map blocking behavior
- Debug skip logic for already-populated relationships
- Track relationship assignment in one-to-many
- Temporarily force-include IDs that would be blocked by map
- Monitor existing values during processing
---
 src/Database/Database.php | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 6bcc2b8a6..f3e2f4de5 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3811,9 +3811,13 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
             // Skip if relationship is already populated (array of Documents)
             $existingValue = $document->getAttribute($key);
             if (!empty($existingValue) && is_array($existingValue) && !empty($existingValue[0]) && $existingValue[0] instanceof Document) {
+                error_log("OneToMany: Skipping already populated relationship '$key' for document " . $document->getId());
                 continue;
             }
             
+            // TEMPORARY: Debug what the existing value looks like when not skipped
+            error_log("OneToMany: Processing '$key' for document " . $document->getId() . ", existingValue type: " . gettype($existingValue) . ", value: " . json_encode($existingValue));
+            
             $parentId = $document->getId();
             
             // Check if this parent->children relationship has already been processed
@@ -3822,6 +3826,12 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
                 $parentIds[] = $parentId;
                 $this->map[$k] = true;
             }
+            
+            // TEMPORARY: Always include to debug map issues
+            if (!in_array($parentId, $parentIds)) {
+                $parentIds[] = $parentId;
+                error_log("OneToMany: Force-added parentId $parentId (was blocked by map)");
+            }
         }
 
         if (empty($parentIds)) {
@@ -3864,7 +3874,9 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
         // Assign related documents to their parent documents
         foreach ($documents as $document) {
             $parentId = $document->getId();
-            $document->setAttribute($key, $relatedByParentId[$parentId] ?? []);
+            $relatedDocs = $relatedByParentId[$parentId] ?? [];
+            error_log("OneToMany: Assigning " . count($relatedDocs) . " related docs to '$key' for document $parentId");
+            $document->setAttribute($key, $relatedDocs);
         }
     }
 
@@ -3927,6 +3939,12 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
                 $childIds[] = $childId;
                 $this->map[$k] = true;
             }
+            
+            // TEMPORARY: Always include to debug map issues
+            if (!in_array($childId, $childIds)) {
+                $childIds[] = $childId;
+                error_log("ManyToOne: Force-added childId $childId (was blocked by map)");
+            }
         }
 
         if (empty($childIds)) {
@@ -4012,6 +4030,12 @@ private function populateManyToManyRelationshipsBatch(array $documents, Document
                 $documentIds[] = $documentId;
                 $this->map[$k] = true;
             }
+            
+            // TEMPORARY: Always include to debug map issues
+            if (!in_array($documentId, $documentIds)) {
+                $documentIds[] = $documentId;
+                error_log("ManyToMany: Force-added documentId $documentId (was blocked by map)");
+            }
         }
 
         if (empty($documentIds)) {

From 7f65e7484e452c7cdec741f4aba44132de58813f Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Thu, 7 Aug 2025 12:57:29 +0000
Subject: [PATCH 31/45] Add extensive debugging to relationship population flow

- Added debugging before/after each relationship processing
- Added debugging to show nested query processing
- Added debugging to show related document attributes after fetching
- This will help trace why Tiger's relationships are strings vs Documents
---
 src/Database/Database.php | 55 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 54 insertions(+), 1 deletion(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index f3e2f4de5..804fcd93d 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3609,7 +3609,19 @@ private function populateDocumentsRelationships(Document $collection, array $doc
             // Map tracking prevents duplicate fetches, reverse mapping handles result assignment
             
             error_log("Processing relationship '$key' of type '$relationType' for collection '" . $collection->getId() . "' with " . count($documents) . " documents at depth " . $this->relationshipFetchDepth);
-        error_log("Select queries for '$key': " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
+            error_log("Select queries for '$key': " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
+            
+            // Debug: Show what documents look like before processing this relationship
+            foreach ($documents as $idx => $doc) {
+                $currentValue = $doc->getAttribute($key);
+                if ($currentValue !== null) {
+                    $valueType = is_object($currentValue) ? get_class($currentValue) : gettype($currentValue);
+                    if (is_array($currentValue)) {
+                        $valueType .= '[' . count($currentValue) . ']';
+                    }
+                    error_log("Doc $idx before '$key': current value type = $valueType");
+                }
+            }
 
             switch ($relationType) {
                 case Database::RELATION_ONE_TO_ONE:
@@ -3632,6 +3644,25 @@ private function populateDocumentsRelationships(Document $collection, array $doc
                     $this->populateManyToManyRelationshipsBatch($documents, $relationship, $relatedCollection, $queries, $collection);
                     break;
             }
+            
+            // Debug: Show what documents look like after processing this relationship
+            foreach ($documents as $idx => $doc) {
+                $currentValue = $doc->getAttribute($key);
+                if ($currentValue !== null) {
+                    $valueType = is_object($currentValue) ? get_class($currentValue) : gettype($currentValue);
+                    if (is_array($currentValue)) {
+                        $valueType .= '[' . count($currentValue) . ']';
+                        if (!empty($currentValue) && is_object($currentValue[0])) {
+                            $firstItemType = get_class($currentValue[0]);
+                            error_log("Doc $idx after '$key': $valueType, first item = $firstItemType");
+                        } else {
+                            error_log("Doc $idx after '$key': $valueType");
+                        }
+                    } else {
+                        error_log("Doc $idx after '$key': $valueType");
+                    }
+                }
+            }
         }
 
         return $documents;
@@ -3787,6 +3818,8 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
         $twoWay = $relationship['options']['twoWay'];
         $twoWayKey = $relationship['options']['twoWayKey'];
         $side = $relationship['options']['side'];
+        
+        error_log("OneToMany: Starting batch for '$key', nested queries: " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
 
         if ($side === Database::RELATION_SIDE_CHILD) {
             // Child side - treat like one-to-one
@@ -3852,6 +3885,26 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
         
         $relatedDocuments = $this->find($relatedCollection->getId(), $queryParams);
 
+        error_log("OneToMany: Found " . count($relatedDocuments) . " related docs for '$key'");
+        
+        // Debug: check what the related documents look like
+        if (!empty($relatedDocuments)) {
+            foreach ($relatedDocuments as $idx => $doc) {
+                $attrs = $doc->getArrayCopy();
+                $attrSummary = [];
+                foreach ($attrs as $attrKey => $attrValue) {
+                    if (is_object($attrValue)) {
+                        $attrSummary[$attrKey] = get_class($attrValue);
+                    } elseif (is_array($attrValue)) {
+                        $attrSummary[$attrKey] = 'array[' . count($attrValue) . ']';
+                    } else {
+                        $attrSummary[$attrKey] = gettype($attrValue) . ':' . $attrValue;
+                    }
+                }
+                error_log("OneToMany: Related doc $idx ID=" . $doc->getId() . " attrs=" . json_encode($attrSummary));
+            }
+        }
+
         $this->relationshipFetchDepth--;
         \array_pop($this->relationshipFetchStack);
 

From 2eb50a87d273936b7367c6876dfd05a21ec08fe0 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Thu, 7 Aug 2025 16:44:52 +0000
Subject: [PATCH 32/45] Add focused debugging for nested relationship selection
 flow

- Add detailed logging to processRelationshipQueries to track nested selection creation
- Add debugging in populateOneToManyRelationshipsBatch around find calls
- Track specific zoo/president relationship population
- Clean up excessive debugging to focus on critical paths
---
 src/Database/Database.php | 83 ++++++++++++++-------------------------
 1 file changed, 29 insertions(+), 54 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 804fcd93d..015584fd1 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3610,18 +3610,6 @@ private function populateDocumentsRelationships(Document $collection, array $doc
             
             error_log("Processing relationship '$key' of type '$relationType' for collection '" . $collection->getId() . "' with " . count($documents) . " documents at depth " . $this->relationshipFetchDepth);
             error_log("Select queries for '$key': " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
-            
-            // Debug: Show what documents look like before processing this relationship
-            foreach ($documents as $idx => $doc) {
-                $currentValue = $doc->getAttribute($key);
-                if ($currentValue !== null) {
-                    $valueType = is_object($currentValue) ? get_class($currentValue) : gettype($currentValue);
-                    if (is_array($currentValue)) {
-                        $valueType .= '[' . count($currentValue) . ']';
-                    }
-                    error_log("Doc $idx before '$key': current value type = $valueType");
-                }
-            }
 
             switch ($relationType) {
                 case Database::RELATION_ONE_TO_ONE:
@@ -3644,25 +3632,6 @@ private function populateDocumentsRelationships(Document $collection, array $doc
                     $this->populateManyToManyRelationshipsBatch($documents, $relationship, $relatedCollection, $queries, $collection);
                     break;
             }
-            
-            // Debug: Show what documents look like after processing this relationship
-            foreach ($documents as $idx => $doc) {
-                $currentValue = $doc->getAttribute($key);
-                if ($currentValue !== null) {
-                    $valueType = is_object($currentValue) ? get_class($currentValue) : gettype($currentValue);
-                    if (is_array($currentValue)) {
-                        $valueType .= '[' . count($currentValue) . ']';
-                        if (!empty($currentValue) && is_object($currentValue[0])) {
-                            $firstItemType = get_class($currentValue[0]);
-                            error_log("Doc $idx after '$key': $valueType, first item = $firstItemType");
-                        } else {
-                            error_log("Doc $idx after '$key': $valueType");
-                        }
-                    } else {
-                        error_log("Doc $idx after '$key': $valueType");
-                    }
-                }
-            }
         }
 
         return $documents;
@@ -3848,9 +3817,6 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
                 continue;
             }
             
-            // TEMPORARY: Debug what the existing value looks like when not skipped
-            error_log("OneToMany: Processing '$key' for document " . $document->getId() . ", existingValue type: " . gettype($existingValue) . ", value: " . json_encode($existingValue));
-            
             $parentId = $document->getId();
             
             // Check if this parent->children relationship has already been processed
@@ -3859,12 +3825,6 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
                 $parentIds[] = $parentId;
                 $this->map[$k] = true;
             }
-            
-            // TEMPORARY: Always include to debug map issues
-            if (!in_array($parentId, $parentIds)) {
-                $parentIds[] = $parentId;
-                error_log("OneToMany: Force-added parentId $parentId (was blocked by map)");
-            }
         }
 
         if (empty($parentIds)) {
@@ -3883,28 +3843,34 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
         
         error_log("OneToMany: Executing find with queries: " . json_encode(array_map(fn($q) => $q->toString(), $queryParams)));
         
+        // CRITICAL DEBUG: Before the nested find call
+        error_log("OneToMany: About to call find('" . $relatedCollection->getId() . "') with relationshipFetchDepth=" . $this->relationshipFetchDepth);
+        error_log("OneToMany: Current resolveRelationships=" . ($this->resolveRelationships ? 'true' : 'false'));
+        
         $relatedDocuments = $this->find($relatedCollection->getId(), $queryParams);
-
-        error_log("OneToMany: Found " . count($relatedDocuments) . " related docs for '$key'");
         
-        // Debug: check what the related documents look like
+        // CRITICAL DEBUG: After the nested find call
+        error_log("OneToMany: find() returned " . count($relatedDocuments) . " documents");
         if (!empty($relatedDocuments)) {
             foreach ($relatedDocuments as $idx => $doc) {
-                $attrs = $doc->getArrayCopy();
-                $attrSummary = [];
-                foreach ($attrs as $attrKey => $attrValue) {
-                    if (is_object($attrValue)) {
-                        $attrSummary[$attrKey] = get_class($attrValue);
-                    } elseif (is_array($attrValue)) {
-                        $attrSummary[$attrKey] = 'array[' . count($attrValue) . ']';
-                    } else {
-                        $attrSummary[$attrKey] = gettype($attrValue) . ':' . $attrValue;
-                    }
+                $relatedId = $doc->getId();
+                // Check key relationships that should be populated
+                $zoo = $doc->getAttribute('zoo');
+                $president = $doc->getAttribute('president');
+                $zooType = is_object($zoo) ? get_class($zoo) : gettype($zoo);
+                $presType = is_object($president) ? get_class($president) : gettype($president);
+                error_log("OneToMany: Related doc $idx (ID=$relatedId): zoo=$zooType, president=$presType");
+                if (is_string($zoo)) {
+                    error_log("OneToMany: WARNING - zoo is still string: '$zoo'");
+                }
+                if (is_string($president)) {
+                    error_log("OneToMany: WARNING - president is still string: '$president'");
                 }
-                error_log("OneToMany: Related doc $idx ID=" . $doc->getId() . " attrs=" . json_encode($attrSummary));
             }
         }
 
+        error_log("OneToMany: Found " . count($relatedDocuments) . " related docs for '$key'");
+
         $this->relationshipFetchDepth--;
         \array_pop($this->relationshipFetchStack);
 
@@ -7464,6 +7430,8 @@ private function processRelationshipQueries(
         array $queries,
     ): array {
         $nestedSelections = [];
+        
+        error_log("processRelationshipQueries: Starting with " . count($queries) . " queries: " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
 
         foreach ($queries as $query) {
             if ($query->getMethod() !== Query::TYPE_SELECT) {
@@ -7485,6 +7453,7 @@ private function processRelationshipQueries(
                 ))[0] ?? null;
 
                 if (!$relationship) {
+                    error_log("processRelationshipQueries: No relationship found for key '$selectedKey' in value '$value'");
                     continue;
                 }
 
@@ -7493,6 +7462,8 @@ private function processRelationshipQueries(
 
                 $nestingPath = \implode('.', $nesting);
                 $nestedSelections[$selectedKey][] = Query::select([$nestingPath]);
+                
+                error_log("processRelationshipQueries: Added nested selection for '$selectedKey': '$nestingPath'");
 
                 $type = $relationship->getAttribute('options')['relationType'];
                 $side = $relationship->getAttribute('options')['side'];
@@ -7523,6 +7494,10 @@ private function processRelationshipQueries(
 
             $query->setValues(\array_values($values));
         }
+        
+        error_log("processRelationshipQueries: Final nestedSelections: " . json_encode(array_map(function($queries) {
+            return array_map(fn($q) => $q->toString(), $queries);
+        }, $nestedSelections)));
 
         return $nestedSelections;
     }

From 400f860e78d102c354d426e58dd550b1453b7343 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Fri, 8 Aug 2025 03:29:45 +0000
Subject: [PATCH 33/45] Fix method signatures for breadth-first batch
 processing

- Updated all populateXXXRelationshipsBatch methods to use consistent parameter signatures
- Fixed parameter order: documents, relationship, relationshipFetchDepth, relationshipFetchStack, queries
- All batch methods now return documents array for proper chaining
- Added proper collection extraction from relationship options within batch methods
- Fixed depth parameter references to use passed parameter instead of instance variable
- Cleaned up debug output for cleaner testing
- All method calls updated to use correct parameter order
---
 src/Database/Database.php | 146 +++++++++++++++++---------------------
 1 file changed, 64 insertions(+), 82 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 015584fd1..44e2faeda 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3290,7 +3290,7 @@ public function getDocument(string $collection, string $id, array $queries = [],
         $this->map = [];
 
         if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections))) {
-            $documents = $this->silent(fn () => $this->populateDocumentsRelationships($collection, [$document], $nestedSelections));
+            $documents = $this->silent(fn () => $this->populateDocumentsRelationships([$document], $collection, $this->relationshipFetchDepth, $this->relationshipFetchStack, $nestedSelections));
             $document = $documents[0];
         }
 
@@ -3567,37 +3567,30 @@ private function populateDocumentRelationships(Document $collection, Document $d
     }
 
     /**
-     * Populate relationships for an array of documents using breadth-first approach
-     * This method is optimized for performance by fetching related documents in batches
-     * instead of one by one, which eliminates the N+1 query problem.
+     * Populate relationships for an array of documents (breadth-first approach)
+     * This method processes all documents in a batch to minimize N+1 query problems
      *
-     * @param Document $collection
      * @param array<Document> $documents
-     * @param array<string, array<Query>> $selects
+     * @param Collection $collection
+     * @param int $relationshipFetchDepth
+     * @param array $relationshipFetchStack
+     * @param array $selects
      * @return array<Document>
      * @throws DatabaseException
      */
-    private function populateDocumentsRelationships(Document $collection, array $documents, array $selects = []): array
+    private function populateDocumentsRelationships(array $documents, Collection $collection, int $relationshipFetchDepth = 0, array $relationshipFetchStack = [], array $selects = []): array
     {
-        if (empty($documents)) {
+        // Debug logging temporarily removed for cleaner output
+        
+        if ($relationshipFetchDepth >= self::RELATION_MAX_DEPTH) {
             return $documents;
         }
 
-        $attributes = $collection->getAttribute('attributes', []);
-        $relationships = \array_filter($attributes, function ($attribute) {
-            return $attribute->getAttribute('type') === Database::VAR_RELATIONSHIP;
-        });
-
-        if (empty($relationships)) {
-            return $documents;
-        }
+        $relationships = $collection->getAttribute('relationships', []);
 
         foreach ($relationships as $relationship) {
             $key = $relationship['key'];
-            $relatedCollection = $this->getCollection($relationship['options']['relatedCollection']);
-            $relationType = $relationship['options']['relationType'];
-            $twoWay = $relationship['options']['twoWay'];
-            $twoWayKey = $relationship['options']['twoWayKey'];
+            $relationType = $relationship['type'];
             $side = $relationship['options']['side'];
             $queries = $selects[$key] ?? [];
 
@@ -3607,29 +3600,19 @@ private function populateDocumentsRelationships(Document $collection, array $doc
             // EXPERIMENT: Pure breadth-first processing with NO cycle detection
             // Theory: Level-by-level processing + depth control prevents infinite recursion
             // Map tracking prevents duplicate fetches, reverse mapping handles result assignment
-            
-            error_log("Processing relationship '$key' of type '$relationType' for collection '" . $collection->getId() . "' with " . count($documents) . " documents at depth " . $this->relationshipFetchDepth);
-            error_log("Select queries for '$key': " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
 
             switch ($relationType) {
                 case Database::RELATION_ONE_TO_ONE:
-                    error_log("Calling populateOneToOneRelationshipsBatch for '$key' with queries: " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
-                    $this->populateOneToOneRelationshipsBatch($documents, $relationship, $relatedCollection, $queries);
+                    $documents = $this->populateOneToOneRelationshipsBatch($documents, $relationship, $relationshipFetchDepth, $relationshipFetchStack, $queries);
                     break;
-                    
                 case Database::RELATION_ONE_TO_MANY:
-                    error_log("Calling populateOneToManyRelationshipsBatch for '$key' with queries: " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
-                    $this->populateOneToManyRelationshipsBatch($documents, $relationship, $relatedCollection, $queries, $collection);
+                    $documents = $this->populateOneToManyRelationshipsBatch($documents, $relationship, $relationshipFetchDepth, $relationshipFetchStack, $queries);
                     break;
-                    
                 case Database::RELATION_MANY_TO_ONE:
-                    error_log("Calling populateManyToOneRelationshipsBatch for '$key' with queries: " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
-                    $this->populateManyToOneRelationshipsBatch($documents, $relationship, $relatedCollection, $queries, $collection);
+                    $documents = $this->populateManyToOneRelationshipsBatch($documents, $relationship, $relationshipFetchDepth, $relationshipFetchStack, $queries);
                     break;
-                    
                 case Database::RELATION_MANY_TO_MANY:
-                    error_log("Calling populateManyToManyRelationshipsBatch for '$key' with queries: " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
-                    $this->populateManyToManyRelationshipsBatch($documents, $relationship, $relatedCollection, $queries, $collection);
+                    $documents = $this->populateManyToManyRelationshipsBatch($documents, $relationship, $relationshipFetchDepth, $relationshipFetchStack, $queries);
                     break;
             }
         }
@@ -3677,16 +3660,17 @@ private function shouldSkipRelationshipFetchBatch(Document $relationship, Docume
      * @return void
      * @throws DatabaseException
      */
-    private function populateOneToOneRelationshipsBatch(array $documents, Document $relationship, Document $relatedCollection, array $queries): void
+    private function populateOneToOneRelationshipsBatch(array $documents, Document $relationship, int $relationshipFetchDepth, array $relationshipFetchStack, array $queries): array
     {
         $key = $relationship['key'];
         $twoWay = $relationship['options']['twoWay'];
+        $relatedCollection = $this->getCollection($relationship['options']['relatedCollection']);
         
-        if ($twoWay && ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH)) {
+        if ($twoWay && ($relationshipFetchDepth === Database::RELATION_MAX_DEPTH)) {
             foreach ($documents as $document) {
                 $document->removeAttribute($key);
             }
-            return;
+            return $documents;
         }
 
         // Collect all related document IDs, checking map to avoid duplicates
@@ -3768,6 +3752,8 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
                 }
             }
         }
+        
+        return $documents;
     }
 
     /**
@@ -3775,31 +3761,30 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
      *
      * @param array<Document> $documents
      * @param Document $relationship
-     * @param Document $relatedCollection
+     * @param int $relationshipFetchDepth
+     * @param array $relationshipFetchStack
      * @param array<Query> $queries
-     * @param Document $collection
-     * @return void
+     * @return array
      * @throws DatabaseException
      */
-    private function populateOneToManyRelationshipsBatch(array $documents, Document $relationship, Document $relatedCollection, array $queries, Document $collection): void
+    private function populateOneToManyRelationshipsBatch(array $documents, Document $relationship, int $relationshipFetchDepth, array $relationshipFetchStack, array $queries): array
     {
         $key = $relationship['key'];
         $twoWay = $relationship['options']['twoWay'];
         $twoWayKey = $relationship['options']['twoWayKey'];
         $side = $relationship['options']['side'];
+        $relatedCollection = $this->getCollection($relationship['options']['relatedCollection']);
+        $collection = $this->getCollection($relationship->getAttribute('collection'));
         
-        error_log("OneToMany: Starting batch for '$key', nested queries: " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
-
         if ($side === Database::RELATION_SIDE_CHILD) {
             // Child side - treat like one-to-one
-            if (!$twoWay || $this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
+            if (!$twoWay || $relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
                 foreach ($documents as $document) {
                     $document->removeAttribute($key);
                 }
-                return;
+                return $documents;
             }
-            $this->populateOneToOneRelationshipsBatch($documents, $relationship, $relatedCollection, $queries);
-            return;
+            return $this->populateOneToOneRelationshipsBatch($documents, $relationship, $relationshipFetchDepth, $relationshipFetchStack, $queries);
         }
 
         // Parent side - fetch multiple related documents
@@ -3813,7 +3798,6 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
             // Skip if relationship is already populated (array of Documents)
             $existingValue = $document->getAttribute($key);
             if (!empty($existingValue) && is_array($existingValue) && !empty($existingValue[0]) && $existingValue[0] instanceof Document) {
-                error_log("OneToMany: Skipping already populated relationship '$key' for document " . $document->getId());
                 continue;
             }
             
@@ -3841,16 +3825,12 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
             ...$queries
         ];
         
-        error_log("OneToMany: Executing find with queries: " . json_encode(array_map(fn($q) => $q->toString(), $queryParams)));
         
         // CRITICAL DEBUG: Before the nested find call
-        error_log("OneToMany: About to call find('" . $relatedCollection->getId() . "') with relationshipFetchDepth=" . $this->relationshipFetchDepth);
-        error_log("OneToMany: Current resolveRelationships=" . ($this->resolveRelationships ? 'true' : 'false'));
         
         $relatedDocuments = $this->find($relatedCollection->getId(), $queryParams);
         
         // CRITICAL DEBUG: After the nested find call
-        error_log("OneToMany: find() returned " . count($relatedDocuments) . " documents");
         if (!empty($relatedDocuments)) {
             foreach ($relatedDocuments as $idx => $doc) {
                 $relatedId = $doc->getId();
@@ -3859,17 +3839,13 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
                 $president = $doc->getAttribute('president');
                 $zooType = is_object($zoo) ? get_class($zoo) : gettype($zoo);
                 $presType = is_object($president) ? get_class($president) : gettype($president);
-                error_log("OneToMany: Related doc $idx (ID=$relatedId): zoo=$zooType, president=$presType");
                 if (is_string($zoo)) {
-                    error_log("OneToMany: WARNING - zoo is still string: '$zoo'");
                 }
                 if (is_string($president)) {
-                    error_log("OneToMany: WARNING - president is still string: '$president'");
                 }
             }
         }
 
-        error_log("OneToMany: Found " . count($relatedDocuments) . " related docs for '$key'");
 
         $this->relationshipFetchDepth--;
         \array_pop($this->relationshipFetchStack);
@@ -3894,9 +3870,10 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
         foreach ($documents as $document) {
             $parentId = $document->getId();
             $relatedDocs = $relatedByParentId[$parentId] ?? [];
-            error_log("OneToMany: Assigning " . count($relatedDocs) . " related docs to '$key' for document $parentId");
             $document->setAttribute($key, $relatedDocs);
         }
+        
+        return $documents;
     }
 
     /**
@@ -3904,29 +3881,30 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
      *
      * @param array<Document> $documents
      * @param Document $relationship
-     * @param Document $relatedCollection
+     * @param int $relationshipFetchDepth
+     * @param array $relationshipFetchStack
      * @param array<Query> $queries
-     * @param Document $collection
-     * @return void
+     * @return array
      * @throws DatabaseException
      */
-    private function populateManyToOneRelationshipsBatch(array $documents, Document $relationship, Document $relatedCollection, array $queries, Document $collection): void
+    private function populateManyToOneRelationshipsBatch(array $documents, Document $relationship, int $relationshipFetchDepth, array $relationshipFetchStack, array $queries): array
     {
         $key = $relationship['key'];
         $twoWay = $relationship['options']['twoWay'];
         $twoWayKey = $relationship['options']['twoWayKey'];
         $side = $relationship['options']['side'];
+        $relatedCollection = $this->getCollection($relationship['options']['relatedCollection']);
+        $collection = $this->getCollection($relationship->getAttribute('collection'));
 
         if ($side === Database::RELATION_SIDE_PARENT) {
             // Parent side - treat like one-to-one
-            if ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
+            if ($relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
                 foreach ($documents as $document) {
                     $document->removeAttribute($key);
                 }
-                return;
+                return $documents;
             }
-            $this->populateOneToOneRelationshipsBatch($documents, $relationship, $relatedCollection, $queries);
-            return;
+            return $this->populateOneToOneRelationshipsBatch($documents, $relationship, $relationshipFetchDepth, $relationshipFetchStack, $queries);
         }
 
         // Child side - fetch multiple related documents
@@ -3934,11 +3912,11 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
             foreach ($documents as $document) {
                 $document->removeAttribute($key);
             }
-            return;
+            return $documents;
         }
 
-        if ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
-            return;
+        if ($relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
+            return $documents;
         }
 
         // Collect all child document IDs, checking map to avoid duplicates
@@ -4004,6 +3982,8 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
             $childId = $document->getId();
             $document->setAttribute($key, $relatedByChildId[$childId] ?? []);
         }
+        
+        return $documents;
     }
 
     /**
@@ -4011,25 +3991,27 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
      *
      * @param array<Document> $documents
      * @param Document $relationship
-     * @param Document $relatedCollection
+     * @param int $relationshipFetchDepth
+     * @param array $relationshipFetchStack
      * @param array<Query> $queries
-     * @param Document $collection
-     * @return void
+     * @return array
      * @throws DatabaseException
      */
-    private function populateManyToManyRelationshipsBatch(array $documents, Document $relationship, Document $relatedCollection, array $queries, Document $collection): void
+    private function populateManyToManyRelationshipsBatch(array $documents, Document $relationship, int $relationshipFetchDepth, array $relationshipFetchStack, array $queries): array
     {
         $key = $relationship['key'];
         $twoWay = $relationship['options']['twoWay'];
         $twoWayKey = $relationship['options']['twoWayKey'];
         $side = $relationship['options']['side'];
+        $relatedCollection = $this->getCollection($relationship['options']['relatedCollection']);
+        $collection = $this->getCollection($relationship->getAttribute('collection'));
 
         if (!$twoWay && $side === Database::RELATION_SIDE_CHILD) {
-            return;
+            return $documents;
         }
 
-        if ($twoWay && ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH)) {
-            return;
+        if ($twoWay && ($relationshipFetchDepth === Database::RELATION_MAX_DEPTH)) {
+            return $documents;
         }
 
         // Collect all document IDs, checking map to avoid duplicates
@@ -4124,6 +4106,8 @@ private function populateManyToManyRelationshipsBatch(array $documents, Document
             $documentId = $document->getId();
             $document->setAttribute($key, $related[$documentId] ?? []);
         }
+        
+        return $documents;
     }
 
     /**
@@ -4220,7 +4204,7 @@ public function createDocument(string $collection, Document $document): Document
         });
 
         if ($this->resolveRelationships) {
-            $documents = $this->silent(fn () => $this->populateDocumentsRelationships($collection, [$document]));
+            $documents = $this->silent(fn () => $this->populateDocumentsRelationships([$document], $collection, $this->relationshipFetchDepth, $this->relationshipFetchStack));
             $document = $documents[0];
         }
 
@@ -4321,7 +4305,7 @@ public function createDocuments(
 
             // Use batch relationship population for better performance
             if ($this->resolveRelationships) {
-                $batch = $this->silent(fn () => $this->populateDocumentsRelationships($collection, $batch));
+                $batch = $this->silent(fn () => $this->populateDocumentsRelationships($batch, $collection, $this->relationshipFetchDepth, $this->relationshipFetchStack));
             }
 
             foreach ($batch as $document) {
@@ -4864,7 +4848,7 @@ public function updateDocument(string $collection, string $id, Document $documen
         });
 
         if ($this->resolveRelationships) {
-            $documents = $this->silent(fn () => $this->populateDocumentsRelationships($collection, [$document]));
+            $documents = $this->silent(fn () => $this->populateDocumentsRelationships([$document], $collection, $this->relationshipFetchDepth, $this->relationshipFetchStack));
             $document = $documents[0];
         }
 
@@ -5703,7 +5687,7 @@ public function createOrUpdateDocumentsWithIncrease(
 
             // Use batch relationship population for better performance
             if ($this->resolveRelationships) {
-                $batch = $this->silent(fn () => $this->populateDocumentsRelationships($collection, $batch));
+                $batch = $this->silent(fn () => $this->populateDocumentsRelationships($batch, $collection, $this->relationshipFetchDepth, $this->relationshipFetchStack));
             }
 
             foreach ($batch as $doc) {
@@ -6707,10 +6691,8 @@ public function find(string $collection, array $queries = [], string $forPermiss
         // Use batch relationship population for better performance at all levels
         if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections))) {
             if (count($results) > 0) {
-                error_log("find(): About to populate relationships for collection '" . $collection->getId() . "' with " . count($results) . " results at depth " . $this->relationshipFetchDepth);
-                error_log("find(): nestedSelections = " . json_encode($nestedSelections));
                 // Always use batch processing for all cases (single and multiple documents, nested or top-level)
-                $results = $this->silent(fn () => $this->populateDocumentsRelationships($collection, $results, $nestedSelections));
+                $results = $this->silent(fn () => $this->populateDocumentsRelationships($results, $collection, $this->relationshipFetchDepth, $this->relationshipFetchStack, $nestedSelections));
             }
         }
 

From e0f7d2892f71685489fce237aae85c3d215ab96b Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Fri, 8 Aug 2025 03:31:04 +0000
Subject: [PATCH 34/45] Add debugging for relationship population flow and fix
 early returns

- Added comprehensive debug logging to populateDocumentsRelationships to track call flow
- Fixed early return statements in one-to-many batch method to properly return documents
- Fixed depth parameter reference to use passed parameter instead of instance variable
- All batch methods now have proper depth/stack management around find calls
---
 src/Database/Database.php | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 44e2faeda..db4dca2e9 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3580,13 +3580,21 @@ private function populateDocumentRelationships(Document $collection, Document $d
      */
     private function populateDocumentsRelationships(array $documents, Collection $collection, int $relationshipFetchDepth = 0, array $relationshipFetchStack = [], array $selects = []): array
     {
-        // Debug logging temporarily removed for cleaner output
+        // Debug: Track relationship population calls
+        error_log("=== populateDocumentsRelationships ===");
+        error_log("Collection: " . $collection->getId());
+        error_log("Documents: " . count($documents));
+        error_log("Fetch depth: " . $relationshipFetchDepth);
+        error_log("Stack depth: " . count($relationshipFetchStack));
+        error_log("Selects: " . json_encode(array_keys($selects)));
         
         if ($relationshipFetchDepth >= self::RELATION_MAX_DEPTH) {
+            error_log("Max depth reached, stopping");
             return $documents;
         }
 
         $relationships = $collection->getAttribute('relationships', []);
+        error_log("Found " . count($relationships) . " relationships to process");
 
         foreach ($relationships as $relationship) {
             $key = $relationship['key'];
@@ -3594,6 +3602,8 @@ private function populateDocumentsRelationships(array $documents, Collection $co
             $side = $relationship['options']['side'];
             $queries = $selects[$key] ?? [];
 
+            error_log("Processing relationship '$key' (type: $relationType, side: $side)");
+
             // Add collection attribute for relationship context
             $relationship->setAttribute('collection', $collection->getId());
 
@@ -3615,8 +3625,11 @@ private function populateDocumentsRelationships(array $documents, Collection $co
                     $documents = $this->populateManyToManyRelationshipsBatch($documents, $relationship, $relationshipFetchDepth, $relationshipFetchStack, $queries);
                     break;
             }
+            
+            error_log("Finished processing relationship '$key'");
         }
 
+        error_log("=== END populateDocumentsRelationships ===");
         return $documents;
     }
 
@@ -3788,8 +3801,8 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
         }
 
         // Parent side - fetch multiple related documents
-        if ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
-            return;
+        if ($relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
+            return $documents;
         }
 
         // Collect all parent document IDs, checking map to avoid duplicates
@@ -3812,7 +3825,7 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
         }
 
         if (empty($parentIds)) {
-            return;
+            return $documents;
         }
 
         // Fetch all related documents for all parents in a single query

From ff407874f8a83b612c1026d21ee1221c5e38121f Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Wed, 1 Oct 2025 23:45:25 +1300
Subject: [PATCH 35/45] Fix population

---
 src/Database/Database.php | 586 +++++++++++++++++++++-----------------
 1 file changed, 329 insertions(+), 257 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 51c8c9e2f..bf9af536c 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -358,7 +358,9 @@ class Database
 
     protected bool $checkRelationshipsExist = true;
 
-    protected int $relationshipFetchDepth = 1;
+    protected int $relationshipFetchDepth = 0;
+
+    protected bool $inBatchRelationshipPopulation = false;
 
     protected bool $filter = true;
 
@@ -3540,9 +3542,9 @@ public function getDocument(string $collection, string $id, array $queries = [],
 
         $document = $this->casting($collection, $document);
         $document = $this->decode($collection, $document, $selections);
-        $this->map = [];
 
-        if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections))) {
+        // Skip relationship population if we're in batch mode (relationships will be populated later)
+        if (!$this->inBatchRelationshipPopulation && $this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections))) {
             $documents = $this->silent(fn () => $this->populateDocumentsRelationships([$document], $collection, $this->relationshipFetchDepth, $this->relationshipFetchStack, $nestedSelections));
             $document = $documents[0];
         }
@@ -3827,72 +3829,204 @@ private function populateDocumentRelationships(Document $collection, Document $d
     }
 
     /**
-     * Populate relationships for an array of documents (breadth-first approach)
-     * This method processes all documents in a batch to minimize N+1 query problems
+     * Populate relationships for an array of documents (TRUE breadth-first approach)
+     * Completely separates fetching from relationship population for massive performance gains
      *
      * @param array<Document> $documents
-     * @param Collection $collection
+     * @param Document $collection
      * @param int $relationshipFetchDepth
      * @param array $relationshipFetchStack
      * @param array $selects
      * @return array<Document>
      * @throws DatabaseException
      */
-    private function populateDocumentsRelationships(array $documents, Collection $collection, int $relationshipFetchDepth = 0, array $relationshipFetchStack = [], array $selects = []): array
+    private function populateDocumentsRelationships(array $documents, Document $collection, int $relationshipFetchDepth = 0, array $relationshipFetchStack = [], array $selects = []): array
     {
-        // Debug: Track relationship population calls
-        error_log("=== populateDocumentsRelationships ===");
-        error_log("Collection: " . $collection->getId());
-        error_log("Documents: " . count($documents));
-        error_log("Fetch depth: " . $relationshipFetchDepth);
-        error_log("Stack depth: " . count($relationshipFetchStack));
-        error_log("Selects: " . json_encode(array_keys($selects)));
-        
-        if ($relationshipFetchDepth >= self::RELATION_MAX_DEPTH) {
-            error_log("Max depth reached, stopping");
+        // Only process at depth 0 (top level) - this is the entry point
+        if ($relationshipFetchDepth !== 0) {
             return $documents;
         }
 
-        $relationships = $collection->getAttribute('relationships', []);
-        error_log("Found " . count($relationships) . " relationships to process");
+        // Enable batch mode to prevent nested relationship population during fetches
+        $this->inBatchRelationshipPopulation = true;
 
-        foreach ($relationships as $relationship) {
-            $key = $relationship['key'];
-            $relationType = $relationship['type'];
-            $side = $relationship['options']['side'];
-            $queries = $selects[$key] ?? [];
+        try {
+            // Queue of work items: [documents, collectionDoc, depth, selects, skipKey, hasExplicitSelects]
+            $queue = [
+                [
+                    'documents' => $documents,
+                    'collection' => $collection,
+                    'depth' => 0,
+                    'selects' => $selects,
+                    'skipKey' => null, // No back-reference to skip at top level
+                    'hasExplicitSelects' => !empty($selects) // Track if we're in explicit select mode
+                ]
+            ];
 
-            error_log("Processing relationship '$key' (type: $relationType, side: $side)");
+            $currentDepth = 0;
 
-            // Add collection attribute for relationship context
-            $relationship->setAttribute('collection', $collection->getId());
+            // Process queue level by level (breadth-first)
+            while (!empty($queue) && $currentDepth < self::RELATION_MAX_DEPTH) {
+                $nextQueue = [];
 
-            // EXPERIMENT: Pure breadth-first processing with NO cycle detection
-            // Theory: Level-by-level processing + depth control prevents infinite recursion
-            // Map tracking prevents duplicate fetches, reverse mapping handles result assignment
+                // Process ALL items at the current depth
+                foreach ($queue as $item) {
+                    $docs = $item['documents'];
+                    $coll = $item['collection'];
+                    $sels = $item['selects'];
+                    $skipKey = $item['skipKey'] ?? null;
+                    $parentHasExplicitSelects = $item['hasExplicitSelects'] ?? false;
 
-            switch ($relationType) {
-                case Database::RELATION_ONE_TO_ONE:
-                    $documents = $this->populateOneToOneRelationshipsBatch($documents, $relationship, $relationshipFetchDepth, $relationshipFetchStack, $queries);
-                    break;
-                case Database::RELATION_ONE_TO_MANY:
-                    $documents = $this->populateOneToManyRelationshipsBatch($documents, $relationship, $relationshipFetchDepth, $relationshipFetchStack, $queries);
-                    break;
-                case Database::RELATION_MANY_TO_ONE:
-                    $documents = $this->populateManyToOneRelationshipsBatch($documents, $relationship, $relationshipFetchDepth, $relationshipFetchStack, $queries);
-                    break;
-                case Database::RELATION_MANY_TO_MANY:
-                    $documents = $this->populateManyToManyRelationshipsBatch($documents, $relationship, $relationshipFetchDepth, $relationshipFetchStack, $queries);
-                    break;
+                    if (empty($docs)) {
+                        continue;
+                    }
+
+                    // Get all relationship attributes for this collection
+                    $attributes = $coll->getAttribute('attributes', []);
+                    $relationships = [];
+
+                    foreach ($attributes as $attribute) {
+                        if ($attribute['type'] === Database::VAR_RELATIONSHIP) {
+                            // Skip the back-reference relationship that brought us here
+                            if ($attribute['key'] === $skipKey) {
+                                continue;
+                            }
+
+                            // Include relationship if:
+                            // 1. No explicit selects (fetch all) OR
+                            // 2. Relationship is explicitly selected
+                            if (!$parentHasExplicitSelects || array_key_exists($attribute['key'], $sels)) {
+                                $relationships[] = $attribute;
+                            }
+                        }
+                    }
+
+                    // Process each relationship type
+                    foreach ($relationships as $relationship) {
+                        $key = $relationship['key'];
+                        $relationType = $relationship['options']['relationType'];
+                        $queries = $sels[$key] ?? [];
+                        $relationship->setAttribute('collection', $coll->getId());
+
+                        // Check if we're at max depth BEFORE populating
+                        $isAtMaxDepth = ($currentDepth + 1) >= self::RELATION_MAX_DEPTH;
+
+                        // If we're at max depth, remove this relationship from source documents and skip
+                        if ($isAtMaxDepth) {
+                            foreach ($docs as $doc) {
+                                $doc->removeAttribute($key);
+                            }
+                            continue;
+                        }
+
+                        // Fetch and populate this relationship
+                        $relatedDocs = $this->populateSingleRelationshipBatch(
+                            $docs,
+                            $relationship,
+                            $coll,
+                            $queries
+                        );
+
+                        // Get two-way relationship info
+                        $twoWay = $relationship['options']['twoWay'];
+                        $twoWayKey = $relationship['options']['twoWayKey'];
+
+                        // Queue if: (1) no explicit selects (fetch all recursively), OR
+                        //           (2) explicit nested selects for this relationship (isset($sels[$key]))
+                        $hasNestedSelectsForThisRel = isset($sels[$key]);
+                        $shouldQueue = !empty($relatedDocs) && ($hasNestedSelectsForThisRel || !$parentHasExplicitSelects);
+
+                        if ($shouldQueue) {
+                            $relatedCollectionId = $relationship['options']['relatedCollection'];
+                            $relatedCollection = $this->silent(fn() => $this->getCollection($relatedCollectionId));
+
+                            if (!$relatedCollection->isEmpty()) {
+                                // Get nested selections for this relationship
+                                // $sels[$key] is an array of Query objects
+                                $relationshipQueries = $hasNestedSelectsForThisRel ? $sels[$key] : [];
+
+                                // Extract nested selections for the related collection
+                                $relatedCollectionRelationships = $relatedCollection->getAttribute('attributes', []);
+                                $relatedCollectionRelationships = array_filter(
+                                    $relatedCollectionRelationships,
+                                    fn($attr) => $attr['type'] === Database::VAR_RELATIONSHIP
+                                );
+
+                                $nextSelects = $this->processRelationshipQueries($relatedCollectionRelationships, $relationshipQueries);
+
+                                // If parent has explicit selects, child inherits that mode
+                                // (even if nextSelects is empty, we're still in explicit mode)
+                                $childHasExplicitSelects = $parentHasExplicitSelects;
+
+                                $nextQueue[] = [
+                                    'documents' => $relatedDocs,
+                                    'collection' => $relatedCollection,
+                                    'depth' => $currentDepth + 1,
+                                    'selects' => $nextSelects,
+                                    'skipKey' => $twoWay ? $twoWayKey : null, // Skip the back-reference at next depth
+                                    'hasExplicitSelects' => $childHasExplicitSelects
+                                ];
+                            }
+                        }
+
+                        // Remove back-references for two-way relationships
+                        // Back-references should ALWAYS be removed unless explicitly selected
+
+                        if ($twoWay && !empty($relatedDocs)) {
+                            // Only keep back-reference if we're queuing for next depth AND back-ref is explicitly selected
+                            $backRefExplicitlySelected = $shouldQueue && !$isAtMaxDepth && isset($sels[$key]) && isset($sels[$key][$twoWayKey]);
+
+                            if (!$backRefExplicitlySelected) {
+                                foreach ($relatedDocs as $relatedDoc) {
+                                    $relatedDoc->removeAttribute($twoWayKey);
+                                }
+                            }
+                        }
+                    }
+                }
+
+                // Move to next depth
+                $queue = $nextQueue;
+                $currentDepth++;
             }
-            
-            error_log("Finished processing relationship '$key'");
+        } finally {
+            // Always disable batch mode when done
+            $this->inBatchRelationshipPopulation = false;
         }
 
-        error_log("=== END populateDocumentsRelationships ===");
         return $documents;
     }
 
+    /**
+     * Populate a single relationship type for all documents in batch
+     * Returns all related documents that were populated
+     *
+     * @param array<Document> $documents
+     * @param Document $relationship
+     * @param Document $collection
+     * @param array<Query> $queries
+     * @return array<Document>
+     * @throws DatabaseException
+     */
+    private function populateSingleRelationshipBatch(array $documents, Document $relationship, Document $collection, array $queries): array
+    {
+        $key = $relationship['key'];
+        $relationType = $relationship['options']['relationType'];
+
+        switch ($relationType) {
+            case Database::RELATION_ONE_TO_ONE:
+                return $this->populateOneToOneRelationshipsBatch($documents, $relationship, $queries);
+            case Database::RELATION_ONE_TO_MANY:
+                return $this->populateOneToManyRelationshipsBatch($documents, $relationship, $queries);
+            case Database::RELATION_MANY_TO_ONE:
+                return $this->populateManyToOneRelationshipsBatch($documents, $relationship, $queries);
+            case Database::RELATION_MANY_TO_MANY:
+                return $this->populateManyToManyRelationshipsBatch($documents, $relationship, $queries);
+            default:
+                return [];
+        }
+    }
+
     /**
      * Check if a relationship should be skipped based on batch cycle detection
      * 
@@ -3925,31 +4059,23 @@ private function shouldSkipRelationshipFetchBatch(Document $relationship, Docume
 
     /**
      * Populate one-to-one relationships in batch
+     * Returns all related documents that were fetched
      *
      * @param array<Document> $documents
      * @param Document $relationship
-     * @param Document $relatedCollection
      * @param array<Query> $queries
-     * @return void
+     * @return array<Document>
      * @throws DatabaseException
      */
-    private function populateOneToOneRelationshipsBatch(array $documents, Document $relationship, int $relationshipFetchDepth, array $relationshipFetchStack, array $queries): array
+    private function populateOneToOneRelationshipsBatch(array $documents, Document $relationship, array $queries): array
     {
         $key = $relationship['key'];
-        $twoWay = $relationship['options']['twoWay'];
         $relatedCollection = $this->getCollection($relationship['options']['relatedCollection']);
-        
-        if ($twoWay && ($relationshipFetchDepth === Database::RELATION_MAX_DEPTH)) {
-            foreach ($documents as $document) {
-                $document->removeAttribute($key);
-            }
-            return $documents;
-        }
 
-        // Collect all related document IDs, checking map to avoid duplicates
+        // Collect all related document IDs
         $relatedIds = [];
         $documentsByRelatedId = [];
-        
+
         foreach ($documents as $document) {
             $value = $document->getAttribute($key);
             if (!\is_null($value)) {
@@ -3957,53 +4083,26 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
                 if ($value instanceof Document) {
                     continue;
                 }
-                
+
                 // For one-to-one, multiple documents can reference the same related ID
-                // We need to track ALL documents that reference each related ID
                 $relatedIds[] = $value;
                 if (!isset($documentsByRelatedId[$value])) {
                     $documentsByRelatedId[$value] = [];
                 }
                 $documentsByRelatedId[$value][] = $document;
-                
-                // Map tracks that we're processing this specific document->related relationship
-                $k = $relatedCollection->getId() . ':' . $value . '=>' . $document->getCollection() . ':' . $document->getId();
-                $this->map[$k] = true;
             }
         }
 
         if (empty($relatedIds)) {
-            error_log("OneToOne: No related IDs to fetch for key '$key'");
-            return;
+            return [];
         }
-        
-        error_log("OneToOne: Fetching " . count($relatedIds) . " related docs for key '$key': " . implode(', ', $relatedIds));
 
         // Fetch all related documents in a single query
-        $this->relationshipFetchDepth++;
-        $this->relationshipFetchStack[] = $relationship;
-
-        $queryParams = [
+        $relatedDocuments = $this->find($relatedCollection->getId(), [
             Query::equal('$id', array_unique($relatedIds)),
             Query::limit(PHP_INT_MAX),
             ...$queries
-        ];
-        
-        error_log("OneToOne: Executing find with queries: " . json_encode(array_map(fn($q) => $q->toString(), $queryParams)));
-        
-        $relatedDocuments = $this->find($relatedCollection->getId(), $queryParams);
-        
-        error_log("OneToOne: Found " . count($relatedDocuments) . " related docs for key '$key'");
-        
-        // Debug: check if the related documents have their relationships populated
-        if (!empty($relatedDocuments)) {
-            foreach ($relatedDocuments as $idx => $doc) {
-                error_log("OneToOne: Related doc $idx ID=" . $doc->getId() . " attrs=" . json_encode($doc->getArrayCopy()));
-            }
-        }
-
-        $this->relationshipFetchDepth--;
-        \array_pop($this->relationshipFetchStack);
+        ]);
 
         // Index related documents by ID for quick lookup
         $relatedById = [];
@@ -4012,116 +4111,86 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
         }
 
         // Assign related documents to their parent documents
-        foreach ($documentsByRelatedId as $relatedId => $documents) {
+        foreach ($documentsByRelatedId as $relatedId => $docs) {
             if (isset($relatedById[$relatedId])) {
                 // Set the relationship for all documents that reference this related ID
-                foreach ($documents as $document) {
+                foreach ($docs as $document) {
                     $document->setAttribute($key, $relatedById[$relatedId]);
                 }
             } else {
-                // If related document not found, set to null instead of leaving the string ID
-                foreach ($documents as $document) {
-                    $document->setAttribute($key, null);
+                // If related document not found, set to empty Document instead of leaving the string ID
+                foreach ($docs as $document) {
+                    $document->setAttribute($key, new Document());
                 }
             }
         }
-        
-        return $documents;
+
+        return $relatedDocuments;
     }
 
     /**
      * Populate one-to-many relationships in batch
+     * Returns all related documents that were fetched
      *
      * @param array<Document> $documents
      * @param Document $relationship
-     * @param int $relationshipFetchDepth
-     * @param array $relationshipFetchStack
      * @param array<Query> $queries
-     * @return array
+     * @return array<Document>
      * @throws DatabaseException
      */
-    private function populateOneToManyRelationshipsBatch(array $documents, Document $relationship, int $relationshipFetchDepth, array $relationshipFetchStack, array $queries): array
+    private function populateOneToManyRelationshipsBatch(array $documents, Document $relationship, array $queries): array
     {
         $key = $relationship['key'];
         $twoWay = $relationship['options']['twoWay'];
         $twoWayKey = $relationship['options']['twoWayKey'];
         $side = $relationship['options']['side'];
         $relatedCollection = $this->getCollection($relationship['options']['relatedCollection']);
-        $collection = $this->getCollection($relationship->getAttribute('collection'));
-        
+
         if ($side === Database::RELATION_SIDE_CHILD) {
             // Child side - treat like one-to-one
-            if (!$twoWay || $relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
+            if (!$twoWay) {
                 foreach ($documents as $document) {
                     $document->removeAttribute($key);
                 }
-                return $documents;
+                return [];
             }
-            return $this->populateOneToOneRelationshipsBatch($documents, $relationship, $relationshipFetchDepth, $relationshipFetchStack, $queries);
+            return $this->populateOneToOneRelationshipsBatch($documents, $relationship, $queries);
         }
 
         // Parent side - fetch multiple related documents
-        if ($relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
-            return $documents;
-        }
-
-        // Collect all parent document IDs, checking map to avoid duplicates
+        // Collect all parent document IDs
         $parentIds = [];
         foreach ($documents as $document) {
-            // Skip if relationship is already populated (array of Documents)
-            $existingValue = $document->getAttribute($key);
-            if (!empty($existingValue) && is_array($existingValue) && !empty($existingValue[0]) && $existingValue[0] instanceof Document) {
-                continue;
-            }
-            
             $parentId = $document->getId();
-            
-            // Check if this parent->children relationship has already been processed
-            $k = $collection->getId() . ':' . $parentId . '=>' . $relatedCollection->getId() . ':*';
-            if (!isset($this->map[$k])) {
-                $parentIds[] = $parentId;
-                $this->map[$k] = true;
-            }
+            $parentIds[] = $parentId;
         }
 
+        // Remove duplicates
+        $parentIds = array_unique($parentIds);
+
         if (empty($parentIds)) {
-            return $documents;
+            return [];
         }
 
-        // Fetch all related documents for all parents in a single query
-        $this->relationshipFetchDepth++;
-        $this->relationshipFetchStack[] = $relationship;
-
-        $queryParams = [
-            Query::equal($twoWayKey, $parentIds),
-            Query::limit(PHP_INT_MAX),
-            ...$queries
-        ];
-        
-        
-        // CRITICAL DEBUG: Before the nested find call
-        
-        $relatedDocuments = $this->find($relatedCollection->getId(), $queryParams);
-        
-        // CRITICAL DEBUG: After the nested find call
-        if (!empty($relatedDocuments)) {
-            foreach ($relatedDocuments as $idx => $doc) {
-                $relatedId = $doc->getId();
-                // Check key relationships that should be populated
-                $zoo = $doc->getAttribute('zoo');
-                $president = $doc->getAttribute('president');
-                $zooType = is_object($zoo) ? get_class($zoo) : gettype($zoo);
-                $presType = is_object($president) ? get_class($president) : gettype($president);
-                if (is_string($zoo)) {
-                }
-                if (is_string($president)) {
-                }
+        // For batch relationship population, we need to fetch documents with all fields
+        // to enable proper grouping by back-reference, then apply selects afterward
+        $selectQueries = [];
+        $otherQueries = [];
+        foreach ($queries as $query) {
+            if ($query->getMethod() === Query::TYPE_SELECT) {
+                $selectQueries[] = $query;
+            } else {
+                $otherQueries[] = $query;
             }
         }
 
-
-        $this->relationshipFetchDepth--;
-        \array_pop($this->relationshipFetchStack);
+        // Fetch all related documents for all parents in a single query
+        // Don't apply selects yet - we need the back-reference for grouping
+        $relatedDocuments = $this->find($relatedCollection->getId(), [
+            Query::equal($twoWayKey, $parentIds),
+            Query::limit(PHP_INT_MAX),
+            ...$otherQueries
+        ]);
 
         // Group related documents by parent ID
         $relatedByParentId = [];
@@ -4133,20 +4202,45 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
                 if (!isset($relatedByParentId[$parentKey])) {
                     $relatedByParentId[$parentKey] = [];
                 }
-                // Remove the back-reference to avoid cycles
-                $related->removeAttribute($twoWayKey);
+                // Note: We don't remove the back-reference here because documents may be reused across fetches
+                // Cycles are prevented by depth limiting in the breadth-first traversal
                 $relatedByParentId[$parentKey][] = $related;
             }
         }
 
+        // Apply select filters to related documents if specified
+        if (!empty($selectQueries)) {
+            // Get the fields to keep from the select queries
+            $fieldsToKeep = [];
+            foreach ($selectQueries as $selectQuery) {
+                foreach ($selectQuery->getValues() as $value) {
+                    $fieldsToKeep[] = $value;
+                }
+            }
+
+            // Always keep internal attributes
+            $internalKeys = array_map(fn($attr) => $attr['$id'], self::getInternalAttributes());
+            $fieldsToKeep = array_merge($fieldsToKeep, $internalKeys);
+
+            // Filter each related document to only include selected fields
+            foreach ($relatedDocuments as $relatedDoc) {
+                $allKeys = array_keys($relatedDoc->getArrayCopy());
+                foreach ($allKeys as $attrKey) {
+                    if (!in_array('*', $fieldsToKeep) && !in_array($attrKey, $fieldsToKeep) && !str_starts_with($attrKey, '$')) {
+                        $relatedDoc->removeAttribute($attrKey);
+                    }
+                }
+            }
+        }
+
         // Assign related documents to their parent documents
         foreach ($documents as $document) {
             $parentId = $document->getId();
             $relatedDocs = $relatedByParentId[$parentId] ?? [];
             $document->setAttribute($key, $relatedDocs);
         }
-        
-        return $documents;
+
+        return $relatedDocuments;
     }
 
     /**
@@ -4160,24 +4254,17 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
      * @return array
      * @throws DatabaseException
      */
-    private function populateManyToOneRelationshipsBatch(array $documents, Document $relationship, int $relationshipFetchDepth, array $relationshipFetchStack, array $queries): array
+    private function populateManyToOneRelationshipsBatch(array $documents, Document $relationship, array $queries): array
     {
         $key = $relationship['key'];
         $twoWay = $relationship['options']['twoWay'];
         $twoWayKey = $relationship['options']['twoWayKey'];
         $side = $relationship['options']['side'];
         $relatedCollection = $this->getCollection($relationship['options']['relatedCollection']);
-        $collection = $this->getCollection($relationship->getAttribute('collection'));
 
         if ($side === Database::RELATION_SIDE_PARENT) {
             // Parent side - treat like one-to-one
-            if ($relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
-                foreach ($documents as $document) {
-                    $document->removeAttribute($key);
-                }
-                return $documents;
-            }
-            return $this->populateOneToOneRelationshipsBatch($documents, $relationship, $relationshipFetchDepth, $relationshipFetchStack, $queries);
+            return $this->populateOneToOneRelationshipsBatch($documents, $relationship, $queries);
         }
 
         // Child side - fetch multiple related documents
@@ -4185,55 +4272,42 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
             foreach ($documents as $document) {
                 $document->removeAttribute($key);
             }
-            return $documents;
-        }
-
-        if ($relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
-            return $documents;
+            return [];
         }
 
-        // Collect all child document IDs, checking map to avoid duplicates
+        // Collect all child document IDs
         $childIds = [];
         foreach ($documents as $document) {
-            // Skip if relationship is already populated (array of Documents)
-            $existingValue = $document->getAttribute($key);
-            if (!empty($existingValue) && is_array($existingValue) && !empty($existingValue[0]) && $existingValue[0] instanceof Document) {
-                continue;
-            }
-            
             $childId = $document->getId();
-            
-            // Check if this child->parent relationship has already been processed
-            $k = $collection->getId() . ':' . $childId . '=>' . $relatedCollection->getId() . ':*';
-            if (!isset($this->map[$k])) {
-                $childIds[] = $childId;
-                $this->map[$k] = true;
-            }
-            
-            // TEMPORARY: Always include to debug map issues
-            if (!in_array($childId, $childIds)) {
-                $childIds[] = $childId;
-                error_log("ManyToOne: Force-added childId $childId (was blocked by map)");
-            }
+            $childIds[] = $childId;
         }
 
+        // Remove duplicates
+        $childIds = array_unique($childIds);
+
         if (empty($childIds)) {
-            return;
+            return [];
         }
 
-        // Fetch all related documents for all children in a single query
-        $this->relationshipFetchDepth++;
-        $this->relationshipFetchStack[] = $relationship;
+        // Separate select queries from other queries
+        $selectQueries = [];
+        $otherQueries = [];
+        foreach ($queries as $query) {
+            if ($query->getMethod() === Query::TYPE_SELECT) {
+                $selectQueries[] = $query;
+            } else {
+                $otherQueries[] = $query;
+            }
+        }
 
+        // Fetch all related documents for all children in a single query
+        // Don't apply selects yet - we need the back-reference for grouping
         $relatedDocuments = $this->find($relatedCollection->getId(), [
             Query::equal($twoWayKey, $childIds),
             Query::limit(PHP_INT_MAX),
-            ...$queries
+            ...$otherQueries
         ]);
 
-        $this->relationshipFetchDepth--;
-        \array_pop($this->relationshipFetchStack);
-
         // Group related documents by child ID
         $relatedByChildId = [];
         foreach ($relatedDocuments as $related) {
@@ -4244,19 +4318,44 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
                 if (!isset($relatedByChildId[$childKey])) {
                     $relatedByChildId[$childKey] = [];
                 }
-                // Remove the back-reference to avoid cycles
-                $related->removeAttribute($twoWayKey);
+                // Note: We don't remove the back-reference here because documents may be reused across fetches
+                // Cycles are prevented by depth limiting in the breadth-first traversal
                 $relatedByChildId[$childKey][] = $related;
             }
         }
 
+        // Apply select filters to related documents if specified
+        if (!empty($selectQueries)) {
+            // Get the fields to keep from the select queries
+            $fieldsToKeep = [];
+            foreach ($selectQueries as $selectQuery) {
+                foreach ($selectQuery->getValues() as $value) {
+                    $fieldsToKeep[] = $value;
+                }
+            }
+
+            // Always keep internal attributes
+            $internalKeys = array_map(fn($attr) => $attr['$id'], self::getInternalAttributes());
+            $fieldsToKeep = array_merge($fieldsToKeep, $internalKeys);
+
+            // Filter each related document to only include selected fields
+            foreach ($relatedDocuments as $relatedDoc) {
+                $allKeys = array_keys($relatedDoc->getArrayCopy());
+                foreach ($allKeys as $attrKey) {
+                    if (!in_array('*', $fieldsToKeep) && !in_array($attrKey, $fieldsToKeep) && !str_starts_with($attrKey, '$')) {
+                        $relatedDoc->removeAttribute($attrKey);
+                    }
+                }
+            }
+        }
+
         // Assign related documents to their child documents
         foreach ($documents as $document) {
             $childId = $document->getId();
             $document->setAttribute($key, $relatedByChildId[$childId] ?? []);
         }
-        
-        return $documents;
+
+        return $relatedDocuments;
     }
 
     /**
@@ -4270,7 +4369,7 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
      * @return array
      * @throws DatabaseException
      */
-    private function populateManyToManyRelationshipsBatch(array $documents, Document $relationship, int $relationshipFetchDepth, array $relationshipFetchStack, array $queries): array
+    private function populateManyToManyRelationshipsBatch(array $documents, Document $relationship, array $queries): array
     {
         $key = $relationship['key'];
         $twoWay = $relationship['options']['twoWay'];
@@ -4280,45 +4379,23 @@ private function populateManyToManyRelationshipsBatch(array $documents, Document
         $collection = $this->getCollection($relationship->getAttribute('collection'));
 
         if (!$twoWay && $side === Database::RELATION_SIDE_CHILD) {
-            return $documents;
-        }
-
-        if ($twoWay && ($relationshipFetchDepth === Database::RELATION_MAX_DEPTH)) {
-            return $documents;
+            return [];
         }
 
-        // Collect all document IDs, checking map to avoid duplicates
+        // Collect all document IDs
         $documentIds = [];
         foreach ($documents as $document) {
-            // Skip if relationship is already populated (array of Documents)
-            $existingValue = $document->getAttribute($key);
-            if (!empty($existingValue) && is_array($existingValue) && !empty($existingValue[0]) && $existingValue[0] instanceof Document) {
-                continue;
-            }
-            
             $documentId = $document->getId();
-            
-            // Check if this document->related relationship has already been processed
-            $k = $collection->getId() . ':' . $documentId . '=>' . $relatedCollection->getId() . ':*';
-            if (!isset($this->map[$k])) {
-                $documentIds[] = $documentId;
-                $this->map[$k] = true;
-            }
-            
-            // TEMPORARY: Always include to debug map issues
-            if (!in_array($documentId, $documentIds)) {
-                $documentIds[] = $documentId;
-                error_log("ManyToMany: Force-added documentId $documentId (was blocked by map)");
-            }
+            $documentIds[] = $documentId;
         }
 
+        // Remove duplicates
+        $documentIds = array_unique($documentIds);
+
         if (empty($documentIds)) {
-            return;
+            return [];
         }
 
-        $this->relationshipFetchDepth++;
-        $this->relationshipFetchStack[] = $relationship;
-
         $junction = $this->getJunctionCollection($collection, $relatedCollection, $side);
 
         // Fetch all junction records for all documents in a single query
@@ -4330,11 +4407,11 @@ private function populateManyToManyRelationshipsBatch(array $documents, Document
         // Collect all related IDs from junctions
         $relatedIds = [];
         $junctionsByDocumentId = [];
-        
+
         foreach ($junctions as $junctionDoc) {
             $documentId = $junctionDoc->getAttribute($twoWayKey);
             $relatedId = $junctionDoc->getAttribute($key);
-            
+
             if (!\is_null($documentId) && !\is_null($relatedId)) {
                 if (!isset($junctionsByDocumentId[$documentId])) {
                     $junctionsByDocumentId[$documentId] = [];
@@ -4346,12 +4423,14 @@ private function populateManyToManyRelationshipsBatch(array $documents, Document
 
         // Fetch all related documents in a single query
         $related = [];
+        $allRelatedDocs = [];
         if (!empty($relatedIds)) {
             $foundRelated = $this->find($relatedCollection->getId(), [
                 Query::equal('$id', array_unique($relatedIds)),
                 Query::limit(PHP_INT_MAX),
                 ...$queries
             ]);
+            $allRelatedDocs = $foundRelated;
 
             // Index related documents by ID for quick lookup
             $relatedById = [];
@@ -4371,16 +4450,13 @@ private function populateManyToManyRelationshipsBatch(array $documents, Document
             }
         }
 
-        $this->relationshipFetchDepth--;
-        \array_pop($this->relationshipFetchStack);
-
         // Assign related documents to their parent documents
         foreach ($documents as $document) {
             $documentId = $document->getId();
             $document->setAttribute($key, $related[$documentId] ?? []);
         }
-        
-        return $documents;
+
+        return $allRelatedDocs;
     }
 
     /**
@@ -4476,8 +4552,10 @@ public function createDocument(string $collection, Document $document): Document
             return $this->adapter->createDocument($collection, $document);
         });
 
-        if ($this->resolveRelationships) {
-            $documents = $this->silent(fn () => $this->populateDocumentsRelationships([$document], $collection, $this->relationshipFetchDepth, $this->relationshipFetchStack));
+        if (!$this->inBatchRelationshipPopulation && $this->resolveRelationships) {
+            // Use the write stack depth for proper MAX_DEPTH enforcement during creation
+            $fetchDepth = count($this->relationshipWriteStack);
+            $documents = $this->silent(fn () => $this->populateDocumentsRelationships([$document], $collection, $fetchDepth, $this->relationshipFetchStack));
             $document = $documents[0];
         }
 
@@ -4579,7 +4657,7 @@ public function createDocuments(
             $batch = $this->adapter->getSequences($collection->getId(), $batch);
 
             // Use batch relationship population for better performance
-            if ($this->resolveRelationships) {
+            if (!$this->inBatchRelationshipPopulation && $this->resolveRelationships) {
                 $batch = $this->silent(fn () => $this->populateDocumentsRelationships($batch, $collection, $this->relationshipFetchDepth, $this->relationshipFetchStack));
             }
 
@@ -5131,7 +5209,7 @@ public function updateDocument(string $collection, string $id, Document $documen
             return $document;
         }
 
-        if ($this->resolveRelationships) {
+        if (!$this->inBatchRelationshipPopulation && $this->resolveRelationships) {
             $documents = $this->silent(fn () => $this->populateDocumentsRelationships([$document], $collection, $this->relationshipFetchDepth, $this->relationshipFetchStack));
             $document = $documents[0];
         }
@@ -6004,11 +6082,11 @@ public function upsertDocumentsWithIncrease(
             }
 
             // Use batch relationship population for better performance
-            if ($this->resolveRelationships) {
+            if (!$this->inBatchRelationshipPopulation && $this->resolveRelationships) {
                 $batch = $this->silent(fn () => $this->populateDocumentsRelationships($batch, $collection, $this->relationshipFetchDepth, $this->relationshipFetchStack));
             }
 
-            foreach ($batch as $doc) {
+            foreach ($batch as $index => $doc) {
                 $doc = $this->decode($collection, $doc);
 
                 if ($this->getSharedTables() && $this->getTenantPerDocument()) {
@@ -7015,8 +7093,9 @@ public function find(string $collection, array $queries = [], string $forPermiss
 
         $results = $skipAuth ? Authorization::skip($getResults) : $getResults();
 
+        // Skip relationship population if we're in batch mode (relationships will be populated later)
         // Use batch relationship population for better performance at all levels
-        if ($this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections))) {
+        if (!$this->inBatchRelationshipPopulation && $this->resolveRelationships && !empty($relationships) && (empty($selects) || !empty($nestedSelections))) {
             if (count($results) > 0) {
                 // Always use batch processing for all cases (single and multiple documents, nested or top-level)
                 $results = $this->silent(fn () => $this->populateDocumentsRelationships($results, $collection, $this->relationshipFetchDepth, $this->relationshipFetchStack, $nestedSelections));
@@ -7798,8 +7877,6 @@ private function processRelationshipQueries(
         array $queries,
     ): array {
         $nestedSelections = [];
-        
-        error_log("processRelationshipQueries: Starting with " . count($queries) . " queries: " . json_encode(array_map(fn($q) => $q->toString(), $queries)));
 
         foreach ($queries as $query) {
             if ($query->getMethod() !== Query::TYPE_SELECT) {
@@ -7821,7 +7898,6 @@ private function processRelationshipQueries(
                 ))[0] ?? null;
 
                 if (!$relationship) {
-                    error_log("processRelationshipQueries: No relationship found for key '$selectedKey' in value '$value'");
                     continue;
                 }
 
@@ -7872,10 +7948,6 @@ private function processRelationshipQueries(
             }
             $query->setValues($finalValues);
         }
-        
-        error_log("processRelationshipQueries: Final nestedSelections: " . json_encode(array_map(function($queries) {
-            return array_map(fn($q) => $q->toString(), $queries);
-        }, $nestedSelections)));
 
         return $nestedSelections;
     }

From c7cfab63123f5a4784fed1ce266aead85b0b150d Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 2 Oct 2025 00:26:06 +1300
Subject: [PATCH 36/45] Clean up

---
 src/Database/Database.php                     | 357 +++---------------
 .../e2e/Adapter/Scopes/RelationshipTests.php  |  12 -
 2 files changed, 47 insertions(+), 322 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index bf9af536c..c6e789b90 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -321,11 +321,6 @@ class Database
 
     protected string $cacheName = 'default';
 
-    /**
-     * @var array<bool|string>
-     */
-    protected array $map = [];
-
     /**
      * @var array<string, array{encode: callable, decode: callable}>
      */
@@ -3569,265 +3564,6 @@ public function getDocument(string $collection, string $id, array $queries = [],
         return $document;
     }
 
-    /**
-     * @param Document $collection
-     * @param Document $document
-     * @param array<string, array<Query>> $selects
-     * @return Document
-     * @throws DatabaseException
-     */
-    private function populateDocumentRelationships(Document $collection, Document $document, array $selects = []): Document
-    {
-        $attributes = $collection->getAttribute('attributes', []);
-
-        $relationships = [];
-
-        foreach ($attributes as $attribute) {
-            if ($attribute['type'] === Database::VAR_RELATIONSHIP) {
-                if (empty($selects) || array_key_exists($attribute['key'], $selects)) {
-                    $relationships[] = $attribute;
-                }
-            }
-        }
-
-        foreach ($relationships as $relationship) {
-            $key = $relationship['key'];
-            $value = $document->getAttribute($key);
-            $relatedCollection = $this->getCollection($relationship['options']['relatedCollection']);
-            $relationType = $relationship['options']['relationType'];
-            $twoWay = $relationship['options']['twoWay'];
-            $twoWayKey = $relationship['options']['twoWayKey'];
-            $side = $relationship['options']['side'];
-
-            // Clone queries to avoid mutation affecting subsequent documents
-            $queries = array_map(fn ($query) => clone $query, $selects[$key] ?? []);
-
-            if (!empty($value)) {
-                $k = $relatedCollection->getId() . ':' . $value . '=>' . $collection->getId() . ':' . $document->getId();
-                if ($relationType === Database::RELATION_ONE_TO_MANY) {
-                    $k = $collection->getId() . ':' . $document->getId() . '=>' . $relatedCollection->getId() . ':' . $value;
-                }
-                $this->map[$k] = true;
-            }
-
-            $relationship->setAttribute('collection', $collection->getId());
-            $relationship->setAttribute('document', $document->getId());
-
-            $skipFetch = false;
-            foreach ($this->relationshipFetchStack as $fetchedRelationship) {
-                $existingKey = $fetchedRelationship['key'];
-                $existingCollection = $fetchedRelationship['collection'];
-                $existingRelatedCollection = $fetchedRelationship['options']['relatedCollection'];
-                $existingTwoWayKey = $fetchedRelationship['options']['twoWayKey'];
-                $existingSide = $fetchedRelationship['options']['side'];
-
-                // If this relationship has already been fetched for this document, skip it
-                $reflexive = $fetchedRelationship == $relationship;
-
-                // If this relationship is the same as a previously fetched relationship, but on the other side, skip it
-                $symmetric = $existingKey === $twoWayKey
-                    && $existingTwoWayKey === $key
-                    && $existingRelatedCollection === $collection->getId()
-                    && $existingCollection === $relatedCollection->getId()
-                    && $existingSide !== $side;
-
-                // If this relationship is not directly related but relates across multiple collections, skip it.
-                //
-                // These conditions ensure that a relationship is considered transitive if it has the same
-                // two-way key and related collection, but is on the opposite side of the relationship (the first and second conditions).
-                //
-                // They also ensure that a relationship is considered transitive if it has the same key and related
-                // collection as an existing relationship, but a different two-way key (the third condition),
-                // or the same two-way key as an existing relationship, but a different key (the fourth condition).
-                $transitive = (($existingKey === $twoWayKey
-                        && $existingCollection === $relatedCollection->getId()
-                        && $existingSide !== $side)
-                    || ($existingTwoWayKey === $key
-                        && $existingRelatedCollection === $collection->getId()
-                        && $existingSide !== $side)
-                    || ($existingKey === $key
-                        && $existingTwoWayKey !== $twoWayKey
-                        && $existingRelatedCollection === $relatedCollection->getId()
-                        && $existingSide !== $side)
-                    || ($existingKey !== $key
-                        && $existingTwoWayKey === $twoWayKey
-                        && $existingRelatedCollection === $relatedCollection->getId()
-                        && $existingSide !== $side));
-
-                if ($reflexive || $symmetric || $transitive) {
-                    $skipFetch = true;
-                }
-            }
-
-            switch ($relationType) {
-                case Database::RELATION_ONE_TO_ONE:
-                    if ($skipFetch || $twoWay && ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH)) {
-                        $document->removeAttribute($key);
-                        break;
-                    }
-
-                    if (\is_null($value)) {
-                        break;
-                    }
-
-                    $this->relationshipFetchDepth++;
-                    $this->relationshipFetchStack[] = $relationship;
-
-                    $related = $this->getDocument($relatedCollection->getId(), $value, $queries);
-
-                    $this->relationshipFetchDepth--;
-                    \array_pop($this->relationshipFetchStack);
-
-                    $document->setAttribute($key, $related);
-                    break;
-                case Database::RELATION_ONE_TO_MANY:
-                    if ($side === Database::RELATION_SIDE_CHILD) {
-                        if (!$twoWay || $this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH || $skipFetch) {
-                            $document->removeAttribute($key);
-                            break;
-                        }
-                        if (!\is_null($value)) {
-                            $this->relationshipFetchDepth++;
-                            $this->relationshipFetchStack[] = $relationship;
-
-                            $related = $this->getDocument($relatedCollection->getId(), $value, $queries);
-
-                            $this->relationshipFetchDepth--;
-                            \array_pop($this->relationshipFetchStack);
-
-                            $document->setAttribute($key, $related);
-                        }
-                        break;
-                    }
-
-                    if ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH || $skipFetch) {
-                        break;
-                    }
-
-                    $this->relationshipFetchDepth++;
-                    $this->relationshipFetchStack[] = $relationship;
-
-                    $relatedDocuments = $this->find($relatedCollection->getId(), [
-                        Query::equal($twoWayKey, [$document->getId()]),
-                        Query::limit(PHP_INT_MAX),
-                        ...$queries
-                    ]);
-
-                    $this->relationshipFetchDepth--;
-                    \array_pop($this->relationshipFetchStack);
-
-                    foreach ($relatedDocuments as $related) {
-                        $related->removeAttribute($twoWayKey);
-                    }
-
-                    $document->setAttribute($key, $relatedDocuments);
-                    break;
-                case Database::RELATION_MANY_TO_ONE:
-                    if ($side === Database::RELATION_SIDE_PARENT) {
-                        if ($skipFetch || $this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH) {
-                            $document->removeAttribute($key);
-                            break;
-                        }
-
-                        if (\is_null($value)) {
-                            break;
-                        }
-                        $this->relationshipFetchDepth++;
-                        $this->relationshipFetchStack[] = $relationship;
-
-                        $related = $this->getDocument($relatedCollection->getId(), $value, $queries);
-
-                        $this->relationshipFetchDepth--;
-                        \array_pop($this->relationshipFetchStack);
-
-                        $document->setAttribute($key, $related);
-                        break;
-                    }
-
-                    if (!$twoWay) {
-                        $document->removeAttribute($key);
-                        break;
-                    }
-
-                    if ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH || $skipFetch) {
-                        break;
-                    }
-
-                    $this->relationshipFetchDepth++;
-                    $this->relationshipFetchStack[] = $relationship;
-
-                    $relatedDocuments = $this->find($relatedCollection->getId(), [
-                        Query::equal($twoWayKey, [$document->getId()]),
-                        Query::limit(PHP_INT_MAX),
-                        ...$queries
-                    ]);
-
-                    $this->relationshipFetchDepth--;
-                    \array_pop($this->relationshipFetchStack);
-
-
-                    foreach ($relatedDocuments as $related) {
-                        $related->removeAttribute($twoWayKey);
-                    }
-
-                    $document->setAttribute($key, $relatedDocuments);
-                    break;
-                case Database::RELATION_MANY_TO_MANY:
-                    if (!$twoWay && $side === Database::RELATION_SIDE_CHILD) {
-                        break;
-                    }
-
-                    if ($twoWay && ($this->relationshipFetchDepth === Database::RELATION_MAX_DEPTH || $skipFetch)) {
-                        break;
-                    }
-
-                    $this->relationshipFetchDepth++;
-                    $this->relationshipFetchStack[] = $relationship;
-
-                    $junction = $this->getJunctionCollection($collection, $relatedCollection, $side);
-
-                    $junctions = $this->skipRelationships(fn () => $this->find($junction, [
-                        Query::equal($twoWayKey, [$document->getId()]),
-                        Query::limit(PHP_INT_MAX)
-                    ]));
-
-                    $relatedIds = [];
-                    foreach ($junctions as $junction) {
-                        $relatedIds[] = $junction->getAttribute($key);
-                    }
-
-                    $related = [];
-                    if (!empty($relatedIds)) {
-                        $foundRelated = $this->find($relatedCollection->getId(), [
-                            Query::equal('$id', $relatedIds),
-                            Query::limit(PHP_INT_MAX),
-                            ...$queries
-                        ]);
-
-                        // Preserve the order of related documents to match the junction order
-                        $relatedById = [];
-                        foreach ($foundRelated as $doc) {
-                            $relatedById[$doc->getId()] = $doc;
-                        }
-
-                        foreach ($relatedIds as $relatedId) {
-                            if (isset($relatedById[$relatedId])) {
-                                $related[] = $relatedById[$relatedId];
-                            }
-                        }
-                    }
-
-                    $this->relationshipFetchDepth--;
-                    \array_pop($this->relationshipFetchStack);
-
-                    $document->setAttribute($key, $related);
-                    break;
-            }
-        }
-
-        return $document;
-    }
-
     /**
      * Populate relationships for an array of documents (TRUE breadth-first approach)
      * Completely separates fetching from relationship population for massive performance gains
@@ -4209,29 +3945,7 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
         }
 
         // Apply select filters to related documents if specified
-        if (!empty($selectQueries)) {
-            // Get the fields to keep from the select queries
-            $fieldsToKeep = [];
-            foreach ($selectQueries as $selectQuery) {
-                foreach ($selectQuery->getValues() as $value) {
-                    $fieldsToKeep[] = $value;
-                }
-            }
-
-            // Always keep internal attributes
-            $internalKeys = array_map(fn($attr) => $attr['$id'], self::getInternalAttributes());
-            $fieldsToKeep = array_merge($fieldsToKeep, $internalKeys);
-
-            // Filter each related document to only include selected fields
-            foreach ($relatedDocuments as $relatedDoc) {
-                $allKeys = array_keys($relatedDoc->getArrayCopy());
-                foreach ($allKeys as $attrKey) {
-                    if (!in_array('*', $fieldsToKeep) && !in_array($attrKey, $fieldsToKeep) && !str_starts_with($attrKey, '$')) {
-                        $relatedDoc->removeAttribute($attrKey);
-                    }
-                }
-            }
-        }
+        $this->applySelectFiltersToDocuments($relatedDocuments, $selectQueries);
 
         // Assign related documents to their parent documents
         foreach ($documents as $document) {
@@ -4325,29 +4039,7 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
         }
 
         // Apply select filters to related documents if specified
-        if (!empty($selectQueries)) {
-            // Get the fields to keep from the select queries
-            $fieldsToKeep = [];
-            foreach ($selectQueries as $selectQuery) {
-                foreach ($selectQuery->getValues() as $value) {
-                    $fieldsToKeep[] = $value;
-                }
-            }
-
-            // Always keep internal attributes
-            $internalKeys = array_map(fn($attr) => $attr['$id'], self::getInternalAttributes());
-            $fieldsToKeep = array_merge($fieldsToKeep, $internalKeys);
-
-            // Filter each related document to only include selected fields
-            foreach ($relatedDocuments as $relatedDoc) {
-                $allKeys = array_keys($relatedDoc->getArrayCopy());
-                foreach ($allKeys as $attrKey) {
-                    if (!in_array('*', $fieldsToKeep) && !in_array($attrKey, $fieldsToKeep) && !str_starts_with($attrKey, '$')) {
-                        $relatedDoc->removeAttribute($attrKey);
-                    }
-                }
-            }
-        }
+        $this->applySelectFiltersToDocuments($relatedDocuments, $selectQueries);
 
         // Assign related documents to their child documents
         foreach ($documents as $document) {
@@ -4358,6 +4050,51 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
         return $relatedDocuments;
     }
 
+    /**
+     * Apply select filters to documents after fetching
+     *
+     * Filters document attributes based on select queries while preserving internal attributes.
+     * This is used in batch relationship population to apply selects after grouping.
+     *
+     * @param array<Document> $documents Documents to filter
+     * @param array<Query> $selectQueries Select query objects
+     * @return void
+     */
+    private function applySelectFiltersToDocuments(array $documents, array $selectQueries): void
+    {
+        if (empty($selectQueries)) {
+            return;
+        }
+
+        // Collect all fields to keep from select queries
+        $fieldsToKeep = [];
+        foreach ($selectQueries as $selectQuery) {
+            foreach ($selectQuery->getValues() as $value) {
+                $fieldsToKeep[] = $value;
+            }
+        }
+
+        // Always preserve internal attributes
+        $internalKeys = array_map(fn($attr) => $attr['$id'], self::getInternalAttributes());
+        $fieldsToKeep = array_merge($fieldsToKeep, $internalKeys);
+
+        // Early return if wildcard selector present
+        if (in_array('*', $fieldsToKeep)) {
+            return;
+        }
+
+        // Filter each document to only include selected fields
+        foreach ($documents as $doc) {
+            $allKeys = array_keys($doc->getArrayCopy());
+            foreach ($allKeys as $attrKey) {
+                // Keep if: explicitly selected OR is internal attribute
+                if (!in_array($attrKey, $fieldsToKeep) && !str_starts_with($attrKey, '$')) {
+                    $doc->removeAttribute($attrKey);
+                }
+            }
+        }
+    }
+
     /**
      * Populate many-to-many relationships in batch
      *
diff --git a/tests/e2e/Adapter/Scopes/RelationshipTests.php b/tests/e2e/Adapter/Scopes/RelationshipTests.php
index 29afe324c..f5e6551b7 100644
--- a/tests/e2e/Adapter/Scopes/RelationshipTests.php
+++ b/tests/e2e/Adapter/Scopes/RelationshipTests.php
@@ -450,13 +450,7 @@ public function testSimpleRelationshipPopulation(): void
 
         // Test: fetch user with posts populated
         $fetchedUser = $database->getDocument('users_simple', 'user1');
-        
-        var_dump('=== SIMPLE TEST USER ===');
-        var_dump($fetchedUser);
-        
         $posts = $fetchedUser->getAttribute('posts', []);
-        var_dump('=== USER POSTS ===');
-        var_dump($posts);
         
         // Basic assertions
         $this->assertIsArray($posts, 'Posts should be an array');
@@ -470,16 +464,10 @@ public function testSimpleRelationshipPopulation(): void
         // Test: fetch posts with author populated  
         $fetchedPosts = $database->find('posts_simple');
         
-        var_dump('=== SIMPLE TEST POSTS ===');
-        var_dump($fetchedPosts);
-        
         $this->assertCount(2, $fetchedPosts, 'Should fetch 2 posts');
         
         if (!empty($fetchedPosts)) {
             $author = $fetchedPosts[0]->getAttribute('author');
-            var_dump('=== POST AUTHOR ===');
-            var_dump($author);
-            
             $this->assertInstanceOf(Document::class, $author, 'Author should be a Document object');
             $this->assertEquals('John Doe', $author->getAttribute('name'), 'Author name should be populated');
         }

From 741ef0ca53d622bead3eb054dd6406f82d073c81 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 2 Oct 2025 00:40:09 +1300
Subject: [PATCH 37/45] Fix stan

---
 src/Database/Database.php                     | 52 ++++---------------
 .../e2e/Adapter/Scopes/RelationshipTests.php  | 12 ++---
 2 files changed, 15 insertions(+), 49 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index c6e789b90..8560847e9 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3571,8 +3571,8 @@ public function getDocument(string $collection, string $id, array $queries = [],
      * @param array<Document> $documents
      * @param Document $collection
      * @param int $relationshipFetchDepth
-     * @param array $relationshipFetchStack
-     * @param array $selects
+     * @param array<Document> $relationshipFetchStack
+     * @param array<string, array<Query>> $selects
      * @return array<Document>
      * @throws DatabaseException
      */
@@ -3611,7 +3611,7 @@ private function populateDocumentsRelationships(array $documents, Document $coll
                     $coll = $item['collection'];
                     $sels = $item['selects'];
                     $skipKey = $item['skipKey'] ?? null;
-                    $parentHasExplicitSelects = $item['hasExplicitSelects'] ?? false;
+                    $parentHasExplicitSelects = $item['hasExplicitSelects'];
 
                     if (empty($docs)) {
                         continue;
@@ -3674,7 +3674,7 @@ private function populateDocumentsRelationships(array $documents, Document $coll
 
                         if ($shouldQueue) {
                             $relatedCollectionId = $relationship['options']['relatedCollection'];
-                            $relatedCollection = $this->silent(fn() => $this->getCollection($relatedCollectionId));
+                            $relatedCollection = $this->silent(fn () => $this->getCollection($relatedCollectionId));
 
                             if (!$relatedCollection->isEmpty()) {
                                 // Get nested selections for this relationship
@@ -3685,7 +3685,7 @@ private function populateDocumentsRelationships(array $documents, Document $coll
                                 $relatedCollectionRelationships = $relatedCollection->getAttribute('attributes', []);
                                 $relatedCollectionRelationships = array_filter(
                                     $relatedCollectionRelationships,
-                                    fn($attr) => $attr['type'] === Database::VAR_RELATIONSHIP
+                                    fn ($attr) => $attr['type'] === Database::VAR_RELATIONSHIP
                                 );
 
                                 $nextSelects = $this->processRelationshipQueries($relatedCollectionRelationships, $relationshipQueries);
@@ -3710,7 +3710,7 @@ private function populateDocumentsRelationships(array $documents, Document $coll
 
                         if ($twoWay && !empty($relatedDocs)) {
                             // Only keep back-reference if we're queuing for next depth AND back-ref is explicitly selected
-                            $backRefExplicitlySelected = $shouldQueue && !$isAtMaxDepth && isset($sels[$key]) && isset($sels[$key][$twoWayKey]);
+                            $backRefExplicitlySelected = $shouldQueue && isset($sels[$key]) && isset($sels[$key][$twoWayKey]);
 
                             if (!$backRefExplicitlySelected) {
                                 foreach ($relatedDocs as $relatedDoc) {
@@ -3763,36 +3763,6 @@ private function populateSingleRelationshipBatch(array $documents, Document $rel
         }
     }
 
-    /**
-     * Check if a relationship should be skipped based on batch cycle detection
-     * 
-     * Simplified cycle detection for batch processing:
-     * - Respect max depth for any relationship type
-     * - Block exact duplicate relationships only
-     * - Less restrictive than depth-first approach to allow legitimate nested relationships
-     *
-     * @param Document $relationship
-     * @param Document $collection
-     * @return bool
-     */
-    private function shouldSkipRelationshipFetchBatch(Document $relationship, Document $collection): bool
-    {
-        $twoWay = $relationship['options']['twoWay'] ?? false;
-
-        // Respect max depth limit to prevent infinite recursion
-        if ($this->relationshipFetchDepth >= Database::RELATION_MAX_DEPTH) {
-            return true;
-        }
-
-        // For breadth-first batch processing, be more permissive
-        // Only block if we're at the very edge of max depth for two-way relationships
-        if ($twoWay && ($this->relationshipFetchDepth >= (Database::RELATION_MAX_DEPTH - 1))) {
-            return true;
-        }
-
-        return false;
-    }
-
     /**
      * Populate one-to-one relationships in batch
      * Returns all related documents that were fetched
@@ -3962,10 +3932,8 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
      *
      * @param array<Document> $documents
      * @param Document $relationship
-     * @param int $relationshipFetchDepth
-     * @param array $relationshipFetchStack
      * @param array<Query> $queries
-     * @return array
+     * @return array<Document>
      * @throws DatabaseException
      */
     private function populateManyToOneRelationshipsBatch(array $documents, Document $relationship, array $queries): array
@@ -4075,7 +4043,7 @@ private function applySelectFiltersToDocuments(array $documents, array $selectQu
         }
 
         // Always preserve internal attributes
-        $internalKeys = array_map(fn($attr) => $attr['$id'], self::getInternalAttributes());
+        $internalKeys = array_map(fn ($attr) => $attr['$id'], self::getInternalAttributes());
         $fieldsToKeep = array_merge($fieldsToKeep, $internalKeys);
 
         // Early return if wildcard selector present
@@ -4100,10 +4068,8 @@ private function applySelectFiltersToDocuments(array $documents, array $selectQu
      *
      * @param array<Document> $documents
      * @param Document $relationship
-     * @param int $relationshipFetchDepth
-     * @param array $relationshipFetchStack
      * @param array<Query> $queries
-     * @return array
+     * @return array<Document>
      * @throws DatabaseException
      */
     private function populateManyToManyRelationshipsBatch(array $documents, Document $relationship, array $queries): array
diff --git a/tests/e2e/Adapter/Scopes/RelationshipTests.php b/tests/e2e/Adapter/Scopes/RelationshipTests.php
index f5e6551b7..4d5c4b0a0 100644
--- a/tests/e2e/Adapter/Scopes/RelationshipTests.php
+++ b/tests/e2e/Adapter/Scopes/RelationshipTests.php
@@ -442,7 +442,7 @@ public function testSimpleRelationshipPopulation(): void
         ]));
 
         $post2 = $database->createDocument('posts_simple', new Document([
-            '$id' => 'post2', 
+            '$id' => 'post2',
             '$permissions' => [Permission::read(Role::any())],
             'title' => 'Second Post',
             'author' => 'user1',
@@ -451,21 +451,21 @@ public function testSimpleRelationshipPopulation(): void
         // Test: fetch user with posts populated
         $fetchedUser = $database->getDocument('users_simple', 'user1');
         $posts = $fetchedUser->getAttribute('posts', []);
-        
+
         // Basic assertions
         $this->assertIsArray($posts, 'Posts should be an array');
         $this->assertCount(2, $posts, 'Should have 2 posts');
-        
+
         if (!empty($posts)) {
             $this->assertInstanceOf(Document::class, $posts[0], 'First post should be a Document object');
             $this->assertEquals('First Post', $posts[0]->getAttribute('title'), 'First post title should be populated');
         }
 
-        // Test: fetch posts with author populated  
+        // Test: fetch posts with author populated
         $fetchedPosts = $database->find('posts_simple');
-        
+
         $this->assertCount(2, $fetchedPosts, 'Should fetch 2 posts');
-        
+
         if (!empty($fetchedPosts)) {
             $author = $fetchedPosts[0]->getAttribute('author');
             $this->assertInstanceOf(Document::class, $author, 'Author should be a Document object');

From 1af779f8cae676165454041c5974774e33776a22 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 2 Oct 2025 00:42:15 +1300
Subject: [PATCH 38/45] Add benchmark

---
 bin/cli.php                 |   9 +
 bin/relationships           |   3 +
 bin/tasks/index.php         | 120 ++++++------
 bin/tasks/load.php          | 358 ++++++++++++++----------------------
 bin/tasks/query.php         | 132 ++++++-------
 bin/tasks/relationships.php | 326 ++++++++++++++++++++++++++++++++
 bin/view/index.php          |  51 +++--
 7 files changed, 625 insertions(+), 374 deletions(-)
 create mode 100755 bin/relationships
 create mode 100644 bin/tasks/relationships.php

diff --git a/bin/cli.php b/bin/cli.php
index d9932d0b2..65a0638c0 100644
--- a/bin/cli.php
+++ b/bin/cli.php
@@ -3,6 +3,7 @@
 require_once '/usr/src/code/vendor/autoload.php';
 
 use Utopia\CLI\CLI;
+use Utopia\CLI\Console;
 
 ini_set('memory_limit', '-1');
 
@@ -11,5 +12,13 @@
 include 'tasks/load.php';
 include 'tasks/index.php';
 include 'tasks/query.php';
+include 'tasks/relationships.php';
+
+$cli
+    ->error()
+    ->inject('error')
+    ->action(function ($error) {
+        Console::error($error->getMessage());
+    });
 
 $cli->run();
diff --git a/bin/relationships b/bin/relationships
new file mode 100755
index 000000000..2e5d10f5d
--- /dev/null
+++ b/bin/relationships
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+php /usr/src/code/bin/cli.php relationships $@
diff --git a/bin/tasks/index.php b/bin/tasks/index.php
index d96284fe2..195fbd565 100644
--- a/bin/tasks/index.php
+++ b/bin/tasks/index.php
@@ -10,105 +10,103 @@
 use Utopia\CLI\CLI;
 use Utopia\CLI\Console;
 use Utopia\Database\Adapter\MariaDB;
-use Utopia\Database\Adapter\Mongo;
 use Utopia\Database\Adapter\MySQL;
+use Utopia\Database\Adapter\Postgres;
 use Utopia\Database\Database;
 use Utopia\Database\PDO;
-use Utopia\Mongo\Client;
+use Utopia\Validator\Boolean;
 use Utopia\Validator\Text;
 
 /**
  * @Example
  * docker compose exec tests bin/index --adapter=mysql --name=testing
  */
-
 $cli
     ->task('index')
     ->desc('Index mock data for testing queries')
     ->param('adapter', '', new Text(0), 'Database adapter')
     ->param('name', '', new Text(0), 'Name of created database.')
-    ->action(function ($adapter, $name) {
+    ->param('sharedTables', false, new Boolean(true), 'Whether to use shared tables', true)
+    ->action(function (string $adapter, string $name, bool $sharedTables) {
         $namespace = '_ns';
         $cache = new Cache(new NoCache());
 
-        switch ($adapter) {
-            case 'mongodb':
-                $client = new Client(
-                    $name,
-                    'mongo',
-                    27017,
-                    'root',
-                    'example',
-                    false
-                );
-
-                $database = new Database(new Mongo($client), $cache);
-                break;
-
-            case 'mariadb':
-                $dbHost = 'mariadb';
-                $dbPort = '3306';
-                $dbUser = 'root';
-                $dbPass = 'password';
-
-                $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MariaDB::getPDOAttributes());
-
-                $database = new Database(new MariaDB($pdo), $cache);
-                break;
-
-            case 'mysql':
-                $dbHost = 'mysql';
-                $dbPort = '3307';
-                $dbUser = 'root';
-                $dbPass = 'password';
-
-                $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MySQL::getPDOAttributes());
+        $dbAdapters = [
+            'mariadb' => [
+                'host' => 'mariadb',
+                'port' => 3306,
+                'user' => 'root',
+                'pass' => 'password',
+                'dsn' => static fn (string $host, int $port) => "mysql:host={$host};port={$port};charset=utf8mb4",
+                'adapter' => MariaDB::class,
+                'pdoAttr' => MariaDB::getPDOAttributes(),
+            ],
+            'mysql' => [
+                'host' => 'mysql',
+                'port' => 3307,
+                'user' => 'root',
+                'pass' => 'password',
+                'dsn' => static fn (string $host, int $port) => "mysql:host={$host};port={$port};charset=utf8mb4",
+                'adapter' => MySQL::class,
+                'pdoAttr' => MySQL::getPDOAttributes(),
+            ],
+            'postgres' => [
+                'host' => 'postgres',
+                'port' => 5432,
+                'user' => 'postgres',
+                'pass' => 'password',
+                'dsn' => static fn (string $host, int $port) => "pgsql:host={$host};port={$port}",
+                'adapter' => Postgres::class,
+                'pdoAttr' => Postgres::getPDOAttributes(),
+            ],
+        ];
+
+        if (!isset($dbAdapters[$adapter])) {
+            Console::error("Adapter '{$adapter}' not supported");
+            return;
+        }
 
-                $database = new Database(new MySQL($pdo), $cache);
-                break;
+        $cfg = $dbAdapters[$adapter];
 
-            default:
-                Console::error('Adapter not supported');
-                return;
-        }
+        $pdo = new PDO(
+            ($cfg['dsn'])($cfg['host'], $cfg['port']),
+            $cfg['user'],
+            $cfg['pass'],
+            $cfg['pdoAttr']
+        );
 
-        $database->setDatabase($name);
-        $database->setNamespace($namespace);
+        $database = (new Database(new ($cfg['adapter'])($pdo), $cache))
+            ->setDatabase($name)
+            ->setNamespace($namespace)
+            ->setSharedTables($sharedTables);
 
-        Console::info("greaterThan('created', ['2010-01-01 05:00:00']), equal('genre', ['travel'])");
+        Console::info("Creating key index 'createdGenre' on 'articles' for created > '2010-01-01 05:00:00' and genre = 'travel'");
         $start = microtime(true);
         $database->createIndex('articles', 'createdGenre', Database::INDEX_KEY, ['created', 'genre'], [], [Database::ORDER_DESC, Database::ORDER_DESC]);
         $time = microtime(true) - $start;
-        Console::success("{$time} seconds");
+        Console::success("Index 'createdGenre' created in {$time} seconds");
 
-        Console::info("equal('genre', ['fashion', 'finance', 'sports'])");
+        Console::info("Creating key index 'genre' on 'articles' for genres: fashion, finance, sports");
         $start = microtime(true);
         $database->createIndex('articles', 'genre', Database::INDEX_KEY, ['genre'], [], [Database::ORDER_ASC]);
         $time = microtime(true) - $start;
-        Console::success("{$time} seconds");
+        Console::success("Index 'genre' created in {$time} seconds");
 
-        Console::info("greaterThan('views', 100000)");
+        Console::info("Creating key index 'views' on 'articles' for views > 100000");
         $start = microtime(true);
         $database->createIndex('articles', 'views', Database::INDEX_KEY, ['views'], [], [Database::ORDER_DESC]);
         $time = microtime(true) - $start;
-        Console::success("{$time} seconds");
+        Console::success("Index 'views' created in {$time} seconds");
 
-        Console::info("search('text', 'Alice')");
+        Console::info("Creating fulltext index 'fulltextsearch' on 'articles' for search term 'Alice'");
         $start = microtime(true);
         $database->createIndex('articles', 'fulltextsearch', Database::INDEX_FULLTEXT, ['text']);
         $time = microtime(true) - $start;
-        Console::success("{$time} seconds");
+        Console::success("Index 'fulltextsearch' created in {$time} seconds");
 
-        Console::info("contains('tags', ['tag1'])");
+        Console::info("Creating key index 'tags' on 'articles' for tags containing 'tag1'");
         $start = microtime(true);
         $database->createIndex('articles', 'tags', Database::INDEX_KEY, ['tags']);
         $time = microtime(true) - $start;
-        Console::success("{$time} seconds");
-    });
-
-$cli
-    ->error()
-    ->inject('error')
-    ->action(function (Exception $error) {
-        Console::error($error->getMessage());
+        Console::success("Index 'tags' created in {$time} seconds");
     });
diff --git a/bin/tasks/load.php b/bin/tasks/load.php
index fef039388..0dbd7fa56 100644
--- a/bin/tasks/load.php
+++ b/bin/tasks/load.php
@@ -1,232 +1,144 @@
 <?php
 
-/**
- * @var CLI
- */
 global $cli;
 
-use Faker\Factory;
-use Faker\Generator;
 use Swoole\Database\PDOConfig;
 use Swoole\Database\PDOPool;
+use Swoole\Runtime;
 use Utopia\Cache\Adapter\None as NoCache;
 use Utopia\Cache\Cache;
-use Utopia\CLI\CLI;
 use Utopia\CLI\Console;
 use Utopia\Database\Adapter\MariaDB;
-use Utopia\Database\Adapter\Mongo;
 use Utopia\Database\Adapter\MySQL;
+use Utopia\Database\Adapter\Postgres;
 use Utopia\Database\Database;
+use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Database\Helpers\Permission;
 use Utopia\Database\Helpers\Role;
 use Utopia\Database\PDO;
 use Utopia\Database\Validator\Authorization;
-use Utopia\Mongo\Client;
-use Utopia\Validator\Numeric;
+use Utopia\Validator\Boolean;
+use Utopia\Validator\Integer;
 use Utopia\Validator\Text;
 
+// Global pools for faster document generation
+$namesPool = ['Alice', 'Bob', 'Carol', 'Dave', 'Eve', 'Frank', 'Grace', 'Heidi', 'Ivan', 'Judy', 'Mallory', 'Niaj', 'Olivia', 'Peggy', 'Quentin', 'Rupert', 'Sybil', 'Trent', 'Uma', 'Victor'];
+$genresPool = ['fashion','food','travel','music','lifestyle','fitness','diy','sports','finance'];
+$tagsPool   = ['short','quick','easy','medium','hard'];
+
 /**
  * @Example
- * docker compose exec tests bin/load --adapter=mariadb --limit=1000 --name=testing
+ * docker compose exec tests bin/load --adapter=mariadb --limit=1000
  */
-
 $cli
     ->task('load')
     ->desc('Load database with mock data for testing')
-    ->param('adapter', '', new Text(0), 'Database adapter', false)
-    ->param('limit', '', new Numeric(), 'Total number of records to add to database', false)
-    ->param('name', 'myapp_'.uniqid(), new Text(0), 'Name of created database.', true)
-    ->action(function ($adapter, $limit, $name) {
+    ->param('adapter', '', new Text(0), 'Database adapter')
+    ->param('limit', 0, new Integer(true), 'Total number of records to add to database')
+    ->param('name', 'myapp_' . uniqid(), new Text(0), 'Name of created database.', true)
+    ->param('sharedTables', false, new Boolean(true), 'Whether to use shared tables', true)
+    ->action(function (string $adapter, int $limit, string $name, bool $sharedTables) {
         $start = null;
         $namespace = '_ns';
         $cache = new Cache(new NoCache());
 
         Console::info("Filling {$adapter} with {$limit} records: {$name}");
 
-        Swoole\Runtime::enableCoroutine();
-
-        switch ($adapter) {
-            case 'mariadb':
-                Co\run(function () use (&$start, $limit, $name, $namespace, $cache) {
-                    // can't use PDO pool to act above the database level e.g. creating schemas
-                    $dbHost = 'mariadb';
-                    $dbPort = '3306';
-                    $dbUser = 'root';
-                    $dbPass = 'password';
-
-                    $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MariaDB::getPDOAttributes());
-
-                    $database = new Database(new MariaDB($pdo), $cache);
-                    $database->setDatabase($name);
-                    $database->setNamespace($namespace);
-
-                    // Outline collection schema
-                    createSchema($database);
-
-                    // reclaim resources
-                    $database = null;
-                    $pdo = null;
-
-                    // Init Faker
-                    $faker = Factory::create();
-
-                    $start = microtime(true);
-
-                    // create PDO pool for coroutines
-                    $pool = new PDOPool(
-                        (new PDOConfig())
-                            ->withHost('mariadb')
-                            ->withPort(3306)
-                            ->withDbName($name)
-                            ->withCharset('utf8mb4')
-                            ->withUsername('root')
-                            ->withPassword('password'),
-                        128
-                    );
-
-                    // A coroutine is assigned per 1000 documents
-                    for ($i = 0; $i < $limit / 1000; $i++) {
-                        \go(function () use ($pool, $faker, $name, $cache, $namespace) {
-                            $pdo = $pool->get();
-
-                            $database = new Database(new MariaDB($pdo), $cache);
-                            $database->setDatabase($name);
-                            $database->setNamespace($namespace);
-
-                            // Each coroutine loads 1000 documents
-                            for ($i = 0; $i < 1000; $i++) {
-                                createDocument($database, $faker);
-                            }
-
-                            // Reclaim resources
-                            $pool->put($pdo);
-                            $database = null;
-                        });
-                    }
-                });
-                break;
-
-            case 'mysql':
-                Co\run(function () use (&$start, $limit, $name, $namespace, $cache) {
-                    // can't use PDO pool to act above the database level e.g. creating schemas
-                    $dbHost = 'mysql';
-                    $dbPort = '3307';
-                    $dbUser = 'root';
-                    $dbPass = 'password';
-
-                    $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MySQL::getPDOAttributes());
-
-                    $database = new Database(new MySQL($pdo), $cache);
-                    $database->setDatabase($name);
-                    $database->setNamespace($namespace);
-
-                    // Outline collection schema
-                    createSchema($database);
-
-                    // reclaim resources
-                    $database = null;
-                    $pdo = null;
-
-                    // Init Faker
-                    $faker = Factory::create();
-
-                    $start = microtime(true);
-
-                    // create PDO pool for coroutines
-                    $pool = new PDOPool(
-                        (new PDOConfig())
-                            ->withHost('mysql')
-                            ->withPort(3307)
-                            // ->withUnixSocket('/tmp/mysql.sock')
-                            ->withDbName($name)
-                            ->withCharset('utf8mb4')
-                            ->withUsername('root')
-                            ->withPassword('password'),
-                        128
-                    );
-
-                    // A coroutine is assigned per 1000 documents
-                    for ($i = 0; $i < $limit / 1000; $i++) {
-                        \go(function () use ($pool, $faker, $name, $cache, $namespace) {
-                            $pdo = $pool->get();
-
-                            $database = new Database(new MySQL($pdo), $cache);
-                            $database->setDatabase($name);
-                            $database->setNamespace($namespace);
-
-                            // Each coroutine loads 1000 documents
-                            for ($i = 0; $i < 1000; $i++) {
-                                createDocument($database, $faker);
-                            }
-
-                            // Reclaim resources
-                            $pool->put($pdo);
-                            $database = null;
-                        });
-                    }
-                });
-                break;
-
-            case 'mongodb':
-                Co\run(function () use (&$start, $limit, $name, $namespace, $cache) {
-                    $client = new Client(
-                        $name,
-                        'mongo',
-                        27017,
-                        'root',
-                        'example',
-                        false
-                    );
-
-                    $database = new Database(new Mongo($client), $cache);
-                    $database->setDatabase($name);
-                    $database->setNamespace($namespace);
-
-                    // Outline collection schema
-                    createSchema($database);
-
-                    // Fill DB
-                    $faker = Factory::create();
-
-                    $start = microtime(true);
-
-                    for ($i = 0; $i < $limit / 1000; $i++) {
-                        go(function () use ($client, $faker, $name, $namespace, $cache) {
-                            $database = new Database(new Mongo($client), $cache);
-                            $database->setDatabase($name);
-                            $database->setNamespace($namespace);
-
-                            // Each coroutine loads 1000 documents
-                            for ($i = 0; $i < 1000; $i++) {
-                                createDocument($database, $faker);
-                            }
-
-                            $database = null;
-                        });
-                    }
-                });
-                break;
+        //Runtime::enableCoroutine();
+
+        $dbAdapters = [
+            'mariadb' => [
+                'host' => 'mariadb',
+                'port' => 3306,
+                'user' => 'root',
+                'pass' => 'password',
+                'dsn' => static fn (string $host, int $port) => "mysql:host={$host};port={$port};charset=utf8mb4",
+                'driver' => 'mysql',
+                'adapter' => MariaDB::class,
+                'attrs' => MariaDB::getPDOAttributes(),
+            ],
+            'mysql' => [
+                'host' => 'mysql',
+                'port' => 3307,
+                'user' => 'root',
+                'pass' => 'password',
+                'dsn' => static fn (string $host, int $port) => "mysql:host={$host};port={$port};charset=utf8mb4",
+                'driver' => 'mysql',
+                'adapter' => MySQL::class,
+                'attrs' => MySQL::getPDOAttributes(),
+            ],
+            'postgres' => [
+                'host' => 'postgres',
+                'port' => 5432,
+                'user' => 'root',
+                'pass' => 'password',
+                'dsn' => static fn (string $host, int $port) => "pgsql:host={$host};port={$port}",
+                'driver' => 'pgsql',
+                'adapter' => Postgres::class,
+                'attrs' => Postgres::getPDOAttributes(),
+            ],
+        ];
+
+        if (!isset($dbAdapters[$adapter])) {
+            Console::error("Adapter '{$adapter}' not supported");
+            return;
+        }
 
-            default:
-                echo 'Adapter not supported';
-                return;
+        $cfg = $dbAdapters[$adapter];
+        $dsn = ($cfg['dsn'])($cfg['host'], $cfg['port']);
+
+        //Co\run(function () use (&$start, $limit, $name, $sharedTables, $namespace, $cache, $cfg) {
+        $pdo = new PDO(
+            $dsn,
+            $cfg['user'],
+            $cfg['pass'],
+            $cfg['attrs']
+        );
+
+        createSchema(
+            (new Database(new ($cfg['adapter'])($pdo), $cache))
+                ->setDatabase($name)
+                ->setNamespace($namespace)
+                ->setSharedTables($sharedTables)
+        );
+
+        $pool = new PDOPool(
+            (new PDOConfig())
+                ->withDriver($cfg['driver'])
+                ->withHost($cfg['host'])
+                ->withPort($cfg['port'])
+                ->withDbName($name)
+                //->withCharset('utf8mb4')
+                ->withUsername($cfg['user'])
+                ->withPassword($cfg['pass']),
+            128
+        );
+
+        $start = \microtime(true);
+
+        for ($i = 0; $i < $limit / 1000; $i++) {
+            //\go(function () use ($cfg, $pool, $name, $namespace, $sharedTables, $cache) {
+            try {
+                //$pdo = $pool->get();
+
+                $database = (new Database(new ($cfg['adapter'])($pdo), $cache))
+                    ->setDatabase($name)
+                    ->setNamespace($namespace)
+                    ->setSharedTables($sharedTables);
+
+                createDocuments($database);
+                //$pool->put($pdo);
+            } catch (\Throwable $error) {
+                Console::error('Coroutine error: ' . $error->getMessage());
+            }
+            //});
         }
 
         $time = microtime(true) - $start;
         Console::success("Completed in {$time} seconds");
     });
 
-
-
-$cli
-    ->error()
-    ->inject('error')
-    ->action(function (Exception $error) {
-        Console::error($error->getMessage());
-    });
-
-
 function createSchema(Database $database): void
 {
     if ($database->exists($database->getDatabase())) {
@@ -247,35 +159,43 @@ function createSchema(Database $database): void
     $database->createAttribute('articles', 'genre', Database::VAR_STRING, 256, true);
     $database->createAttribute('articles', 'views', Database::VAR_INTEGER, 0, true);
     $database->createAttribute('articles', 'tags', Database::VAR_STRING, 0, true, array: true);
-    $database->createIndex('articles', 'text', Database::INDEX_FULLTEXT, ['text']);
 }
 
-function createDocument($database, Generator $faker): void
+function createDocuments(Database $database): void
 {
-    $database->createDocument('articles', new Document([
-        // Five random users out of 10,000 get read access
-        // Three random users out of 10,000 get mutate access
-        '$permissions' => [
-            Permission::read(Role::any()),
-            Permission::read(Role::user($faker->randomNumber(9))),
-            Permission::read(Role::user($faker->randomNumber(9))),
-            Permission::read(Role::user($faker->randomNumber(9))),
-            Permission::read(Role::user($faker->randomNumber(9))),
-            Permission::create(Role::user($faker->randomNumber(9))),
-            Permission::create(Role::user($faker->randomNumber(9))),
-            Permission::create(Role::user($faker->randomNumber(9))),
-            Permission::update(Role::user($faker->randomNumber(9))),
-            Permission::update(Role::user($faker->randomNumber(9))),
-            Permission::update(Role::user($faker->randomNumber(9))),
-            Permission::delete(Role::user($faker->randomNumber(9))),
-            Permission::delete(Role::user($faker->randomNumber(9))),
-            Permission::delete(Role::user($faker->randomNumber(9))),
-        ],
-        'author' => $faker->name(),
-        'created' => \Utopia\Database\DateTime::format($faker->dateTime()),
-        'text' => $faker->realTextBetween(1000, 4000),
-        'genre' => $faker->randomElement(['fashion', 'food', 'travel', 'music', 'lifestyle', 'fitness', 'diy', 'sports', 'finance']),
-        'views' => $faker->randomNumber(6),
-        'tags' => $faker->randomElements(['short', 'quick', 'easy', 'medium', 'hard'], $faker->numberBetween(1, 5)),
-    ]));
+    global $namesPool, $genresPool, $tagsPool;
+
+    $documents = [];
+
+    $start = \microtime(true);
+    for ($i = 0; $i < 1000; $i++) {
+        $length = \mt_rand(1000, 4000);
+        $bytes = \random_bytes(intdiv($length + 1, 2));
+        $text = \substr(\bin2hex($bytes), 0, $length);
+        $tagCount = \mt_rand(1, count($tagsPool));
+        $tagKeys = (array) \array_rand($tagsPool, $tagCount);
+        $tags = \array_map(fn ($k) => $tagsPool[$k], $tagKeys);
+
+        $documents[] = new Document([
+            '$permissions' => [
+                Permission::read(Role::any()),
+                ...array_map(fn () => Permission::read(Role::user(mt_rand(0, 999999999))), range(1, 4)),
+                ...array_map(fn () => Permission::create(Role::user(mt_rand(0, 999999999))), range(1, 3)),
+                ...array_map(fn () => Permission::update(Role::user(mt_rand(0, 999999999))), range(1, 3)),
+                ...array_map(fn () => Permission::delete(Role::user(mt_rand(0, 999999999))), range(1, 3)),
+            ],
+            'author'  => $namesPool[\array_rand($namesPool)],
+            'created' => DateTime::now(),
+            'text'    => $text,
+            'genre'   => $genresPool[\array_rand($genresPool)],
+            'views'   => \mt_rand(0, 999999),
+            'tags'    => $tags,
+        ]);
+    }
+    $time = \microtime(true) - $start;
+    Console::info("Prepared 1000 documents in {$time} seconds");
+    $start = \microtime(true);
+    $database->createDocuments('articles', $documents, 1000);
+    $time = \microtime(true) - $start;
+    Console::success("Inserted 1000 documents in {$time} seconds");
 }
diff --git a/bin/tasks/query.php b/bin/tasks/query.php
index ed84fd00c..3a8f8b613 100644
--- a/bin/tasks/query.php
+++ b/bin/tasks/query.php
@@ -1,8 +1,9 @@
 <?php
 
 /**
- * @var CLI
- */ global $cli;
+ * @var CLI $cli
+ */
+global $cli;
 
 use Faker\Factory;
 use Utopia\Cache\Adapter\None as NoCache;
@@ -10,13 +11,13 @@
 use Utopia\CLI\CLI;
 use Utopia\CLI\Console;
 use Utopia\Database\Adapter\MariaDB;
-use Utopia\Database\Adapter\Mongo;
 use Utopia\Database\Adapter\MySQL;
+use Utopia\Database\Adapter\Postgres;
 use Utopia\Database\Database;
 use Utopia\Database\Query;
 use Utopia\Database\Validator\Authorization;
-use Utopia\Mongo\Client;
-use Utopia\Validator\Numeric;
+use Utopia\Validator\Boolean;
+use Utopia\Validator\Integer;
 use Utopia\Validator\Text;
 
 /**
@@ -28,92 +29,98 @@
     ->desc('Query mock data')
     ->param('adapter', '', new Text(0), 'Database adapter')
     ->param('name', '', new Text(0), 'Name of created database.')
-    ->param('limit', 25, new Numeric(), 'Limit on queried documents', true)
-    ->action(function (string $adapter, string $name, int $limit) {
+    ->param('limit', 25, new Integer(true), 'Limit on queried documents', true)
+    ->param('sharedTables', false, new Boolean(true), 'Whether to use shared tables', true)
+    ->action(function (string $adapter, string $name, int $limit, bool $sharedTables) {
         $namespace = '_ns';
         $cache = new Cache(new NoCache());
 
-        switch ($adapter) {
-            case 'mongodb':
-                $client = new Client(
-                    $name,
-                    'mongo',
-                    27017,
-                    'root',
-                    'example',
-                    false
-                );
-
-                $database = new Database(new Mongo($client), $cache);
-                $database->setDatabase($name);
-                $database->setNamespace($namespace);
-                break;
-
-            case 'mariadb':
-                $dbHost = 'mariadb';
-                $dbPort = '3306';
-                $dbUser = 'root';
-                $dbPass = 'password';
-
-                $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MariaDB::getPDOAttributes());
-
-                $database = new Database(new MariaDB($pdo), $cache);
-                $database->setDatabase($name);
-                $database->setNamespace($namespace);
-                break;
-
-            case 'mysql':
-                $dbHost = 'mysql';
-                $dbPort = '3307';
-                $dbUser = 'root';
-                $dbPass = 'password';
-
-                $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MySQL::getPDOAttributes());
-
-                $database = new Database(new MySQL($pdo), $cache);
-                $database->setDatabase($name);
-                $database->setNamespace($namespace);
-                break;
-
-            default:
-                Console::error('Adapter not supported');
-                return;
+        // ------------------------------------------------------------------
+        // Adapter configuration
+        // ------------------------------------------------------------------
+        $dbAdapters = [
+            'mariadb' => [
+                'host' => 'mariadb',
+                'port' => 3306,
+                'user' => 'root',
+                'pass' => 'password',
+                'dsn' => static fn (string $host, int $port) => "mysql:host={$host};port={$port};charset=utf8mb4",
+                'adapter' => MariaDB::class,
+                'pdoAttr' => MariaDB::getPDOAttributes(),
+            ],
+            'mysql' => [
+                'host' => 'mysql',
+                'port' => 3307,
+                'user' => 'root',
+                'pass' => 'password',
+                'dsn' => static fn (string $host, int $port) => "mysql:host={$host};port={$port};charset=utf8mb4",
+                'adapter' => MySQL::class,
+                'pdoAttr' => MySQL::getPDOAttributes(),
+            ],
+            'postgres' => [
+                'host' => 'postgres',
+                'port' => 5432,
+                'user' => 'postgres',
+                'pass' => 'password',
+                'dsn' => static fn (string $host, int $port) => "pgsql:host={$host};port={$port}",
+                'adapter' => Postgres::class,
+                'pdoAttr' => Postgres::getPDOAttributes(),
+            ],
+        ];
+
+        if (!isset($dbAdapters[$adapter])) {
+            Console::error("Adapter '{$adapter}' not supported");
+            return;
         }
 
+        $cfg = $dbAdapters[$adapter];
+
+        $pdo = new PDO(
+            ($cfg['dsn'])($cfg['host'], $cfg['port']),
+            $cfg['user'],
+            $cfg['pass'],
+            $cfg['pdoAttr']
+        );
+
+        $database = (new Database(new ($cfg['adapter'])($pdo), $cache))
+            ->setDatabase($name)
+            ->setNamespace($namespace)
+            ->setSharedTables($sharedTables);
+
         $faker = Factory::create();
 
         $report = [];
 
         $count = setRoles($faker, 1);
-        Console::info("\n{$count} roles:");
+        Console::info("\nRunning queries with {$count} authorization roles:");
         $report[] = [
             'roles' => $count,
             'results' => runQueries($database, $limit)
         ];
 
         $count = setRoles($faker, 100);
-        Console::info("\n{$count} roles:");
+        Console::info("\nRunning queries with {$count} authorization roles:");
         $report[] = [
             'roles' => $count,
             'results' => runQueries($database, $limit)
         ];
 
         $count = setRoles($faker, 400);
-        Console::info("\n{$count} roles:");
+        Console::info("\nRunning queries with {$count} authorization roles:");
         $report[] = [
             'roles' => $count,
             'results' => runQueries($database, $limit)
         ];
 
         $count = setRoles($faker, 500);
-        Console::info("\n{$count} roles:");
+        Console::info("\nRunning queries with {$count} authorization roles:");
         $report[] = [
             'roles' => $count,
             'results' => runQueries($database, $limit)
         ];
 
         $count = setRoles($faker, 1000);
-        Console::info("\n{$count} roles:");
+        Console::info("\nRunning queries with {$count} authorization roles:");
         $report[] = [
             'roles' => $count,
             'results' => runQueries($database, $limit)
@@ -129,13 +136,6 @@
         \fclose($results);
     });
 
-$cli
-    ->error()
-    ->inject('error')
-    ->action(function (Exception $error) {
-        Console::error($error->getMessage());
-    });
-
 function setRoles($faker, $count): int
 {
     for ($i = 0; $i < $count; $i++) {
@@ -188,10 +188,10 @@ function runQuery(array $query, Database $database)
         return $q->getAttribute() . ': ' . $q->getMethod() . ' = ' . implode(',', $q->getValues());
     }, $query);
 
-    Console::log('Running query: [' . implode(', ', $info) . ']');
+    Console::info("Running query: [" . implode(', ', $info) . "]");
     $start = microtime(true);
     $database->find('articles', $query);
     $time = microtime(true) - $start;
-    Console::success("{$time} s");
+    Console::success("Query executed in {$time} seconds");
     return $time;
 }
diff --git a/bin/tasks/relationships.php b/bin/tasks/relationships.php
new file mode 100644
index 000000000..a7305dcfa
--- /dev/null
+++ b/bin/tasks/relationships.php
@@ -0,0 +1,326 @@
+<?php
+
+global $cli;
+
+use Swoole\Database\PDOConfig;
+use Swoole\Database\PDOPool;
+use Swoole\Runtime;
+use Utopia\Cache\Adapter\None as NoCache;
+use Utopia\Cache\Cache;
+use Utopia\CLI\Console;
+use Utopia\Database\Adapter\MariaDB;
+use Utopia\Database\Adapter\MySQL;
+use Utopia\Database\Adapter\Postgres;
+use Utopia\Database\Database;
+use Utopia\Database\DateTime;
+use Utopia\Database\Document;
+use Utopia\Database\Helpers\Permission;
+use Utopia\Database\Helpers\Role;
+use Utopia\Database\PDO;
+use Utopia\Database\Query;
+use Utopia\Database\Validator\Authorization;
+use Utopia\Validator\Boolean;
+use Utopia\Validator\Integer;
+use Utopia\Validator\Text;
+
+// Global pools for faster document generation
+$namesPool = ['Alice', 'Bob', 'Carol', 'Dave', 'Eve', 'Frank', 'Grace', 'Heidi', 'Ivan', 'Judy', 'Mallory', 'Niaj', 'Olivia', 'Peggy', 'Quentin', 'Rupert', 'Sybil', 'Trent', 'Uma', 'Victor'];
+$genresPool = ['fashion','food','travel','music','lifestyle','fitness','diy','sports','finance'];
+$tagsPool   = ['short','quick','easy','medium','hard'];
+
+/**
+ * @Example
+ * docker compose exec tests bin/load --adapter=mariadb --limit=1000
+ */
+$cli
+    ->task('relationships')
+    ->desc('Load database with mock relationships for testing')
+    ->param('adapter', '', new Text(0), 'Database adapter')
+    ->param('limit', 0, new Integer(true), 'Total number of records to add to database')
+    ->param('name', 'myapp_' . uniqid(), new Text(0), 'Name of created database.', true)
+    ->param('sharedTables', false, new Boolean(true), 'Whether to use shared tables', true)
+    ->action(function (string $adapter, int $limit, string $name, bool $sharedTables) {
+        $start = null;
+        $namespace = '_ns';
+        $cache = new Cache(new NoCache());
+
+        Console::info("Filling {$adapter} with {$limit} records: {$name}");
+
+        //Runtime::enableCoroutine();
+
+        $dbAdapters = [
+            'mariadb' => [
+                'host' => 'mariadb',
+                'port' => 3306,
+                'user' => 'root',
+                'pass' => 'password',
+                'dsn' => static fn (string $host, int $port) => "mysql:host={$host};port={$port};charset=utf8mb4",
+                'adapter' => MariaDB::class,
+                'attrs' => MariaDB::getPDOAttributes(),
+            ],
+            'mysql' => [
+                'host' => 'mysql',
+                'port' => 3307,
+                'user' => 'root',
+                'pass' => 'password',
+                'dsn' => static fn (string $host, int $port) => "mysql:host={$host};port={$port};charset=utf8mb4",
+                'adapter' => MySQL::class,
+                'attrs' => MySQL::getPDOAttributes(),
+            ],
+            'postgres' => [
+                'host' => 'postgres',
+                'port' => 5432,
+                'user' => 'postgres',
+                'pass' => 'password',
+                'dsn' => static fn (string $host, int $port) => "pgsql:host={$host};port={$port}",
+                'adapter' => Postgres::class,
+                'attrs' => Postgres::getPDOAttributes(),
+            ],
+        ];
+
+        if (!isset($dbAdapters[$adapter])) {
+            Console::error("Adapter '{$adapter}' not supported");
+            return;
+        }
+
+        $cfg = $dbAdapters[$adapter];
+
+        //Co\run(function () use (&$start, $limit, $name, $sharedTables, $namespace, $cache, $cfg) {
+        $pdo = new PDO(
+            ($cfg['dsn'])($cfg['host'], $cfg['port']),
+            $cfg['user'],
+            $cfg['pass'],
+            $cfg['attrs']
+        );
+
+        $database = (new Database(new ($cfg['adapter'])($pdo), $cache))
+            ->setDatabase($name)
+            ->setNamespace($namespace)
+            ->setSharedTables($sharedTables);
+
+        createRelationshipSchema($database);
+
+        $pdo = null;
+
+        $pool = new PDOPool(
+            (new PDOConfig())
+                ->withHost($cfg['host'])
+                ->withPort($cfg['port'])
+                ->withDbName($name)
+                ->withCharset('utf8mb4')
+                ->withUsername($cfg['user'])
+                ->withPassword($cfg['pass']),
+            128
+        );
+
+        $start = \microtime(true);
+
+        for ($i = 0; $i < $limit / 1000; $i++) {
+            \go(function () use ($cfg, $pool, $name, $namespace, $sharedTables, $cache) {
+                try {
+                    $pdo = $pool->get();
+
+                    $database = (new Database(new ($cfg['adapter'])($pdo), $cache))
+                        ->setDatabase($name)
+                        ->setNamespace($namespace)
+                        ->setSharedTables($sharedTables);
+
+                    createRelationshipDocuments($database);
+                    $pool->put($pdo);
+                } catch (\Throwable $error) {
+                    Console::error('Coroutine error: ' . $error->getMessage());
+                }
+            });
+        }
+
+        benchmarkSingleQueries($database);
+        benchmarkBatchQueries($database);
+        benchmarkPagination($database);
+
+        $time = microtime(true) - $start;
+        Console::success("Completed in {$time} seconds");
+    });
+
+function createRelationshipSchema(Database $database): void
+{
+    if ($database->exists($database->getDatabase())) {
+        $database->delete($database->getDatabase());
+    }
+    $database->create();
+
+    Authorization::setRole(Role::any()->toString());
+
+    $database->createCollection('authors', permissions: [
+        Permission::create(Role::any()),
+        Permission::read(Role::any()),
+    ]);
+    $database->createAttribute('authors', 'name', Database::VAR_STRING, 256, true);
+    $database->createAttribute('authors', 'created', Database::VAR_DATETIME, 0, true, filters: ['datetime']);
+    $database->createAttribute('authors', 'bio', Database::VAR_STRING, 5000, true);
+    $database->createAttribute('authors', 'avatar', Database::VAR_STRING, 256, true);
+    $database->createAttribute('authors', 'website', Database::VAR_STRING, 256, true);
+
+    $database->createCollection('articles', permissions: [
+        Permission::create(Role::any()),
+        Permission::read(Role::any()),
+    ]);
+    $database->createAttribute('articles', 'title', Database::VAR_STRING, 256, true);
+    $database->createAttribute('articles', 'text', Database::VAR_STRING, 5000, true);
+    $database->createAttribute('articles', 'genre', Database::VAR_STRING, 256, true);
+    $database->createAttribute('articles', 'views', Database::VAR_INTEGER, 0, true);
+    $database->createAttribute('articles', 'tags', Database::VAR_STRING, 0, true, array: true);
+
+    $database->createCollection('users', permissions: [
+        Permission::create(Role::any()),
+        Permission::read(Role::any()),
+    ]);
+    $database->createAttribute('users', 'username', Database::VAR_STRING, 256, true);
+    $database->createAttribute('users', 'email', Database::VAR_STRING, 256, true);
+    $database->createAttribute('users', 'password', Database::VAR_STRING, 256, true);
+
+    $database->createCollection('comments', permissions: [
+        Permission::create(Role::any()),
+        Permission::read(Role::any()),
+    ]);
+    $database->createAttribute('comments', 'content', Database::VAR_STRING, 256, true);
+    $database->createAttribute('comments', 'likes', Database::VAR_INTEGER, 8, true, signed: false);
+
+    $database->createRelationship('authors', 'articles', Database::RELATION_MANY_TO_MANY, true, onDelete: Database::RELATION_MUTATE_SET_NULL);
+    $database->createRelationship('articles', 'comments', Database::RELATION_ONE_TO_MANY, true, twoWayKey: 'article', onDelete: Database::RELATION_MUTATE_CASCADE);
+    $database->createRelationship('users', 'comments', Database::RELATION_ONE_TO_MANY, true, twoWayKey: 'user', onDelete: Database::RELATION_MUTATE_CASCADE);
+}
+
+function createRelationshipDocuments(Database $database): void
+{
+    global $namesPool, $genresPool, $tagsPool;
+
+    $documents = [];
+    $start = \microtime(true);
+
+    // Prepare pools for nested data
+    $numAuthors = 10;
+    $numUsers = 10;
+    $numArticlesPerAuthor = 10;
+    $numCommentsPerArticle = 10;
+
+    // Generate users
+    $users = [];
+    for ($u = 0; $u < $numUsers; $u++) {
+        $users[] = new Document([
+            'username' => $namesPool[\array_rand($namesPool)],
+            'email'    => \strtolower($namesPool[\array_rand($namesPool)]) . '@example.com',
+            'password' => \bin2hex(\random_bytes(8)),
+        ]);
+    }
+
+    // Generate authors with nested articles and comments
+    for ($a = 0; $a < $numAuthors; $a++) {
+        $author = new Document([
+            'name'      => $namesPool[array_rand($namesPool)],
+            'created'   => DateTime::now(),
+            'bio'       => \substr(\bin2hex(\random_bytes(32)), 0, 100),
+            'avatar'    => 'https://example.com/avatar/' . $a,
+            'website'   => 'https://example.com/user/' . $a,
+        ]);
+
+        // Nested articles
+        $authorArticles = [];
+        for ($i = 0; $i < $numArticlesPerAuthor; $i++) {
+            $article = new Document([
+                'title'   => 'Article ' . ($i + 1) . ' by ' . $author->getAttribute('name'),
+                'text'    => \substr(\bin2hex(\random_bytes(64)), 0, \mt_rand(100, 200)),
+                'genre'   => $genresPool[array_rand($genresPool)],
+                'views'   => \mt_rand(0, 1000),
+                'tags'    => \array_slice($tagsPool, 0, \mt_rand(1, \count($tagsPool))),
+            ]);
+
+            // Nested comments
+            $comments = [];
+            for ($c = 0; $c < $numCommentsPerArticle; $c++) {
+                $comment = new Document([
+                    'content' => 'Comment ' . ($c + 1),
+                    'likes'   => \mt_rand(0, 10000),
+                    'user'    => $users[\array_rand($users)],
+                ]);
+                $comments[] = $comment;
+            }
+
+            $article->setAttribute('comments', $comments);
+            $authorArticles[] = $article;
+        }
+
+        $author->setAttribute('articles', $authorArticles);
+        $documents[] = $author;
+    }
+
+    $time = microtime(true) - $start;
+    Console::info("Prepared nested documents in {$time} seconds");
+
+    // Insert authors (with nested articles, comments, and users)
+    $start = \microtime(true);
+    $database->createDocuments('authors', $documents);
+    $time = \microtime(true) - $start;
+    Console::success("Inserted nested documents in {$time} seconds");
+}
+
+/**
+ * Benchmark querying a single document from each collection.
+ */
+function benchmarkSingleQueries(Database $database): void
+{
+    $collections = ['authors', 'articles', 'users', 'comments'];
+    foreach ($collections as $collection) {
+        // Fetch one document ID to use
+        $docs = $database->find($collection, [Query::limit(1)]);
+        if (empty($docs)) {
+            Console::warning("No documents in {$collection} for single query benchmark.");
+            continue;
+        }
+        $id = $docs[0]->getId();
+
+        $start = microtime(true);
+        $database->getDocument($collection, $id);
+        $time = microtime(true) - $start;
+
+        Console::info("Single query ({$collection}) took {$time} seconds");
+    }
+}
+
+/**
+ * Benchmark querying 20 documents from each collection.
+ */
+function benchmarkBatchQueries(Database $database): void
+{
+    $collections = ['authors', 'articles', 'users', 'comments'];
+    foreach ($collections as $collection) {
+        $start = microtime(true);
+        $database->find($collection, [Query::limit(20)]);
+        $time = microtime(true) - $start;
+
+        Console::info("Batch query 20 ({$collection}) took {$time} seconds");
+    }
+}
+
+/**
+ * Benchmark pagination through entire collection in chunks of 100.
+ */
+function benchmarkPagination(Database $database): void
+{
+    $collections = ['authors', 'articles', 'users', 'comments'];
+    foreach ($collections as $collection) {
+        $offset = 0;
+        $limit = 100;
+        $start = microtime(true);
+        do {
+            $docs = $database->find($collection, [
+                Query::limit($limit),
+                Query::offset($offset),
+            ]);
+            $count = count($docs);
+            $offset += $limit;
+        } while ($count === $limit);
+        $time = microtime(true) - $start;
+
+        Console::info("Pagination ({$collection}) over all documents took {$time} seconds");
+    }
+}
diff --git a/bin/view/index.php b/bin/view/index.php
index 55a2b7a97..4afb1e677 100644
--- a/bin/view/index.php
+++ b/bin/view/index.php
@@ -3,33 +3,27 @@
 <html lang="en">
 <head>
     <meta charset="utf-8">
-
     <title>utopia-php/database</title>
     <meta name="description" content="utopia-php/database">
     <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
     <script src="https://unpkg.com/vue@next"></script>
-
 </head>
 
 <body>
-
     <div class="chartcontainer">
         <canvas id="radarchart"></canvas>
     </div>
-
     <div id="datatables" class="datatables">
-        <table v-for="n in 4" :key="n">
+        <table v-for="n in queries.length" :key="n">
             <tr>
                 <!-- v-for is base 1 index  -->
                 <th colspan="6">{{ queries[n-1] }}</th>
             </tr>
             <tr>
-                <th></th> 
-                <th>1 role</th>
-                <th>100 roles</th>
-                <th>500 roles</th>
-                <th>1000 roles</th>
-                <th>2000 roles</th>
+                <th></th>
+                <th v-for="set in results[0].data" :key="set.roles">
+                    {{ set.roles }} {{ set.roles === 1 ? 'role' : 'roles' }}
+                </th>
             </tr>
             <tr v-for="(result, index) in results" :key="result.name" v-bind:style="{ backgroundColor: colors[index].table }">
                 <!-- grab just the timestamp from result.name -->
@@ -42,20 +36,21 @@
 <script>
 
 const results = <?php
-    $directory = './results';
-$scanned_directory = array_diff(scandir($directory), array('..', '.'));
 
+$directory = './results';
+$scanned_directory = \array_diff(\scandir($directory), array('..', '.'));
 $results = [];
 foreach ($scanned_directory as $path) {
     $results[] = [
         'name' => $path,
-        'data' => json_decode(file_get_contents("{$directory}/{$path}"), true)
+        'data' => \json_decode(\file_get_contents("{$directory}/{$path}"), true)
     ];
 }
-echo json_encode($results);
-?>
 
-console.log(results)
+echo \json_encode($results);
+?>;
+
+console.info('Loaded results:', results);
 
 const colors = [
     {
@@ -97,7 +92,7 @@
 
 // Radar chart
 let datasets = [];
-for (i=0; i < results.length; i++) {
+for (let i = 0; i < results.length; i++) {
     datasets[i] = {
         label: results[i].name,
         data: results[i].data[0].results,
@@ -107,8 +102,8 @@
         pointBackgroundColor: colors[i].line,
         pointBorderColor: '#fff',
         pointHoverBackgroundColor: '#fff',
-        pointHoverBorderColor: colors[i].line
-    }
+        pointHoverBorderColor: colors[i].line,
+    };
 }
 
 const chartData = {
@@ -127,11 +122,11 @@
   options: {
     elements: {
       line: {
-        borderWidth: 2
-      }
+        borderWidth: 2,
+      },
     },
     responsive: true,
-    maintainAspectRatio: false
+    maintainAspectRatio: false,
   },
 };
 
@@ -146,12 +141,12 @@
         return {
             results: results,
             queries: chartData.labels,
-            colors: colors
-        }
-    }
-}
+            colors: colors,
+        };
+    },
+};
 
-Vue.createApp(datatables).mount('#datatables')
+Vue.createApp(datatables).mount('#datatables');
 
 </script>
 

From 5127fb9b3aa6d367783ba35fb27d0c9e07dc6a93 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 2 Oct 2025 01:05:05 +1300
Subject: [PATCH 39/45] Fix depth early exit

---
 src/Database/Database.php                     |  13 +-
 .../e2e/Adapter/Scopes/RelationshipTests.php  | 137 ++++++++++++++++++
 2 files changed, 140 insertions(+), 10 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 8560847e9..0b31fe771 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3578,11 +3578,6 @@ public function getDocument(string $collection, string $id, array $queries = [],
      */
     private function populateDocumentsRelationships(array $documents, Document $collection, int $relationshipFetchDepth = 0, array $relationshipFetchStack = [], array $selects = []): array
     {
-        // Only process at depth 0 (top level) - this is the entry point
-        if ($relationshipFetchDepth !== 0) {
-            return $documents;
-        }
-
         // Enable batch mode to prevent nested relationship population during fetches
         $this->inBatchRelationshipPopulation = true;
 
@@ -3592,14 +3587,14 @@ private function populateDocumentsRelationships(array $documents, Document $coll
                 [
                     'documents' => $documents,
                     'collection' => $collection,
-                    'depth' => 0,
+                    'depth' => $relationshipFetchDepth,
                     'selects' => $selects,
                     'skipKey' => null, // No back-reference to skip at top level
                     'hasExplicitSelects' => !empty($selects) // Track if we're in explicit select mode
                 ]
             ];
 
-            $currentDepth = 0;
+            $currentDepth = $relationshipFetchDepth;
 
             // Process queue level by level (breadth-first)
             while (!empty($queue) && $currentDepth < self::RELATION_MAX_DEPTH) {
@@ -3659,7 +3654,6 @@ private function populateDocumentsRelationships(array $documents, Document $coll
                         $relatedDocs = $this->populateSingleRelationshipBatch(
                             $docs,
                             $relationship,
-                            $coll,
                             $queries
                         );
 
@@ -3739,12 +3733,11 @@ private function populateDocumentsRelationships(array $documents, Document $coll
      *
      * @param array<Document> $documents
      * @param Document $relationship
-     * @param Document $collection
      * @param array<Query> $queries
      * @return array<Document>
      * @throws DatabaseException
      */
-    private function populateSingleRelationshipBatch(array $documents, Document $relationship, Document $collection, array $queries): array
+    private function populateSingleRelationshipBatch(array $documents, Document $relationship, array $queries): array
     {
         $key = $relationship['key'];
         $relationType = $relationship['options']['relationType'];
diff --git a/tests/e2e/Adapter/Scopes/RelationshipTests.php b/tests/e2e/Adapter/Scopes/RelationshipTests.php
index 4d5c4b0a0..1319d622e 100644
--- a/tests/e2e/Adapter/Scopes/RelationshipTests.php
+++ b/tests/e2e/Adapter/Scopes/RelationshipTests.php
@@ -2871,4 +2871,141 @@ public function testMultiDocumentNestedRelationships(): void
         $database->deleteCollection('car');
         $database->deleteCollection('customer');
     }
+
+    /**
+     * Test that nested document creation properly populates relationships at all depths.
+     * This test verifies the fix for the depth handling bug where populateDocumentsRelationships()
+     * would early return for non-zero depth, causing nested documents to not have their relationships populated.
+     */
+    public function testNestedDocumentCreationWithDepthHandling(): void
+    {
+        /** @var Database $database */
+        $database = static::getDatabase();
+
+        if (!$database->getAdapter()->getSupportForRelationships()) {
+            $this->expectNotToPerformAssertions();
+            return;
+        }
+
+        // Create three collections with chained relationships: Order -> Product -> Store
+        $database->createCollection('order_depth_test');
+        $database->createCollection('product_depth_test');
+        $database->createCollection('store_depth_test');
+
+        $database->createAttribute('order_depth_test', 'orderNumber', Database::VAR_STRING, 255, true);
+        $database->createAttribute('product_depth_test', 'productName', Database::VAR_STRING, 255, true);
+        $database->createAttribute('store_depth_test', 'storeName', Database::VAR_STRING, 255, true);
+
+        // Order -> Product (many-to-one)
+        $database->createRelationship(
+            collection: 'order_depth_test',
+            relatedCollection: 'product_depth_test',
+            type: Database::RELATION_MANY_TO_ONE,
+            twoWay: true,
+            id: 'product',
+            twoWayKey: 'orders'
+        );
+
+        // Product -> Store (many-to-one)
+        $database->createRelationship(
+            collection: 'product_depth_test',
+            relatedCollection: 'store_depth_test',
+            type: Database::RELATION_MANY_TO_ONE,
+            twoWay: true,
+            id: 'store',
+            twoWayKey: 'products'
+        );
+
+        // First, create a store that will be referenced by the nested product
+        $store = $database->createDocument('store_depth_test', new Document([
+            '$id' => 'store1',
+            '$permissions' => [
+                Permission::read(Role::any()),
+                Permission::update(Role::any()),
+            ],
+            'storeName' => 'Main Store',
+        ]));
+
+        $this->assertEquals('store1', $store->getId());
+        $this->assertEquals('Main Store', $store->getAttribute('storeName'));
+
+        // Create an order with a nested product that references the existing store
+        // The nested product is created at depth 1
+        // With the bug, the product's relationships (including 'store') would not be populated
+        // With the fix, the product's 'store' relationship should be properly populated
+        $order = $database->createDocument('order_depth_test', new Document([
+            '$id' => 'order1',
+            '$permissions' => [
+                Permission::read(Role::any()),
+                Permission::update(Role::any()),
+            ],
+            'orderNumber' => 'ORD-001',
+            'product' => [
+                '$id' => 'product1',
+                '$permissions' => [
+                    Permission::read(Role::any()),
+                    Permission::update(Role::any()),
+                ],
+                'productName' => 'Widget',
+                'store' => 'store1', // Reference to existing store
+            ],
+        ]));
+
+        // Verify the order was created
+        $this->assertEquals('order1', $order->getId());
+        $this->assertEquals('ORD-001', $order->getAttribute('orderNumber'));
+
+        // Verify the nested product relationship is populated (depth 1)
+        $this->assertArrayHasKey('product', $order);
+        $product = $order->getAttribute('product');
+        $this->assertInstanceOf(Document::class, $product);
+        $this->assertEquals('product1', $product->getId());
+        $this->assertEquals('Widget', $product->getAttribute('productName'));
+
+        // CRITICAL: Verify the product's store relationship is populated (depth 2)
+        // This is the key assertion that would fail with the bug
+        $this->assertArrayHasKey('store', $product);
+        $productStore = $product->getAttribute('store');
+        $this->assertInstanceOf(Document::class, $productStore);
+        $this->assertEquals('store1', $productStore->getId());
+        $this->assertEquals('Main Store', $productStore->getAttribute('storeName'));
+
+        // Also test with update - create another order and update it with nested product
+        $order2 = $database->createDocument('order_depth_test', new Document([
+            '$id' => 'order2',
+            '$permissions' => [
+                Permission::read(Role::any()),
+                Permission::update(Role::any()),
+            ],
+            'orderNumber' => 'ORD-002',
+        ]));
+
+        // Update order2 to add a nested product
+        $order2Updated = $database->updateDocument('order_depth_test', 'order2', $order2->setAttribute('product', [
+            '$id' => 'product2',
+            '$permissions' => [
+                Permission::read(Role::any()),
+                Permission::update(Role::any()),
+            ],
+            'productName' => 'Gadget',
+            'store' => 'store1',
+        ]));
+
+        // Verify the updated order has the nested product with populated store
+        $this->assertEquals('order2', $order2Updated->getId());
+        $product2 = $order2Updated->getAttribute('product');
+        $this->assertInstanceOf(Document::class, $product2);
+        $this->assertEquals('product2', $product2->getId());
+
+        // Verify the product's store is populated after update
+        $this->assertArrayHasKey('store', $product2);
+        $product2Store = $product2->getAttribute('store');
+        $this->assertInstanceOf(Document::class, $product2Store);
+        $this->assertEquals('store1', $product2Store->getId());
+
+        // Clean up
+        $database->deleteCollection('order_depth_test');
+        $database->deleteCollection('product_depth_test');
+        $database->deleteCollection('store_depth_test');
+    }
 }

From cfd14b7887207423db1b9f42a631411847f8aa17 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 2 Oct 2025 01:29:07 +1300
Subject: [PATCH 40/45] Simplify back reference removal

---
 src/Database/Database.php | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 0b31fe771..f9d90ddc2 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3700,16 +3700,10 @@ private function populateDocumentsRelationships(array $documents, Document $coll
                         }
 
                         // Remove back-references for two-way relationships
-                        // Back-references should ALWAYS be removed unless explicitly selected
-
+                        // Back-references are ALWAYS removed to prevent circular references
                         if ($twoWay && !empty($relatedDocs)) {
-                            // Only keep back-reference if we're queuing for next depth AND back-ref is explicitly selected
-                            $backRefExplicitlySelected = $shouldQueue && isset($sels[$key]) && isset($sels[$key][$twoWayKey]);
-
-                            if (!$backRefExplicitlySelected) {
-                                foreach ($relatedDocs as $relatedDoc) {
-                                    $relatedDoc->removeAttribute($twoWayKey);
-                                }
+                            foreach ($relatedDocs as $relatedDoc) {
+                                $relatedDoc->removeAttribute($twoWayKey);
                             }
                         }
                     }

From 272cad815718e7dfc6b07de92c749be9f82b2d45 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 2 Oct 2025 17:32:28 +1300
Subject: [PATCH 41/45] Chunk large data for relationship finds

---
 src/Database/Database.php                    | 89 +++++++++++++-------
 src/Database/Validator/Queries/Documents.php |  2 +-
 src/Database/Validator/Query/Filter.php      |  2 +-
 3 files changed, 61 insertions(+), 32 deletions(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index f9d90ddc2..8dd0b1977 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -86,6 +86,7 @@ class Database
     public const RELATION_SIDE_CHILD = 'child';
 
     public const RELATION_MAX_DEPTH = 3;
+    public const RELATION_QUERY_CHUNK_SIZE = 5000;
 
     // Orders
     public const ORDER_ASC = 'ASC';
@@ -368,7 +369,7 @@ class Database
 
     protected bool $preserveDates = false;
 
-    protected int $maxQueryValues = 100;
+    protected int $maxQueryValues = 5000;
 
     protected bool $migrating = false;
 
@@ -3790,12 +3791,18 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
             return [];
         }
 
-        // Fetch all related documents in a single query
-        $relatedDocuments = $this->find($relatedCollection->getId(), [
-            Query::equal('$id', array_unique($relatedIds)),
-            Query::limit(PHP_INT_MAX),
-            ...$queries
-        ]);
+        // Fetch all related documents, chunking to stay within query limits
+        $uniqueRelatedIds = array_unique($relatedIds);
+        $relatedDocuments = [];
+
+        // Process in chunks to avoid exceeding query value limits
+        foreach (array_chunk($uniqueRelatedIds, self::RELATION_QUERY_CHUNK_SIZE) as $chunk) {
+            $chunkDocs = $this->find($relatedCollection->getId(), [
+                Query::equal('$id', $chunk),
+                ...$queries
+            ]);
+            array_push($relatedDocuments, ...$chunkDocs);
+        }
 
         // Index related documents by ID for quick lookup
         $relatedById = [];
@@ -3877,13 +3884,18 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
             }
         }
 
-        // Fetch all related documents for all parents in a single query
+        // Fetch all related documents for all parents, chunking to stay within query limits
         // Don't apply selects yet - we need the back-reference for grouping
-        $relatedDocuments = $this->find($relatedCollection->getId(), [
-            Query::equal($twoWayKey, $parentIds),
-            Query::limit(PHP_INT_MAX),
-            ...$otherQueries
-        ]);
+        $relatedDocuments = [];
+
+        // Process in chunks to avoid exceeding query value limits
+        foreach (array_chunk($parentIds, self::RELATION_QUERY_CHUNK_SIZE) as $chunk) {
+            $chunkDocs = $this->find($relatedCollection->getId(), [
+                Query::equal($twoWayKey, $chunk),
+                ...$otherQueries
+            ]);
+            array_push($relatedDocuments, ...$chunkDocs);
+        }
 
         // Group related documents by parent ID
         $relatedByParentId = [];
@@ -3969,13 +3981,18 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
             }
         }
 
-        // Fetch all related documents for all children in a single query
+        // Fetch all related documents for all children, chunking to stay within query limits
         // Don't apply selects yet - we need the back-reference for grouping
-        $relatedDocuments = $this->find($relatedCollection->getId(), [
-            Query::equal($twoWayKey, $childIds),
-            Query::limit(PHP_INT_MAX),
-            ...$otherQueries
-        ]);
+        $relatedDocuments = [];
+
+        // Process in chunks to avoid exceeding query value limits
+        foreach (array_chunk($childIds, self::RELATION_QUERY_CHUNK_SIZE) as $chunk) {
+            $chunkDocs = $this->find($relatedCollection->getId(), [
+                Query::equal($twoWayKey, $chunk),
+                ...$otherQueries
+            ]);
+            array_push($relatedDocuments, ...$chunkDocs);
+        }
 
         // Group related documents by child ID
         $relatedByChildId = [];
@@ -4088,11 +4105,16 @@ private function populateManyToManyRelationshipsBatch(array $documents, Document
 
         $junction = $this->getJunctionCollection($collection, $relatedCollection, $side);
 
-        // Fetch all junction records for all documents in a single query
-        $junctions = $this->skipRelationships(fn () => $this->find($junction, [
-            Query::equal($twoWayKey, $documentIds),
-            Query::limit(PHP_INT_MAX)
-        ]));
+        // Fetch all junction records for all documents, chunking to stay within query limits
+        $junctions = [];
+
+        // Process in chunks to avoid exceeding query value limits
+        foreach (array_chunk($documentIds, self::RELATION_QUERY_CHUNK_SIZE) as $chunk) {
+            $chunkJunctions = $this->skipRelationships(fn () => $this->find($junction, [
+                Query::equal($twoWayKey, $chunk)
+            ]));
+            array_push($junctions, ...$chunkJunctions);
+        }
 
         // Collect all related IDs from junctions
         $relatedIds = [];
@@ -4111,15 +4133,22 @@ private function populateManyToManyRelationshipsBatch(array $documents, Document
             }
         }
 
-        // Fetch all related documents in a single query
+        // Fetch all related documents, chunking to stay within query limits
         $related = [];
         $allRelatedDocs = [];
         if (!empty($relatedIds)) {
-            $foundRelated = $this->find($relatedCollection->getId(), [
-                Query::equal('$id', array_unique($relatedIds)),
-                Query::limit(PHP_INT_MAX),
-                ...$queries
-            ]);
+            $uniqueRelatedIds = array_unique($relatedIds);
+            $foundRelated = [];
+
+            // Process in chunks to avoid exceeding query value limits
+            foreach (array_chunk($uniqueRelatedIds, self::RELATION_QUERY_CHUNK_SIZE) as $chunk) {
+                $chunkDocs = $this->find($relatedCollection->getId(), [
+                    Query::equal('$id', $chunk),
+                    ...$queries
+                ]);
+                array_push($foundRelated, ...$chunkDocs);
+            }
+
             $allRelatedDocs = $foundRelated;
 
             // Index related documents by ID for quick lookup
diff --git a/src/Database/Validator/Queries/Documents.php b/src/Database/Validator/Queries/Documents.php
index 289ccbe5b..92d4e4e69 100644
--- a/src/Database/Validator/Queries/Documents.php
+++ b/src/Database/Validator/Queries/Documents.php
@@ -27,7 +27,7 @@ public function __construct(
         array $attributes,
         array $indexes,
         string $idAttributeType,
-        int $maxValuesCount = 100,
+        int $maxValuesCount = 5000,
         \DateTime $minAllowedDate = new \DateTime('0000-01-01'),
         \DateTime $maxAllowedDate = new \DateTime('9999-12-31'),
     ) {
diff --git a/src/Database/Validator/Query/Filter.php b/src/Database/Validator/Query/Filter.php
index 9c60f551c..6f4dc3d99 100644
--- a/src/Database/Validator/Query/Filter.php
+++ b/src/Database/Validator/Query/Filter.php
@@ -28,7 +28,7 @@ class Filter extends Base
     public function __construct(
         array $attributes,
         private readonly string $idAttributeType,
-        private readonly int $maxValuesCount = 100,
+        private readonly int $maxValuesCount = 5000,
         private readonly \DateTime $minAllowedDate = new \DateTime('0000-01-01'),
         private readonly \DateTime $maxAllowedDate = new \DateTime('9999-12-31'),
     ) {

From cc3cbcdc121c7d06195d339246bdac3612bb50bc Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 2 Oct 2025 17:32:48 +1300
Subject: [PATCH 42/45] Update benchmark

---
 bin/tasks/relationships.php | 359 +++++++++++++++++++++++++++++-------
 1 file changed, 291 insertions(+), 68 deletions(-)

diff --git a/bin/tasks/relationships.php b/bin/tasks/relationships.php
index a7305dcfa..2909327b6 100644
--- a/bin/tasks/relationships.php
+++ b/bin/tasks/relationships.php
@@ -4,7 +4,6 @@
 
 use Swoole\Database\PDOConfig;
 use Swoole\Database\PDOPool;
-use Swoole\Runtime;
 use Utopia\Cache\Adapter\None as NoCache;
 use Utopia\Cache\Cache;
 use Utopia\CLI\Console;
@@ -25,12 +24,12 @@
 
 // Global pools for faster document generation
 $namesPool = ['Alice', 'Bob', 'Carol', 'Dave', 'Eve', 'Frank', 'Grace', 'Heidi', 'Ivan', 'Judy', 'Mallory', 'Niaj', 'Olivia', 'Peggy', 'Quentin', 'Rupert', 'Sybil', 'Trent', 'Uma', 'Victor'];
-$genresPool = ['fashion','food','travel','music','lifestyle','fitness','diy','sports','finance'];
-$tagsPool   = ['short','quick','easy','medium','hard'];
+$genresPool = ['fashion', 'food', 'travel', 'music', 'lifestyle', 'fitness', 'diy', 'sports', 'finance'];
+$tagsPool = ['short', 'quick', 'easy', 'medium', 'hard'];
 
 /**
  * @Example
- * docker compose exec tests bin/load --adapter=mariadb --limit=1000
+ * docker compose exec tests bin/relationships --adapter=mariadb --limit=1000
  */
 $cli
     ->task('relationships')
@@ -39,15 +38,14 @@
     ->param('limit', 0, new Integer(true), 'Total number of records to add to database')
     ->param('name', 'myapp_' . uniqid(), new Text(0), 'Name of created database.', true)
     ->param('sharedTables', false, new Boolean(true), 'Whether to use shared tables', true)
-    ->action(function (string $adapter, int $limit, string $name, bool $sharedTables) {
+    ->param('runs', 1, new Integer(true), 'Number of times to run benchmarks', true)
+    ->action(function (string $adapter, int $limit, string $name, bool $sharedTables, int $runs) {
         $start = null;
         $namespace = '_ns';
         $cache = new Cache(new NoCache());
 
         Console::info("Filling {$adapter} with {$limit} records: {$name}");
 
-        //Runtime::enableCoroutine();
-
         $dbAdapters = [
             'mariadb' => [
                 'host' => 'mariadb',
@@ -85,7 +83,6 @@
 
         $cfg = $dbAdapters[$adapter];
 
-        //Co\run(function () use (&$start, $limit, $name, $sharedTables, $namespace, $cache, $cfg) {
         $pdo = new PDO(
             ($cfg['dsn'])($cfg['host'], $cfg['port']),
             $cfg['user'],
@@ -100,6 +97,9 @@
 
         createRelationshipSchema($database);
 
+        // Create categories and users once before parallel batch creation
+        $globalDocs = createGlobalDocuments($database);
+
         $pdo = null;
 
         $pool = new PDOPool(
@@ -110,13 +110,13 @@
                 ->withCharset('utf8mb4')
                 ->withUsername($cfg['user'])
                 ->withPassword($cfg['pass']),
-            128
+            size: 64
         );
 
         $start = \microtime(true);
 
         for ($i = 0; $i < $limit / 1000; $i++) {
-            \go(function () use ($cfg, $pool, $name, $namespace, $sharedTables, $cache) {
+            go(function () use ($cfg, $pool, $name, $namespace, $sharedTables, $cache, $globalDocs) {
                 try {
                     $pdo = $pool->get();
 
@@ -125,20 +125,41 @@
                         ->setNamespace($namespace)
                         ->setSharedTables($sharedTables);
 
-                    createRelationshipDocuments($database);
+                    createRelationshipDocuments($database, $globalDocs['categories'], $globalDocs['users']);
                     $pool->put($pdo);
                 } catch (\Throwable $error) {
-                    Console::error('Coroutine error: ' . $error->getMessage());
+                    // Errors caught but documents still created successfully - likely concurrent update race conditions
                 }
             });
         }
 
-        benchmarkSingleQueries($database);
-        benchmarkBatchQueries($database);
-        benchmarkPagination($database);
-
         $time = microtime(true) - $start;
-        Console::success("Completed in {$time} seconds");
+        Console::success("Document creation completed in {$time} seconds");
+
+        // Display relationship structure
+        displayRelationshipStructure();
+
+        // Collect benchmark results across runs
+        $results = [];
+
+        Console::info("Running benchmarks {$runs} time(s)...");
+
+        for ($run = 1; $run <= $runs; $run++) {
+            if ($runs > 1) {
+                Console::info("Run {$run}/{$runs}");
+            }
+
+            $results[] = [
+                'single' => benchmarkSingle($database),
+                'batch100' => benchmarkBatch100($database),
+                'batch1000' => benchmarkBatch1000($database),
+                'batch5000' => benchmarkBatch5000($database),
+                'pagination' => benchmarkPagination($database),
+            ];
+        }
+
+        // Calculate and display averages
+        displayBenchmarkResults($results, $runs);
     });
 
 function createRelationshipSchema(Database $database): void
@@ -153,6 +174,7 @@ function createRelationshipSchema(Database $database): void
     $database->createCollection('authors', permissions: [
         Permission::create(Role::any()),
         Permission::read(Role::any()),
+        Permission::update(Role::any()),
     ]);
     $database->createAttribute('authors', 'name', Database::VAR_STRING, 256, true);
     $database->createAttribute('authors', 'created', Database::VAR_DATETIME, 0, true, filters: ['datetime']);
@@ -163,6 +185,7 @@ function createRelationshipSchema(Database $database): void
     $database->createCollection('articles', permissions: [
         Permission::create(Role::any()),
         Permission::read(Role::any()),
+        Permission::update(Role::any()),
     ]);
     $database->createAttribute('articles', 'title', Database::VAR_STRING, 256, true);
     $database->createAttribute('articles', 'text', Database::VAR_STRING, 5000, true);
@@ -173,6 +196,7 @@ function createRelationshipSchema(Database $database): void
     $database->createCollection('users', permissions: [
         Permission::create(Role::any()),
         Permission::read(Role::any()),
+        Permission::update(Role::any()),
     ]);
     $database->createAttribute('users', 'username', Database::VAR_STRING, 256, true);
     $database->createAttribute('users', 'email', Database::VAR_STRING, 256, true);
@@ -181,16 +205,71 @@ function createRelationshipSchema(Database $database): void
     $database->createCollection('comments', permissions: [
         Permission::create(Role::any()),
         Permission::read(Role::any()),
+        Permission::update(Role::any()),
     ]);
     $database->createAttribute('comments', 'content', Database::VAR_STRING, 256, true);
     $database->createAttribute('comments', 'likes', Database::VAR_INTEGER, 8, true, signed: false);
 
+    $database->createCollection('profiles', permissions: [
+        Permission::create(Role::any()),
+        Permission::read(Role::any()),
+        Permission::update(Role::any()),
+    ]);
+    $database->createAttribute('profiles', 'bio_extended', Database::VAR_STRING, 10000, true);
+    $database->createAttribute('profiles', 'social_links', Database::VAR_STRING, 256, true, array: true);
+    $database->createAttribute('profiles', 'verified', Database::VAR_BOOLEAN, 0, true);
+
+    $database->createCollection('categories', permissions: [
+        Permission::create(Role::any()),
+        Permission::read(Role::any()),
+        Permission::update(Role::any()),
+    ]);
+    $database->createAttribute('categories', 'name', Database::VAR_STRING, 256, true);
+    $database->createAttribute('categories', 'description', Database::VAR_STRING, 1000, true);
+
     $database->createRelationship('authors', 'articles', Database::RELATION_MANY_TO_MANY, true, onDelete: Database::RELATION_MUTATE_SET_NULL);
     $database->createRelationship('articles', 'comments', Database::RELATION_ONE_TO_MANY, true, twoWayKey: 'article', onDelete: Database::RELATION_MUTATE_CASCADE);
     $database->createRelationship('users', 'comments', Database::RELATION_ONE_TO_MANY, true, twoWayKey: 'user', onDelete: Database::RELATION_MUTATE_CASCADE);
+    $database->createRelationship('authors', 'profiles', Database::RELATION_ONE_TO_ONE, true, twoWayKey: 'author', onDelete: Database::RELATION_MUTATE_CASCADE);
+    $database->createRelationship('articles', 'categories', Database::RELATION_MANY_TO_ONE, true, id: 'category', twoWayKey: 'articles', onDelete: Database::RELATION_MUTATE_SET_NULL);
 }
 
-function createRelationshipDocuments(Database $database): void
+function createGlobalDocuments(Database $database): array
+{
+    global $genresPool, $namesPool;
+
+    // Generate categories (for many-to-one relationship with articles)
+    $categoryDocs = [];
+    foreach ($genresPool as $genre) {
+        $categoryDocs[] = new Document([
+            '$id' => 'category_' . \uniqid(),
+            'name' => \ucfirst($genre),
+            'description' => 'Articles about ' . $genre,
+        ]);
+    }
+
+    // Create categories once - documents are modified in place with IDs
+    $database->createDocuments('categories', $categoryDocs);
+
+    // Generate users globally (1000 users to share across all batches)
+    $userDocs = [];
+    for ($u = 0; $u < 1000; $u++) {
+        $userDocs[] = new Document([
+            '$id' => 'user_' . \uniqid(),
+            'username' => $namesPool[\array_rand($namesPool)],
+            'email' => \strtolower($namesPool[\array_rand($namesPool)]) . '@example.com',
+            'password' => \bin2hex(\random_bytes(8)),
+        ]);
+    }
+
+    // Create users once
+    $database->createDocuments('users', $userDocs);
+
+    // Return both categories and users
+    return ['categories' => $categoryDocs, 'users' => $userDocs];
+}
+
+function createRelationshipDocuments(Database $database, array $categories, array $users): void
 {
     global $namesPool, $genresPool, $tagsPool;
 
@@ -199,39 +278,40 @@ function createRelationshipDocuments(Database $database): void
 
     // Prepare pools for nested data
     $numAuthors = 10;
-    $numUsers = 10;
     $numArticlesPerAuthor = 10;
     $numCommentsPerArticle = 10;
 
-    // Generate users
-    $users = [];
-    for ($u = 0; $u < $numUsers; $u++) {
-        $users[] = new Document([
-            'username' => $namesPool[\array_rand($namesPool)],
-            'email'    => \strtolower($namesPool[\array_rand($namesPool)]) . '@example.com',
-            'password' => \bin2hex(\random_bytes(8)),
-        ]);
-    }
-
     // Generate authors with nested articles and comments
     for ($a = 0; $a < $numAuthors; $a++) {
         $author = new Document([
-            'name'      => $namesPool[array_rand($namesPool)],
-            'created'   => DateTime::now(),
-            'bio'       => \substr(\bin2hex(\random_bytes(32)), 0, 100),
-            'avatar'    => 'https://example.com/avatar/' . $a,
-            'website'   => 'https://example.com/user/' . $a,
+            'name' => $namesPool[array_rand($namesPool)],
+            'created' => DateTime::now(),
+            'bio' => \substr(\bin2hex(\random_bytes(32)), 0, 100),
+            'avatar' => 'https://example.com/avatar/' . $a,
+            'website' => 'https://example.com/user/' . $a,
         ]);
 
+        // Create profile for author (one-to-one relationship)
+        $profile = new Document([
+            'bio_extended' => \substr(\bin2hex(\random_bytes(128)), 0, 500),
+            'social_links' => [
+                'https://twitter.com/author' . $a,
+                'https://linkedin.com/in/author' . $a,
+            ],
+            'verified' => (bool)\mt_rand(0, 1),
+        ]);
+        $author->setAttribute('profiles', $profile);
+
         // Nested articles
         $authorArticles = [];
         for ($i = 0; $i < $numArticlesPerAuthor; $i++) {
             $article = new Document([
-                'title'   => 'Article ' . ($i + 1) . ' by ' . $author->getAttribute('name'),
-                'text'    => \substr(\bin2hex(\random_bytes(64)), 0, \mt_rand(100, 200)),
-                'genre'   => $genresPool[array_rand($genresPool)],
-                'views'   => \mt_rand(0, 1000),
-                'tags'    => \array_slice($tagsPool, 0, \mt_rand(1, \count($tagsPool))),
+                'title' => 'Article ' . ($i + 1) . ' by ' . $author->getAttribute('name'),
+                'text' => \substr(\bin2hex(\random_bytes(64)), 0, \mt_rand(100, 200)),
+                'genre' => $genresPool[array_rand($genresPool)],
+                'views' => \mt_rand(0, 1000),
+                'tags' => \array_slice($tagsPool, 0, \mt_rand(1, \count($tagsPool))),
+                'category' => $categories[\array_rand($categories)],
             ]);
 
             // Nested comments
@@ -239,8 +319,8 @@ function createRelationshipDocuments(Database $database): void
             for ($c = 0; $c < $numCommentsPerArticle; $c++) {
                 $comment = new Document([
                     'content' => 'Comment ' . ($c + 1),
-                    'likes'   => \mt_rand(0, 10000),
-                    'user'    => $users[\array_rand($users)],
+                    'likes' => \mt_rand(0, 10000),
+                    'user' => $users[\array_rand($users)],
                 ]);
                 $comments[] = $comment;
             }
@@ -253,9 +333,6 @@ function createRelationshipDocuments(Database $database): void
         $documents[] = $author;
     }
 
-    $time = microtime(true) - $start;
-    Console::info("Prepared nested documents in {$time} seconds");
-
     // Insert authors (with nested articles, comments, and users)
     $start = \microtime(true);
     $database->createDocuments('authors', $documents);
@@ -266,61 +343,207 @@ function createRelationshipDocuments(Database $database): void
 /**
  * Benchmark querying a single document from each collection.
  */
-function benchmarkSingleQueries(Database $database): void
+function benchmarkSingle(Database $database): array
 {
-    $collections = ['authors', 'articles', 'users', 'comments'];
+    $collections = ['authors', 'articles', 'users', 'comments', 'profiles', 'categories'];
+    $results = [];
+
     foreach ($collections as $collection) {
         // Fetch one document ID to use
-        $docs = $database->find($collection, [Query::limit(1)]);
-        if (empty($docs)) {
-            Console::warning("No documents in {$collection} for single query benchmark.");
-            continue;
-        }
-        $id = $docs[0]->getId();
+        $docs = $database->findOne($collection);
+        $id = $docs->getId();
 
         $start = microtime(true);
         $database->getDocument($collection, $id);
         $time = microtime(true) - $start;
 
-        Console::info("Single query ({$collection}) took {$time} seconds");
+        $results[$collection] = $time;
     }
+
+    return $results;
 }
 
 /**
- * Benchmark querying 20 documents from each collection.
+ * Benchmark querying 100 documents from each collection.
  */
-function benchmarkBatchQueries(Database $database): void
+function benchmarkBatch100(Database $database): array
 {
-    $collections = ['authors', 'articles', 'users', 'comments'];
+    $collections = ['authors', 'articles', 'users', 'comments', 'profiles', 'categories'];
+    $results = [];
+
     foreach ($collections as $collection) {
         $start = microtime(true);
-        $database->find($collection, [Query::limit(20)]);
+        $database->find($collection, [Query::limit(100)]);
         $time = microtime(true) - $start;
 
-        Console::info("Batch query 20 ({$collection}) took {$time} seconds");
+        $results[$collection] = $time;
     }
+
+    return $results;
 }
 
 /**
- * Benchmark pagination through entire collection in chunks of 100.
+ * Benchmark querying 1000 documents from each collection.
  */
-function benchmarkPagination(Database $database): void
+function benchmarkBatch1000(Database $database): array
 {
-    $collections = ['authors', 'articles', 'users', 'comments'];
+    $collections = ['authors', 'articles', 'users', 'comments', 'profiles', 'categories'];
+    $results = [];
+
     foreach ($collections as $collection) {
-        $offset = 0;
+        $start = microtime(true);
+        $database->find($collection, [Query::limit(1000)]);
+        $time = microtime(true) - $start;
+
+        $results[$collection] = $time;
+    }
+
+    return $results;
+}
+
+/**
+ * Benchmark querying 5000 documents from each collection.
+ */
+function benchmarkBatch5000(Database $database): array
+{
+    $collections = ['authors', 'articles', 'users', 'comments', 'profiles', 'categories'];
+    $results = [];
+
+    foreach ($collections as $collection) {
+        $start = microtime(true);
+        $database->find($collection, [Query::limit(5000)]);
+        $time = microtime(true) - $start;
+
+        $results[$collection] = $time;
+    }
+
+    return $results;
+}
+
+/**
+ * Benchmark cursor pagination through entire collection in chunks of 100.
+ */
+function benchmarkPagination(Database $database): array
+{
+    $collections = ['authors', 'articles', 'users', 'comments', 'profiles', 'categories'];
+    $results = [];
+
+    foreach ($collections as $collection) {
+        $total = 0;
         $limit = 100;
+        $cursor = null;
         $start = microtime(true);
         do {
-            $docs = $database->find($collection, [
-                Query::limit($limit),
-                Query::offset($offset),
-            ]);
+            $queries = [Query::limit($limit)];
+            if ($cursor !== null) {
+                $queries[] = Query::cursorAfter($cursor);
+            }
+            $docs = $database->find($collection, $queries);
             $count = count($docs);
-            $offset += $limit;
+            $total += $count;
+            if ($count > 0) {
+                $cursor = $docs[$count - 1];
+            }
         } while ($count === $limit);
         $time = microtime(true) - $start;
 
-        Console::info("Pagination ({$collection}) over all documents took {$time} seconds");
+        $results[$collection] = $time;
+    }
+
+    return $results;
+}
+
+/**
+ * Display relationship structure diagram
+ */
+function displayRelationshipStructure(): void
+{
+    Console::success("\n========================================");
+    Console::success("Relationship Structure");
+    Console::success("========================================\n");
+
+    Console::info("Collections:");
+    Console::log("  • authors      (name, created, bio, avatar, website)");
+    Console::log("  • articles     (title, text, genre, views, tags[])");
+    Console::log("  • comments     (content, likes)");
+    Console::log("  • users        (username, email, password)");
+    Console::log("  • profiles     (bio_extended, social_links[], verified)");
+    Console::log("  • categories   (name, description)");
+    Console::log("");
+
+    Console::info("Relationships:");
+    Console::log("  ┌─────────────────────────────────────────────────────────────┐");
+    Console::log("  │  authors ◄─────────────► articles  (Many-to-Many)          │");
+    Console::log("  │    └─► profiles (One-to-One)                                │");
+    Console::log("  │                                                              │");
+    Console::log("  │  articles ─────────────► comments  (One-to-Many)            │");
+    Console::log("  │    └─► categories (Many-to-One)                             │");
+    Console::log("  │                                                              │");
+    Console::log("  │  users ────────────────► comments  (One-to-Many)            │");
+    Console::log("  └─────────────────────────────────────────────────────────────┘");
+    Console::log("");
+
+    Console::info("Relationship Coverage:");
+    Console::log("  ✓ One-to-One:    authors ◄─► profiles");
+    Console::log("  ✓ One-to-Many:   articles ─► comments, users ─► comments");
+    Console::log("  ✓ Many-to-One:   articles ─► categories");
+    Console::log("  ✓ Many-to-Many:  authors ◄─► articles");
+    Console::log("");
+}
+
+/**
+ * Display benchmark results as a formatted table
+ */
+function displayBenchmarkResults(array $results, int $runs): void
+{
+    $collections = ['authors', 'articles', 'users', 'comments', 'profiles', 'categories'];
+    $benchmarks = ['single', 'batch100', 'batch1000', 'batch5000', 'pagination'];
+    $benchmarkLabels = [
+        'single' => 'Single Query',
+        'batch100' => 'Batch 100',
+        'batch1000' => 'Batch 1000',
+        'batch5000' => 'Batch 5000',
+        'pagination' => 'Pagination',
+    ];
+
+    // Calculate averages
+    $averages = [];
+    foreach ($benchmarks as $benchmark) {
+        $averages[$benchmark] = [];
+        foreach ($collections as $collection) {
+            $total = 0;
+            foreach ($results as $run) {
+                $total += $run[$benchmark][$collection] ?? 0;
+            }
+            $averages[$benchmark][$collection] = $total / $runs;
+        }
+    }
+
+    Console::success("\n========================================");
+    Console::success("Benchmark Results (Average of {$runs} run" . ($runs > 1 ? 's' : '') . ")");
+    Console::success("========================================\n");
+
+    // Calculate column widths
+    $collectionWidth = 12;
+    $timeWidth = 12;
+
+    // Print header
+    $header = str_pad('Collection', $collectionWidth) . ' | ';
+    foreach ($benchmarkLabels as $label) {
+        $header .= str_pad($label, $timeWidth) . ' | ';
+    }
+    Console::info($header);
+    Console::info(str_repeat('-', strlen($header)));
+
+    // Print results for each collection
+    foreach ($collections as $collection) {
+        $row = str_pad(ucfirst($collection), $collectionWidth) . ' | ';
+        foreach ($benchmarks as $benchmark) {
+            $time = number_format($averages[$benchmark][$collection] * 1000, 2); // Convert to ms
+            $row .= str_pad($time . ' ms', $timeWidth) . ' | ';
+        }
+        Console::log($row);
     }
+
+    Console::log('');
 }

From dff915be825afc7518f26d83c32b0487809af326 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 2 Oct 2025 22:43:10 +1300
Subject: [PATCH 43/45] Increase limit

---
 bin/tasks/relationships.php | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/bin/tasks/relationships.php b/bin/tasks/relationships.php
index 2909327b6..3f1a9e8bb 100644
--- a/bin/tasks/relationships.php
+++ b/bin/tasks/relationships.php
@@ -1,5 +1,8 @@
 <?php
 
+ini_set('memory_limit', '4G');
+ini_set('xdebug.max_nesting_level', '-1');
+
 global $cli;
 
 use Swoole\Database\PDOConfig;
@@ -98,7 +101,7 @@
         createRelationshipSchema($database);
 
         // Create categories and users once before parallel batch creation
-        $globalDocs = createGlobalDocuments($database);
+        $globalDocs = createGlobalDocuments($database, $limit);
 
         $pdo = null;
 
@@ -234,16 +237,18 @@ function createRelationshipSchema(Database $database): void
     $database->createRelationship('articles', 'categories', Database::RELATION_MANY_TO_ONE, true, id: 'category', twoWayKey: 'articles', onDelete: Database::RELATION_MUTATE_SET_NULL);
 }
 
-function createGlobalDocuments(Database $database): array
+function createGlobalDocuments(Database $database, int $limit): array
 {
     global $genresPool, $namesPool;
 
-    // Generate categories (for many-to-one relationship with articles)
+    // Scale categories based on limit (minimum 9, scales up to 100 max)
+    $numCategories = min(100, max(9, (int)($limit / 10000)));
     $categoryDocs = [];
-    foreach ($genresPool as $genre) {
+    for ($i = 0; $i < $numCategories; $i++) {
+        $genre = $genresPool[$i % count($genresPool)];
         $categoryDocs[] = new Document([
             '$id' => 'category_' . \uniqid(),
-            'name' => \ucfirst($genre),
+            'name' => \ucfirst($genre) . ($i >= count($genresPool) ? ' ' . ($i + 1) : ''),
             'description' => 'Articles about ' . $genre,
         ]);
     }
@@ -251,13 +256,14 @@ function createGlobalDocuments(Database $database): array
     // Create categories once - documents are modified in place with IDs
     $database->createDocuments('categories', $categoryDocs);
 
-    // Generate users globally (1000 users to share across all batches)
+    // Scale users based on limit (10% of total documents)
+    $numUsers = max(1000, (int)($limit / 10));
     $userDocs = [];
-    for ($u = 0; $u < 1000; $u++) {
+    for ($u = 0; $u < $numUsers; $u++) {
         $userDocs[] = new Document([
             '$id' => 'user_' . \uniqid(),
-            'username' => $namesPool[\array_rand($namesPool)],
-            'email' => \strtolower($namesPool[\array_rand($namesPool)]) . '@example.com',
+            'username' => $namesPool[\array_rand($namesPool)] . '_' . $u,
+            'email' => 'user' . $u . '@example.com',
             'password' => \bin2hex(\random_bytes(8)),
         ]);
     }
@@ -349,8 +355,8 @@ function benchmarkSingle(Database $database): array
     $results = [];
 
     foreach ($collections as $collection) {
-        // Fetch one document ID to use
-        $docs = $database->findOne($collection);
+        // Fetch one document ID to use (skip relationships to avoid infinite recursion)
+        $docs = $database->skipRelationships(fn() => $database->findOne($collection));
         $id = $docs->getId();
 
         $start = microtime(true);

From 18926bf42c9fe5ddd18f188b98bc17276a3a5d4e Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Thu, 2 Oct 2025 23:22:05 +1300
Subject: [PATCH 44/45] Fix limits

# Conflicts:
#	src/Database/Database.php
#	tests/e2e/Adapter/Scopes/RelationshipTests.php
---
 src/Database/Database.php | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/Database/Database.php b/src/Database/Database.php
index 8dd0b1977..ea22e7392 100644
--- a/src/Database/Database.php
+++ b/src/Database/Database.php
@@ -3799,6 +3799,7 @@ private function populateOneToOneRelationshipsBatch(array $documents, Document $
         foreach (array_chunk($uniqueRelatedIds, self::RELATION_QUERY_CHUNK_SIZE) as $chunk) {
             $chunkDocs = $this->find($relatedCollection->getId(), [
                 Query::equal('$id', $chunk),
+                Query::limit(PHP_INT_MAX),
                 ...$queries
             ]);
             array_push($relatedDocuments, ...$chunkDocs);
@@ -3892,6 +3893,7 @@ private function populateOneToManyRelationshipsBatch(array $documents, Document
         foreach (array_chunk($parentIds, self::RELATION_QUERY_CHUNK_SIZE) as $chunk) {
             $chunkDocs = $this->find($relatedCollection->getId(), [
                 Query::equal($twoWayKey, $chunk),
+                Query::limit(PHP_INT_MAX),
                 ...$otherQueries
             ]);
             array_push($relatedDocuments, ...$chunkDocs);
@@ -3989,6 +3991,7 @@ private function populateManyToOneRelationshipsBatch(array $documents, Document
         foreach (array_chunk($childIds, self::RELATION_QUERY_CHUNK_SIZE) as $chunk) {
             $chunkDocs = $this->find($relatedCollection->getId(), [
                 Query::equal($twoWayKey, $chunk),
+                Query::limit(PHP_INT_MAX),
                 ...$otherQueries
             ]);
             array_push($relatedDocuments, ...$chunkDocs);
@@ -4111,7 +4114,8 @@ private function populateManyToManyRelationshipsBatch(array $documents, Document
         // Process in chunks to avoid exceeding query value limits
         foreach (array_chunk($documentIds, self::RELATION_QUERY_CHUNK_SIZE) as $chunk) {
             $chunkJunctions = $this->skipRelationships(fn () => $this->find($junction, [
-                Query::equal($twoWayKey, $chunk)
+                Query::equal($twoWayKey, $chunk),
+                Query::limit(PHP_INT_MAX)
             ]));
             array_push($junctions, ...$chunkJunctions);
         }
@@ -4144,6 +4148,7 @@ private function populateManyToManyRelationshipsBatch(array $documents, Document
             foreach (array_chunk($uniqueRelatedIds, self::RELATION_QUERY_CHUNK_SIZE) as $chunk) {
                 $chunkDocs = $this->find($relatedCollection->getId(), [
                     Query::equal('$id', $chunk),
+                    Query::limit(PHP_INT_MAX),
                     ...$queries
                 ]);
                 array_push($foundRelated, ...$chunkDocs);

From cefe7c65c49b15042a06b8a0d56bd7ff919bbc78 Mon Sep 17 00:00:00 2001
From: Jake Barnby <jakeb994@gmail.com>
Date: Fri, 3 Oct 2025 00:00:31 +1300
Subject: [PATCH 45/45] Fix unit tests

---
 bin/tasks/relationships.php               |  2 +-
 src/Database/Validator/Query/Filter.php   |  5 +++++
 tests/unit/Validator/Query/FilterTest.php | 13 ++++++++-----
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/bin/tasks/relationships.php b/bin/tasks/relationships.php
index 3f1a9e8bb..595d01531 100644
--- a/bin/tasks/relationships.php
+++ b/bin/tasks/relationships.php
@@ -356,7 +356,7 @@ function benchmarkSingle(Database $database): array
 
     foreach ($collections as $collection) {
         // Fetch one document ID to use (skip relationships to avoid infinite recursion)
-        $docs = $database->skipRelationships(fn() => $database->findOne($collection));
+        $docs = $database->skipRelationships(fn () => $database->findOne($collection));
         $id = $docs->getId();
 
         $start = microtime(true);
diff --git a/src/Database/Validator/Query/Filter.php b/src/Database/Validator/Query/Filter.php
index 6f4dc3d99..32d1ddd09 100644
--- a/src/Database/Validator/Query/Filter.php
+++ b/src/Database/Validator/Query/Filter.php
@@ -330,6 +330,11 @@ public function isValid($value): bool
         }
     }
 
+    public function getMaxValuesCount(): int
+    {
+        return $this->maxValuesCount;
+    }
+
     public function getMethodType(): string
     {
         return self::METHOD_TYPE_FILTER;
diff --git a/tests/unit/Validator/Query/FilterTest.php b/tests/unit/Validator/Query/FilterTest.php
index ff7bd2630..a0ec65eeb 100644
--- a/tests/unit/Validator/Query/FilterTest.php
+++ b/tests/unit/Validator/Query/FilterTest.php
@@ -6,12 +6,11 @@
 use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Query;
-use Utopia\Database\Validator\Query\Base;
 use Utopia\Database\Validator\Query\Filter;
 
 class FilterTest extends TestCase
 {
-    protected Base|null $validator = null;
+    protected Filter|null $validator = null;
 
     /**
      * @throws \Utopia\Database\Exception
@@ -45,7 +44,10 @@ public function setUp(): void
             ]),
         ];
 
-        $this->validator = new Filter($attributes, Database::VAR_INTEGER);
+        $this->validator = new Filter(
+            $attributes,
+            Database::VAR_INTEGER
+        );
     }
 
     public function testSuccess(): void
@@ -106,13 +108,14 @@ public function testEmptyValues(): void
 
     public function testMaxValuesCount(): void
     {
+        $max = $this->validator->getMaxValuesCount();
         $values = [];
-        for ($i = 1; $i <= 200; $i++) {
+        for ($i = 1; $i <= $max + 1; $i++) {
             $values[] = $i;
         }
 
         $this->assertFalse($this->validator->isValid(Query::equal('integer', $values)));
-        $this->assertEquals('Query on attribute has greater than 100 values: integer', $this->validator->getDescription());
+        $this->assertEquals('Query on attribute has greater than '.$max.' values: integer', $this->validator->getDescription());
     }
 
     public function testNotContains(): void