From c651c220a7520f81089a3b6a2bef22bb49304a33 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 14 Jul 2025 11:50:06 +0300 Subject: [PATCH 01/22] transactions support --- bin/init-mongo-replica-set.sh | 32 +++++ composer.json | 2 +- composer.lock | 28 ++-- docker-compose.yml | 13 +- mongo-keyfile | 16 +++ src/Database/Adapter.php | 3 - src/Database/Adapter/MariaDB.php | 1 - src/Database/Adapter/Mongo.php | 225 +++++++++++++++++++++++------- tests/e2e/Adapter/MongoDBTest.php | 35 +++++ 9 files changed, 288 insertions(+), 67 deletions(-) create mode 100644 bin/init-mongo-replica-set.sh create mode 100644 mongo-keyfile diff --git a/bin/init-mongo-replica-set.sh b/bin/init-mongo-replica-set.sh new file mode 100644 index 000000000..4233e4545 --- /dev/null +++ b/bin/init-mongo-replica-set.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +set -e + +echo "Waiting for MongoDB to be ready..." +until docker compose exec mongo mongosh --eval "print('MongoDB is ready')" > /dev/null 2>&1; do + sleep 1 +done + +echo "Initializing MongoDB replica set..." + +# First, initialize the replica set without authentication +echo "Initializing replica set..." +docker compose exec mongo mongosh --eval 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongo:27017"}]})' + +# Wait for the replica set to be ready +echo "Waiting for replica set to be ready..." +until docker compose exec mongo mongosh --eval "rs.status().ok" | grep -q "1"; do + sleep 2 +done + +echo "Replica set initialized successfully!" + +# Now create the admin user and enable authentication +echo "Creating admin user and enabling authentication..." +docker compose exec mongo mongosh --eval 'use admin; db.createUser({user: "root", pwd: "password", roles: [{role: "root", db: "admin"}]})' + +# Test authentication +echo "Testing authentication..." +docker compose exec mongo mongosh admin -u root -p password --eval 'db.runCommand({ping: 1})' + +echo "MongoDB replica set is ready for transactions!" \ No newline at end of file diff --git a/composer.json b/composer.json index 7300239c4..c03a6747d 100755 --- a/composer.json +++ b/composer.json @@ -39,7 +39,7 @@ "utopia-php/framework": "0.33.*", "utopia-php/cache": "0.13.*", "utopia-php/pools": "0.8.*", - "utopia-php/mongo": "dev-feat-bulk-writes as 0.3.1" + "utopia-php/mongo": "dev-feat-mongo-transactions as 0.3.1" }, "require-dev": { "fakerphp/faker": "1.23.*", diff --git a/composer.lock b/composer.lock index 59db2f26b..2bc7a510d 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "cd1babfd7f7750ad399c915edd6209ad", + "content-hash": "a3b0ff08e6addea30a6380a0b19a0529", "packages": [ { "name": "brick/math", @@ -1993,16 +1993,16 @@ }, { "name": "utopia-php/mongo", - "version": "dev-feat-bulk-writes", + "version": "dev-feat-mongo-transactions", "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "414d4d099386ba742d1620fe2a75afd6105ad611" + "reference": "71c062c42bca6a4f709ddb86670089d5d4a5d7d5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/414d4d099386ba742d1620fe2a75afd6105ad611", - "reference": "414d4d099386ba742d1620fe2a75afd6105ad611", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/71c062c42bca6a4f709ddb86670089d5d4a5d7d5", + "reference": "71c062c42bca6a4f709ddb86670089d5d4a5d7d5", "shasum": "" }, "require": { @@ -2047,9 +2047,9 @@ ], "support": { "issues": "https://github.com/utopia-php/mongo/issues", - "source": "https://github.com/utopia-php/mongo/tree/feat-bulk-writes" + "source": "https://github.com/utopia-php/mongo/tree/feat-mongo-transactions" }, - "time": "2025-07-08T17:47:22+00:00" + "time": "2025-07-14T08:20:00+00:00" }, { "name": "utopia-php/pools", @@ -2290,16 +2290,16 @@ }, { "name": "laravel/pint", - "version": "v1.23.0", + "version": "v1.24.0", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "9ab851dba4faa51a3c3223dd3d07044129021024" + "reference": "0345f3b05f136801af8c339f9d16ef29e6b4df8a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/9ab851dba4faa51a3c3223dd3d07044129021024", - "reference": "9ab851dba4faa51a3c3223dd3d07044129021024", + "url": "https://api.github.com/repos/laravel/pint/zipball/0345f3b05f136801af8c339f9d16ef29e6b4df8a", + "reference": "0345f3b05f136801af8c339f9d16ef29e6b4df8a", "shasum": "" }, "require": { @@ -2310,7 +2310,7 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.76.0", + "friendsofphp/php-cs-fixer": "^3.82.2", "illuminate/view": "^11.45.1", "larastan/larastan": "^3.5.0", "laravel-zero/framework": "^11.45.0", @@ -2355,7 +2355,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-07-03T10:37:47+00:00" + "time": "2025-07-10T18:09:32+00:00" }, { "name": "myclabs/deep-copy", @@ -4261,7 +4261,7 @@ "aliases": [ { "package": "utopia-php/mongo", - "version": "dev-feat-bulk-writes", + "version": "dev-feat-mongo-transactions", "alias": "0.3.1", "alias_normalized": "0.3.1.0" } diff --git a/docker-compose.yml b/docker-compose.yml index 1af22637e..26911a3af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,10 +79,19 @@ services: - database ports: - "9706:27017" + volumes: + - ./mongo-keyfile:/etc/mongo-keyfile:ro + - mongo-data:/data/db environment: MONGO_INITDB_DATABASE: utopia_testing MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: password + MONGO_INITDB_ROOT_PASSWORD: password + command: > + mongod --replSet rs0 + --auth + --keyFile /etc/mongo-keyfile +# Manyally initate the replica set +#docker compose exec mongo mongosh admin -u root -p password --eval 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongo:27017"}]})' mongo-express: image: mongo-express @@ -146,5 +155,7 @@ services: networks: - database +volumes: + mongo-data: networks: database: diff --git a/mongo-keyfile b/mongo-keyfile new file mode 100644 index 000000000..5585939eb --- /dev/null +++ b/mongo-keyfile @@ -0,0 +1,16 @@ +ydIuYSvU/9QLt7fkH32IdXbP2z2+w+fzSEoolW8Q1Z8nLhRyrZF0Zq7a0KzeNI7K +gPIl1ikI6ob6h0+RxYmGeOOUjjkcBlkvYrmABDKsRipTkTTp4z0fUBTIUJV0lVvs +N9+VpM0/pLLIhI8jb38aa7pmsoufBQ3uiNR68ZFykPqzZQ4d5VfMqfZk7z3dpFlh +DURPOOG0HAFe68MLXVFYdaHGW4yomuTPrpzWSiUhFAPFEBYg4elARQc4CaiinFds +SQi/SrUsYMGODPr+on9/lboia/SInaSP+dzDqpsbL29atvIVHtU29RlPJdZ2V1ub +Oe2O1xN9F59TtjNUgDiAtMGKTMS/0S1mbPC6Og5JAR7U4xZ7/6S5n3+p0RjYyTlH +fhssJ7pc/bveN6mShNrsIKK0Z50YYjablzm07EDJYhfEWMG5Wu1AvEVqEH68ioDl +JL5QO63A2bXvMN7dXS69+E0hHn6xaZYu+CnKedvgWdyhraCT1Q01ZyDyv2y7isGD +1BAlNLlt+cPMCitETcxZne+JHdkL/mDKffHUPM4Drtzchg4DbiG49uC9Ib7zTws+ +NcburXY+9B8j7WN7ZHXhiB7/OWJ/IHJCZTdKz70mEPH4AHoRFpZNM5eMnYxYdbQD +40MhAS7fuOYhtFIQiQ+SCeFMucE3KYvp1JpTVQwT4SNrIlHPqfPn5xFBcgDjhvwT +hHJCgXP4HrRuf47Ta6kHy2UFQ7r5JOqSZSOFwP+tUyfhjEB5ZWJ1qCUZxFagoc9A +//9SoyulZwCxEr2ijmes1Nzv56hSTjYb6pPjFWd92G87w+VZv4R/vF5nwcYUyuIS +iQWPs/kOzb4NeJW24lNzR2zH2BsJt3OI+BFY64cc8O0o6EtFWcoabwyJYKe6RXPX +0S4ngcnGzRP+tVa6LsrjAYrNpmZDrP9x93pXQHfByTS2oSaI1eGeAagFTu/HS2kC +uCJ0HfH99sRSgJ1Ab+2C8G8305meDAbtdCtvl/1anPnV6ISy diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 3d59e3744..490d058cf 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -374,10 +374,7 @@ public function withTransaction(callable $callback): mixed for ($attempts = 0; $attempts < 3; $attempts++) { try { $this->startTransaction(); - //var_dump($attempts); $result = $callback(); - //var_dump($result); - $this->commitTransaction(); return $result; diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 8062839c8..dc626b665 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1173,7 +1173,6 @@ public function createOrUpdateDocuments( $bindValues = []; $documentIds = []; $documentTenants = []; - foreach ($changes as $change) { $document = $change->getNew(); $attributes = $document->getAttributes(); diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 5fade391a..6836b9422 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -11,7 +11,6 @@ use Utopia\Database\DateTime; use Utopia\Database\Document; use Utopia\Database\Exception as DatabaseException; -use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\Duplicate; use Utopia\Database\Exception\Timeout; use Utopia\Database\Query; @@ -44,6 +43,14 @@ class Mongo extends Adapter //protected ?int $timeout = null; + /** + * Transaction/session state for MongoDB transactions + */ + private ?object $sessionId = null; // Store raw BSON id object + private ?int $txnNumber = null; + protected int $inTransaction = 0; + private bool $firstOpInTransaction = false; + /** * Constructor. * @@ -74,19 +81,153 @@ public function clearTimeout(string $event): void $this->timeout = 0; } + /** + * @template T + * @param callable(): T $callback + * @return T + * @throws \Throwable + */ + public function withTransaction(callable $callback): mixed + { + // We removed the attmpts to retry the transaction. + // Since if it's rolling back the second time, it will fail + //becouse we already run one abortTransaction. + try { + $this->startTransaction(); + $result = $callback(); + $this->commitTransaction(); + return $result; + } catch (\Throwable $action) { + try { + $this->rollbackTransaction(); + } catch (\Throwable $rollback) { + $this->inTransaction = 0; + // Throw the original exception, not the rollback one + // Since if it's a duplicate key error, the rollback will fail + //and we want to throw the original exception. + } + $this->inTransaction = 0; + throw $action; + } + } + + public function startTransaction(): bool { - return true; + try { + if ($this->inTransaction === 0) { + if (!$this->sessionId) { + $this->sessionId = $this->client->startSession(); // Store raw id object + } + $this->txnNumber = ($this->txnNumber ?? 0) + 1; + $this->firstOpInTransaction = true; + + // Initialize the transaction on MongoDB's side with a dummy find operation + // This ensures the transaction is active even if validation fails later. + $this->client->query([ + 'find' => 'system.version', + 'filter' => $this->client->toObject([]), + 'limit' => 1, + 'lsid' => ['id' => $this->sessionId], + 'txnNumber' => new \MongoDB\BSON\Int64($this->txnNumber), // Long type for txnNumber + 'autocommit' => false, + 'startTransaction' => true + ], 'admin'); + + $this->firstOpInTransaction = false; + } + $this->inTransaction++; + return true; + } catch (\Throwable $e) { + throw new DatabaseException('Failed to start transaction: ' . $e->getMessage(), $e->getCode(), $e); + } } public function commitTransaction(): bool { - return true; + try { + if ($this->inTransaction === 0) { + throw new DatabaseException('No transaction in progress'); + } + $this->inTransaction--; + if ($this->inTransaction === 0) { + if (!$this->sessionId) { + throw new DatabaseException('No session in progress'); + } + $result = $this->client->commitTransaction( + ['id' => $this->sessionId], // Pass raw id object + $this->txnNumber, + false + ); + if (($result->ok ?? 0) !== 1.0) { + throw new DatabaseException('Failed to commit transaction'); + } + + // Session is now closed by the client using endSessions, reset our state + $this->sessionId = null; + $this->txnNumber = null; + + return true; + } + return true; + } catch (\Throwable $e) { + throw new DatabaseException('Failed to commit transaction: ' . $e->getMessage(), $e->getCode(), $e); + } } public function rollbackTransaction(): bool { - return true; + + try { + if ($this->inTransaction === 0) { + throw new DatabaseException('No transaction in progress'); + } + $this->inTransaction--; + if ($this->inTransaction === 0) { + if (!$this->sessionId) { + throw new DatabaseException('No session in progress'); + } + + $result = $this->client->abortTransaction( + ['id' => $this->sessionId], // Pass raw id object + $this->txnNumber, + false + ); + if (($result->ok ?? 0) !== 1.0) { + throw new DatabaseException('Failed to rollback transaction'); + } + + // Session is now closed by the client using endSessions, reset our state + $this->sessionId = null; + $this->txnNumber = null; + + return true; + } + return true; + } catch (\Throwable $e) { + throw new DatabaseException('Failed to rollback transaction: ' . $e->getMessage(), $e->getCode(), $e); + } + } + + /** + * Helper to add transaction/session context to command options if in transaction + */ + private function addTransactionContext(array $options = []): array + { + + if ($this->inTransaction) { + $options['lsid'] = ['id' => $this->sessionId]; + $options['txnNumber'] = new \MongoDB\BSON\Int64($this->txnNumber); + $options['autocommit'] = false; + + if ($this->firstOpInTransaction) { + // For MongoDB, the first operation in a transaction should include startTransaction + $options['startTransaction'] = true; + $this->firstOpInTransaction = false; + } + + } + return $options; } /** @@ -199,7 +340,6 @@ public function createCollection(string $name, array $attributes = [], array $in // Returns an array/object with the result document try { $this->getClient()->createCollection($id); - } catch (MongoException $e) { throw new Duplicate($e->getMessage(), $e->getCode(), $e); } @@ -394,7 +534,6 @@ public function createAttributes(string $collection, array $attributes): bool public function deleteAttribute(string $collection, string $id): bool { $collection = $this->getNamespace() . '_' . $this->filter($collection); - $this->getClient()->update( $collection, [], @@ -595,7 +734,6 @@ public function createIndex(string $collection, string $id, string $type, array $id = $this->filter($id); $indexes = []; - $options = []; // pass in custom index name $indexes['name'] = $id; @@ -636,7 +774,7 @@ public function createIndex(string $collection, string $id, string $type, array ]; } - return $this->client->createIndexes($name, [$indexes], $options); + return $this->client->createIndexes($name, [$indexes]); } /** @@ -767,9 +905,8 @@ public function createDocument(string $collection, Document $document): Document if (!empty($sequence)) { $record['_id'] = $sequence; } - - $result = $this->insertDocument($name, $this->removeNullKeys($record)); - + $options = $this->addTransactionContext([]); + $result = $this->insertDocument($name, $this->removeNullKeys($record), $options); $result = $this->replaceChars('_', '$', $result); $result = $this->timeToDocument($result); @@ -790,6 +927,9 @@ public function createDocuments(string $collection, array $documents): array { $name = $this->getNamespace() . '_' . $this->filter($collection); + // Initialize transaction context before validation to ensure transaction is active + $options = $this->addTransactionContext([]); + $records = []; $hasSequence = null; $documents = array_map(fn ($doc) => clone $doc, $documents); @@ -819,7 +959,7 @@ public function createDocuments(string $collection, array $documents): array $records[] = $this->removeNullKeys($record); } - $documents = $this->client->insertMany($name, $records); + $documents = $this->client->insertMany($name, $records, $options); foreach ($documents as $index => $document) { $documents[$index] = $this->replaceChars('_', '$', $this->client->toArray($document)); @@ -839,26 +979,12 @@ public function createDocuments(string $collection, array $documents): array * @return array * @throws Duplicate */ - private function insertDocument(string $name, array $document): array + private function insertDocument(string $name, array $document, array $options = []): array { try { - $bla = $this->client->insert($name, $document); - - $filters = []; - $filters['_uid'] = $document['_uid']; - - if ($this->sharedTables) { - $filters['_tenant'] = $this->getTenant(); - } - - $result = $this->client->find( - $name, - $filters, - ['limit' => 1] - )->cursor->firstBatch[0]; - - return $this->client->toArray($result); + $result = $this->client->insert($name, $document, $options); + return $result; } catch (MongoException $e) { throw new Duplicate($e->getMessage()); } @@ -890,7 +1016,8 @@ public function updateDocument(string $collection, string $id, Document $documen try { unset($record['_id']); // Don't update _id - $this->client->update($name, $filters, $record); + $options = $this->addTransactionContext([]); + $this->client->update($name, $filters, $record, $options); } catch (MongoException $e) { throw new Duplicate($e->getMessage()); } @@ -934,7 +1061,8 @@ public function updateDocuments(string $collection, Document $updates, array $do ]; try { - $this->client->update($name, $filters, $updateQuery, multi: true); + $options = $this->addTransactionContext([]); + $this->client->update($name, $filters, $updateQuery, multi: true, options: $options); } catch (MongoException $e) { throw new Duplicate($e->getMessage()); } @@ -1009,13 +1137,13 @@ public function createOrUpdateDocuments(string $collection, string $attribute, a 'filter' => $filter, 'update' => $update, ]; - } - - // Use the new bulkUpsert method + } + + $options = $this->addTransactionContext([]); $this->client->bulkUpsert( $name, $operations, - ["ordered" => false] // TODO Do we want to continue if an error is thrown? + options: $options ); // Get sequences for documents that were created @@ -1106,6 +1234,7 @@ public function increaseDocumentAttribute(string $collection, string $id, string $filters[$attribute] = ['$gte' => $min]; } + $options = $this->addTransactionContext([]); $this->client->update( $this->getNamespace() . '_' . $this->filter($collection), $filters, @@ -1113,6 +1242,7 @@ public function increaseDocumentAttribute(string $collection, string $id, string '$inc' => [$attribute => $value], '$set' => ['_updatedAt' => $this->toMongoDatetime($updatedAt)], ], + options: $options ); return true; @@ -1137,7 +1267,8 @@ public function deleteDocument(string $collection, string $id): bool $filters['_tenant'] = $this->getTenant(); } - $result = $this->client->delete($name, $filters); + $options = $this->addTransactionContext([]); + $result = $this->client->delete($name, $filters, 1, [], $options); return (!!$result); } @@ -1162,15 +1293,15 @@ public function deleteDocuments(string $collection, array $sequences, array $per $filters = $this->replaceInternalIdsKeys($filters, '$', '_', $this->operators); $filters = $this->timeFilter($filters); - - $options = []; + $options = $this->addTransactionContext([]); try { $count = $this->client->delete( collection: $name, filters: $filters, - options: $options, - limit: 0 + limit: 0, + deleteOptions: [], + options: $options ); } catch (MongoException $e) { $this->processException($e); @@ -1249,7 +1380,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, if ($this->sharedTables) { $filters['_tenant'] = $this->getTenant(); } - + // permissions if (Authorization::$status) { $roles = \implode('|', Authorization::getRoles()); @@ -1306,7 +1437,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $prevAttr = $this->filter($this->getInternalKeyForAttribute($originalPrev)); $tmp = $cursor[$originalPrev]; - if($originalPrev === '$sequence'){ + if ($originalPrev === '$sequence') { $tmp = new ObjectId($tmp); } @@ -1317,11 +1448,11 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $tmp = $cursor[$originalAttribute]; - if($originalAttribute === '$sequence'){ + if ($originalAttribute === '$sequence') { $tmp = new ObjectId($tmp); /** If there is only $sequence attribute in $orderAttributes skip Or And operators **/ - if(count($orderAttributes) === 1){ + if (count($orderAttributes) === 1) { $filters[$attribute] = [ $operator => $tmp ]; @@ -1356,16 +1487,16 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, } catch (MongoException $e) { throw $this->processException($e); } - + if (empty($results)) { return $found; } - + foreach ($this->client->toArray($results) as $result) { $record = $this->replaceChars('_', '$', (array)$result); - + $record = $this->timeToDocument($record); $found[] = new Document($record); diff --git a/tests/e2e/Adapter/MongoDBTest.php b/tests/e2e/Adapter/MongoDBTest.php index 55b21f8e4..2e18f8058 100644 --- a/tests/e2e/Adapter/MongoDBTest.php +++ b/tests/e2e/Adapter/MongoDBTest.php @@ -105,4 +105,39 @@ protected static function deleteIndex(string $collection, string $index): bool { return true; } + + /** + * Test that sessions are properly closed after transactions + */ + public function testSessionCleanup(): void + { + $database = static::getDatabase(); + $adapter = $database->getAdapter(); + + // Create a collection for testing + $collection = $database->createCollection('test_session_cleanup'); + + // Start a transaction + $adapter->startTransaction(); + + // Create a document in the transaction + $document = $database->createDocument('test_session_cleanup', new \Utopia\Database\Document([ + 'name' => 'test', + 'value' => 123 + ])); + + // Commit the transaction - session is closed using endSessions command + $adapter->commitTransaction(); + + // Verify the document was created + $this->assertNotNull($document->getId()); + + // The session should now be closed (sessionId should be null) + // We can verify this by checking that a new transaction starts fresh + $adapter->startTransaction(); + $adapter->rollbackTransaction(); + + // Clean up + $database->deleteCollection('test_session_cleanup'); + } } From b262c2168a79705b9fef8b0dadeb55696fa0f576 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 14 Jul 2025 18:48:53 +0300 Subject: [PATCH 02/22] remove replica-set file --- bin/init-mongo-replica-set.sh | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 bin/init-mongo-replica-set.sh diff --git a/bin/init-mongo-replica-set.sh b/bin/init-mongo-replica-set.sh deleted file mode 100644 index 4233e4545..000000000 --- a/bin/init-mongo-replica-set.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -set -e - -echo "Waiting for MongoDB to be ready..." -until docker compose exec mongo mongosh --eval "print('MongoDB is ready')" > /dev/null 2>&1; do - sleep 1 -done - -echo "Initializing MongoDB replica set..." - -# First, initialize the replica set without authentication -echo "Initializing replica set..." -docker compose exec mongo mongosh --eval 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongo:27017"}]})' - -# Wait for the replica set to be ready -echo "Waiting for replica set to be ready..." -until docker compose exec mongo mongosh --eval "rs.status().ok" | grep -q "1"; do - sleep 2 -done - -echo "Replica set initialized successfully!" - -# Now create the admin user and enable authentication -echo "Creating admin user and enabling authentication..." -docker compose exec mongo mongosh --eval 'use admin; db.createUser({user: "root", pwd: "password", roles: [{role: "root", db: "admin"}]})' - -# Test authentication -echo "Testing authentication..." -docker compose exec mongo mongosh admin -u root -p password --eval 'db.runCommand({ping: 1})' - -echo "MongoDB replica set is ready for transactions!" \ No newline at end of file From ba11cba078c70343b105a4df380b3c1cd1843ea7 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 15 Jul 2025 15:10:27 +0300 Subject: [PATCH 03/22] cleanup --- src/Database/Adapter/Mongo.php | 40 ++++++++++++++++++++++++------- tests/e2e/Adapter/MongoDBTest.php | 38 +++-------------------------- 2 files changed, 34 insertions(+), 44 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 6836b9422..a0f110174 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -89,9 +89,10 @@ public function clearTimeout(string $event): void */ public function withTransaction(callable $callback): mixed { - // We removed the attmpts to retry the transaction. - // Since if it's rolling back the second time, it will fail - //becouse we already run one abortTransaction. + // Removed the attmpts to retry the transaction. + //Unlike pdo if we run theabortTransaction more then once (same transactioId), + // it will throw an error the there is no transaction in progress. + try { $this->startTransaction(); $result = $callback(); @@ -163,7 +164,8 @@ public function commitTransaction(): bool throw new DatabaseException('Failed to commit transaction'); } - // Session is now closed by the client using endSessions, reset our state + // Session is now closed by the client using endSessions, state is reseted + // TODO do we want session per transaction or to manage it on the connection level? $this->sessionId = null; $this->txnNumber = null; @@ -225,7 +227,6 @@ private function addTransactionContext(array $options = []): array $options['startTransaction'] = true; $this->firstOpInTransaction = false; } - } return $options; } @@ -865,14 +866,14 @@ public function getDocument(string $collection, string $id, array $queries = [], } $result = $this->client->find($name, $filters, $options)->cursor->firstBatch; - + if (empty($result)) { return new Document([]); } $result = $this->replaceChars('_', '$', (array)$result[0]); $result = $this->timeToDocument($result); - + return new Document($result); } @@ -981,10 +982,31 @@ public function createDocuments(string $collection, array $documents): array */ private function insertDocument(string $name, array $document, array $options = []): array { - + try { $result = $this->client->insert($name, $document, $options); - return $result; + + + $filters = []; + $filters['_uid'] = $document['_uid']; + + if ($this->sharedTables) { + $filters['_tenant'] = $this->getTenant(); + } + + // in order to get the document we need to pass the transaction context to the find. + $this->client->find( + $name, + $filters, + array_merge($options, ['limit' => 1]) + )->cursor->firstBatch[0]; + + /** + * TODO Do we even need this find? + * We can just return the result from the insertDocument. + */ + + return $this->client->toArray($result); } catch (MongoException $e) { throw new Duplicate($e->getMessage()); } diff --git a/tests/e2e/Adapter/MongoDBTest.php b/tests/e2e/Adapter/MongoDBTest.php index 2e18f8058..4f16b9581 100644 --- a/tests/e2e/Adapter/MongoDBTest.php +++ b/tests/e2e/Adapter/MongoDBTest.php @@ -9,6 +9,9 @@ use Utopia\Database\Adapter\Mongo; use Utopia\Database\Database; use Utopia\Mongo\Client; +use Utopia\Database\Helpers\Permission; +use Utopia\Database\Helpers\Role; +use Utopia\Database\Document; class MongoDBTest extends Base { @@ -105,39 +108,4 @@ protected static function deleteIndex(string $collection, string $index): bool { return true; } - - /** - * Test that sessions are properly closed after transactions - */ - public function testSessionCleanup(): void - { - $database = static::getDatabase(); - $adapter = $database->getAdapter(); - - // Create a collection for testing - $collection = $database->createCollection('test_session_cleanup'); - - // Start a transaction - $adapter->startTransaction(); - - // Create a document in the transaction - $document = $database->createDocument('test_session_cleanup', new \Utopia\Database\Document([ - 'name' => 'test', - 'value' => 123 - ])); - - // Commit the transaction - session is closed using endSessions command - $adapter->commitTransaction(); - - // Verify the document was created - $this->assertNotNull($document->getId()); - - // The session should now be closed (sessionId should be null) - // We can verify this by checking that a new transaction starts fresh - $adapter->startTransaction(); - $adapter->rollbackTransaction(); - - // Clean up - $database->deleteCollection('test_session_cleanup'); - } } From 72c903e557e6b1ca02a72363622765fab34f4dfc Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 21 Jul 2025 10:13:03 +0300 Subject: [PATCH 04/22] clenup --- src/Database/Adapter/Mongo.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index d0468bae4..62feb2323 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -341,7 +341,6 @@ public function createCollection(string $name, array $attributes = [], array $in // Returns an array/object with the result document try { $this->getClient()->createCollection($id); - } catch (MongoException $e) { throw new Duplicate($e->getMessage(), $e->getCode(), $e); } @@ -536,7 +535,6 @@ public function createAttributes(string $collection, array $attributes): bool public function deleteAttribute(string $collection, string $id): bool { $collection = $this->getNamespace() . '_' . $this->filter($collection); - $this->getClient()->update( $collection, [], @@ -869,6 +867,7 @@ public function getDocument(string $collection, string $id, array $queries = [], } $result = $this->client->find($name, $filters, $options)->cursor->firstBatch; + if (empty($result)) { return new Document([]); } From 8421a343521e7aa1d2f459f28da674e4d29c62d4 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 21 Jul 2025 13:19:52 +0300 Subject: [PATCH 05/22] sync with feat-mongo-2 --- composer.lock | 113 +++++++++++++++++++++++++----- src/Database/Adapter/Mongo.php | 25 +++---- tests/e2e/Adapter/MongoDBTest.php | 3 - 3 files changed, 108 insertions(+), 33 deletions(-) diff --git a/composer.lock b/composer.lock index 2bc7a510d..c2909437c 100644 --- a/composer.lock +++ b/composer.lock @@ -193,23 +193,24 @@ }, { "name": "mongodb/mongodb", - "version": "1.21.1", + "version": "2.1.0", "source": { "type": "git", "url": "https://github.com/mongodb/mongo-php-library.git", - "reference": "37bc8df3a67ddf8380704a5ba5dbd00e92ec1f6a" + "reference": "3bbe7ba9578724c7e1f47fcd17c881c0995baaad" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/37bc8df3a67ddf8380704a5ba5dbd00e92ec1f6a", - "reference": "37bc8df3a67ddf8380704a5ba5dbd00e92ec1f6a", + "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/3bbe7ba9578724c7e1f47fcd17c881c0995baaad", + "reference": "3bbe7ba9578724c7e1f47fcd17c881c0995baaad", "shasum": "" }, "require": { "composer-runtime-api": "^2.0", - "ext-mongodb": "^1.21.0", + "ext-mongodb": "^2.1", "php": "^8.1", - "psr/log": "^1.1.4|^2|^3" + "psr/log": "^1.1.4|^2|^3", + "symfony/polyfill-php85": "^1.32" }, "replace": { "mongodb/builder": "*" @@ -263,9 +264,9 @@ ], "support": { "issues": "https://github.com/mongodb/mongo-php-library/issues", - "source": "https://github.com/mongodb/mongo-php-library/tree/1.21.1" + "source": "https://github.com/mongodb/mongo-php-library/tree/2.1.0" }, - "time": "2025-02-28T17:24:20+00:00" + "time": "2025-05-23T10:48:05+00:00" }, { "name": "nyholm/psr7", @@ -1711,6 +1712,82 @@ ], "time": "2024-09-09T11:45:10+00:00" }, + { + "name": "symfony/polyfill-php85", + "version": "v1.32.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php85.git", + "reference": "6fedf31ce4e3648f4ff5ca58bfd53127d38f05fd" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/6fedf31ce4e3648f4ff5ca58bfd53127d38f05fd", + "reference": "6fedf31ce4e3648f4ff5ca58bfd53127d38f05fd", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Php85\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 8.5+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-php85/tree/v1.32.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2025-05-02T08:40:52+00:00" + }, { "name": "symfony/service-contracts", "version": "v3.6.0", @@ -1997,17 +2074,17 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "71c062c42bca6a4f709ddb86670089d5d4a5d7d5" + "reference": "e9ee5bce6262e1504c368c10a5ae7df2f3737a0e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/71c062c42bca6a4f709ddb86670089d5d4a5d7d5", - "reference": "71c062c42bca6a4f709ddb86670089d5d4a5d7d5", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/e9ee5bce6262e1504c368c10a5ae7df2f3737a0e", + "reference": "e9ee5bce6262e1504c368c10a5ae7df2f3737a0e", "shasum": "" }, "require": { "ext-mongodb": "*", - "mongodb/mongodb": "^1.21", + "mongodb/mongodb": "2.1.0", "php": ">=8.0" }, "require-dev": { @@ -2049,7 +2126,7 @@ "issues": "https://github.com/utopia-php/mongo/issues", "source": "https://github.com/utopia-php/mongo/tree/feat-mongo-transactions" }, - "time": "2025-07-14T08:20:00+00:00" + "time": "2025-07-21T10:12:18+00:00" }, { "name": "utopia-php/pools", @@ -2627,16 +2704,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.12.27", + "version": "1.12.28", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "3a6e423c076ab39dfedc307e2ac627ef579db162" + "reference": "fcf8b71aeab4e1a1131d1783cef97b23a51b87a9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/3a6e423c076ab39dfedc307e2ac627ef579db162", - "reference": "3a6e423c076ab39dfedc307e2ac627ef579db162", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/fcf8b71aeab4e1a1131d1783cef97b23a51b87a9", + "reference": "fcf8b71aeab4e1a1131d1783cef97b23a51b87a9", "shasum": "" }, "require": { @@ -2681,7 +2758,7 @@ "type": "github" } ], - "time": "2025-05-21T20:51:45+00:00" + "time": "2025-07-17T17:15:39+00:00" }, { "name": "phpunit/php-code-coverage", diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 62feb2323..d956b2666 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -535,6 +535,7 @@ public function createAttributes(string $collection, array $attributes): bool public function deleteAttribute(string $collection, string $id): bool { $collection = $this->getNamespace() . '_' . $this->filter($collection); + $this->getClient()->update( $collection, [], @@ -867,14 +868,14 @@ public function getDocument(string $collection, string $id, array $queries = [], } $result = $this->client->find($name, $filters, $options)->cursor->firstBatch; - + if (empty($result)) { return new Document([]); } $result = $this->replaceChars('_', '$', (array)$result[0]); $result = $this->timeToDocument($result); - + return new Document($result); } @@ -982,14 +983,14 @@ public function createDocuments(string $collection, array $documents): array */ private function insertDocument(string $name, array $document, array $options = []): array { - + try { $result = $this->client->insert($name, $document, $options); - - + + $filters = []; $filters['_uid'] = $document['_uid']; - + if ($this->sharedTables) { $filters['_tenant'] = $this->getTenant(); } @@ -999,12 +1000,12 @@ private function insertDocument(string $name, array $document, array $options = $name, $filters, array_merge($options, ['limit' => 1]) - )->cursor->firstBatch[0]; - - /** - * TODO Do we even need this find? - * We can just return the result from the insertDocument. - */ + )->cursor->firstBatch[0]; + + /** + * TODO Do we even need this find? + * We can just return the result from the insertDocument. + */ return $this->client->toArray($result); } catch (MongoException $e) { diff --git a/tests/e2e/Adapter/MongoDBTest.php b/tests/e2e/Adapter/MongoDBTest.php index 4f16b9581..55b21f8e4 100644 --- a/tests/e2e/Adapter/MongoDBTest.php +++ b/tests/e2e/Adapter/MongoDBTest.php @@ -9,9 +9,6 @@ use Utopia\Database\Adapter\Mongo; use Utopia\Database\Database; use Utopia\Mongo\Client; -use Utopia\Database\Helpers\Permission; -use Utopia\Database\Helpers\Role; -use Utopia\Database\Document; class MongoDBTest extends Base { From 8cd9ac4fb3801410191f041ad3b6be75e5ce01b9 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 21 Jul 2025 13:48:02 +0300 Subject: [PATCH 06/22] updates --- composer.lock | 113 +++++++++++++++++++++++++++------ src/Database/Adapter/Mongo.php | 7 +- 2 files changed, 101 insertions(+), 19 deletions(-) diff --git a/composer.lock b/composer.lock index 2bc7a510d..c2909437c 100644 --- a/composer.lock +++ b/composer.lock @@ -193,23 +193,24 @@ }, { "name": "mongodb/mongodb", - "version": "1.21.1", + "version": "2.1.0", "source": { "type": "git", "url": "https://github.com/mongodb/mongo-php-library.git", - "reference": "37bc8df3a67ddf8380704a5ba5dbd00e92ec1f6a" + "reference": "3bbe7ba9578724c7e1f47fcd17c881c0995baaad" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/37bc8df3a67ddf8380704a5ba5dbd00e92ec1f6a", - "reference": "37bc8df3a67ddf8380704a5ba5dbd00e92ec1f6a", + "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/3bbe7ba9578724c7e1f47fcd17c881c0995baaad", + "reference": "3bbe7ba9578724c7e1f47fcd17c881c0995baaad", "shasum": "" }, "require": { "composer-runtime-api": "^2.0", - "ext-mongodb": "^1.21.0", + "ext-mongodb": "^2.1", "php": "^8.1", - "psr/log": "^1.1.4|^2|^3" + "psr/log": "^1.1.4|^2|^3", + "symfony/polyfill-php85": "^1.32" }, "replace": { "mongodb/builder": "*" @@ -263,9 +264,9 @@ ], "support": { "issues": "https://github.com/mongodb/mongo-php-library/issues", - "source": "https://github.com/mongodb/mongo-php-library/tree/1.21.1" + "source": "https://github.com/mongodb/mongo-php-library/tree/2.1.0" }, - "time": "2025-02-28T17:24:20+00:00" + "time": "2025-05-23T10:48:05+00:00" }, { "name": "nyholm/psr7", @@ -1711,6 +1712,82 @@ ], "time": "2024-09-09T11:45:10+00:00" }, + { + "name": "symfony/polyfill-php85", + "version": "v1.32.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php85.git", + "reference": "6fedf31ce4e3648f4ff5ca58bfd53127d38f05fd" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/6fedf31ce4e3648f4ff5ca58bfd53127d38f05fd", + "reference": "6fedf31ce4e3648f4ff5ca58bfd53127d38f05fd", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Php85\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 8.5+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-php85/tree/v1.32.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2025-05-02T08:40:52+00:00" + }, { "name": "symfony/service-contracts", "version": "v3.6.0", @@ -1997,17 +2074,17 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "71c062c42bca6a4f709ddb86670089d5d4a5d7d5" + "reference": "e9ee5bce6262e1504c368c10a5ae7df2f3737a0e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/71c062c42bca6a4f709ddb86670089d5d4a5d7d5", - "reference": "71c062c42bca6a4f709ddb86670089d5d4a5d7d5", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/e9ee5bce6262e1504c368c10a5ae7df2f3737a0e", + "reference": "e9ee5bce6262e1504c368c10a5ae7df2f3737a0e", "shasum": "" }, "require": { "ext-mongodb": "*", - "mongodb/mongodb": "^1.21", + "mongodb/mongodb": "2.1.0", "php": ">=8.0" }, "require-dev": { @@ -2049,7 +2126,7 @@ "issues": "https://github.com/utopia-php/mongo/issues", "source": "https://github.com/utopia-php/mongo/tree/feat-mongo-transactions" }, - "time": "2025-07-14T08:20:00+00:00" + "time": "2025-07-21T10:12:18+00:00" }, { "name": "utopia-php/pools", @@ -2627,16 +2704,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.12.27", + "version": "1.12.28", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "3a6e423c076ab39dfedc307e2ac627ef579db162" + "reference": "fcf8b71aeab4e1a1131d1783cef97b23a51b87a9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/3a6e423c076ab39dfedc307e2ac627ef579db162", - "reference": "3a6e423c076ab39dfedc307e2ac627ef579db162", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/fcf8b71aeab4e1a1131d1783cef97b23a51b87a9", + "reference": "fcf8b71aeab4e1a1131d1783cef97b23a51b87a9", "shasum": "" }, "require": { @@ -2681,7 +2758,7 @@ "type": "github" } ], - "time": "2025-05-21T20:51:45+00:00" + "time": "2025-07-17T17:15:39+00:00" }, { "name": "phpunit/php-code-coverage", diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 62feb2323..b343ed073 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -89,6 +89,11 @@ public function clearTimeout(string $event): void */ public function withTransaction(callable $callback): mixed { + // If the database is not a replica set, we can't use transactions + if(!$this->client->isReplicaSet()){ + return true; + } + // Removed the attmpts to retry the transaction. //Unlike pdo if we run theabortTransaction more then once (same transactioId), // it will throw an error the there is no transaction in progress. @@ -1162,7 +1167,7 @@ public function createOrUpdateDocuments(string $collection, string $attribute, a } $options = $this->addTransactionContext([]); - $this->client->bulkUpsert( + $this->client->upsert( $name, $operations, options: $options From 142d167e8ab62bc6fd8568aee9115de364f5b5dd Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 21 Jul 2025 16:58:51 +0300 Subject: [PATCH 07/22] updates --- src/Database/Adapter/Mongo.php | 42 ++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index b97ada4fa..5069dbf27 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -153,20 +153,21 @@ public function commitTransaction(): bool { try { if ($this->inTransaction === 0) { - throw new DatabaseException('No transaction in progress'); + return false; } $this->inTransaction--; if ($this->inTransaction === 0) { if (!$this->sessionId) { - throw new DatabaseException('No session in progress'); + return false; } - $result = $this->client->commitTransaction( - ['id' => $this->sessionId], // Pass raw id object - $this->txnNumber, - false - ); - if (($result->ok ?? 0) !== 1.0) { - throw new DatabaseException('Failed to commit transaction'); + try { + $result = $this->client->commitTransaction( + ['id' => $this->sessionId], // Pass raw id object + $this->txnNumber, + false + ); + } catch (\Throwable $e) { + throw new DatabaseException($e->getMessage(), $e->getCode(), $e); } // Session is now closed by the client using endSessions, state is reseted @@ -195,13 +196,14 @@ public function rollbackTransaction(): bool throw new DatabaseException('No session in progress'); } - $result = $this->client->abortTransaction( - ['id' => $this->sessionId], // Pass raw id object - $this->txnNumber, - false - ); - if (($result->ok ?? 0) !== 1.0) { - throw new DatabaseException('Failed to rollback transaction'); + try { + $result = $this->client->abortTransaction( + ['id' => $this->sessionId], // Pass raw id object + $this->txnNumber, + false + ); + } catch (\Throwable $e) { + throw new DatabaseException($e->getMessage(), $e->getCode(), $e); } // Session is now closed by the client using endSessions, reset our state @@ -936,7 +938,6 @@ public function createDocuments(string $collection, array $documents): array $name = $this->getNamespace() . '_' . $this->filter($collection); $options = $this->addTransactionContext([]); - $records = []; $hasSequence = null; $documents = array_map(fn ($doc) => clone $doc, $documents); @@ -991,8 +992,6 @@ private function insertDocument(string $name, array $document, array $options = try { $result = $this->client->insert($name, $document, $options); - - $filters = []; $filters['_uid'] = $document['_uid']; @@ -1070,6 +1069,7 @@ public function updateDocuments(string $collection, Document $updates, array $do { $name = $this->getNamespace() . '_' . $this->filter($collection); + $options = $this->addTransactionContext([]); $queries = [ Query::equal('$sequence', \array_map(fn ($document) => $document->getSequence(), $documents)) ]; @@ -1089,7 +1089,6 @@ public function updateDocuments(string $collection, Document $updates, array $do ]; try { - $options = $this->addTransactionContext([]); $this->client->update($name, $filters, $updateQuery, multi: true, options: $options); } catch (MongoException $e) { throw new Duplicate($e->getMessage()); @@ -1112,6 +1111,7 @@ public function createOrUpdateDocuments(string $collection, string $attribute, a try { $name = $this->getNamespace() . '_' . $this->filter($collection); + $attribute = $this->filter($attribute); $documentIds = []; @@ -1168,6 +1168,7 @@ public function createOrUpdateDocuments(string $collection, string $attribute, a } $options = $this->addTransactionContext([]); + $this->client->upsert( $name, $operations, @@ -1401,6 +1402,7 @@ protected function getInternalKeyForAttribute(string $attribute): string public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, string $forPermission = Database::PERMISSION_READ): array { $name = $this->getNamespace() . '_' . $this->filter($collection); + $queries = array_map(fn ($query) => clone $query, $queries); $filters = $this->buildFilters($queries); From e6b2bba2ae38b29bcd8e699039ac3fc407ba0c2a Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 21 Jul 2025 17:02:17 +0300 Subject: [PATCH 08/22] updates --- src/Database/Adapter/Mongo.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 5069dbf27..2169ab0f8 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -188,12 +188,12 @@ public function rollbackTransaction(): bool try { if ($this->inTransaction === 0) { - throw new DatabaseException('No transaction in progress'); + return false; } $this->inTransaction--; if ($this->inTransaction === 0) { if (!$this->sessionId) { - throw new DatabaseException('No session in progress'); + return false; } try { @@ -1402,7 +1402,7 @@ protected function getInternalKeyForAttribute(string $attribute): string public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, string $forPermission = Database::PERMISSION_READ): array { $name = $this->getNamespace() . '_' . $this->filter($collection); - + $queries = array_map(fn ($query) => clone $query, $queries); $filters = $this->buildFilters($queries); From ec67b5946bedde271bf3697b5587013c0740c259 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 21 Jul 2025 17:06:13 +0300 Subject: [PATCH 09/22] updates --- src/Database/Adapter/Mongo.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 2169ab0f8..12e1ca778 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -348,6 +348,7 @@ public function createCollection(string $name, array $attributes = [], array $in // Returns an array/object with the result document try { $this->getClient()->createCollection($id); + } catch (MongoException $e) { throw new Duplicate($e->getMessage(), $e->getCode(), $e); } @@ -1111,7 +1112,6 @@ public function createOrUpdateDocuments(string $collection, string $attribute, a try { $name = $this->getNamespace() . '_' . $this->filter($collection); - $attribute = $this->filter($attribute); $documentIds = []; @@ -1322,7 +1322,8 @@ public function deleteDocuments(string $collection, array $sequences, array $per $filters = $this->replaceInternalIdsKeys($filters, '$', '_', $this->operators); $filters = $this->timeFilter($filters); - $options = $this->addTransactionContext([]); + + $options = []; try { $count = $this->client->delete( @@ -1402,7 +1403,6 @@ protected function getInternalKeyForAttribute(string $attribute): string public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, string $forPermission = Database::PERMISSION_READ): array { $name = $this->getNamespace() . '_' . $this->filter($collection); - $queries = array_map(fn ($query) => clone $query, $queries); $filters = $this->buildFilters($queries); From 1760ea5674d9986455d1a0eb291c3346ee34ac5d Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 24 Jul 2025 18:57:17 +0300 Subject: [PATCH 10/22] sunc against upsert pr --- src/Database/Adapter/Mongo.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index fa191d10c..08f501fb8 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -90,7 +90,7 @@ public function clearTimeout(string $event): void public function withTransaction(callable $callback): mixed { // If the database is not a replica set, we can't use transactions - if(!$this->client->isReplicaSet()){ + if (!$this->client->isReplicaSet()) { return true; } From 9edd05fd127d47a9da564b0875e67e93b4bc0792 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 24 Jul 2025 23:48:54 +0300 Subject: [PATCH 11/22] Update composer.lock and docker-compose.yml for dependency versions and MongoDB configuration adjustments --- composer.lock | 12 ++++++------ docker-compose.yml | 16 +++++++++++----- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/composer.lock b/composer.lock index c2909437c..7d0e83076 100644 --- a/composer.lock +++ b/composer.lock @@ -2074,23 +2074,23 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "e9ee5bce6262e1504c368c10a5ae7df2f3737a0e" + "reference": "68dcddf5f266822f4a5b361a1aeafa5a9036d4bf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/e9ee5bce6262e1504c368c10a5ae7df2f3737a0e", - "reference": "e9ee5bce6262e1504c368c10a5ae7df2f3737a0e", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/68dcddf5f266822f4a5b361a1aeafa5a9036d4bf", + "reference": "68dcddf5f266822f4a5b361a1aeafa5a9036d4bf", "shasum": "" }, "require": { - "ext-mongodb": "*", + "ext-mongodb": "2.1.1", "mongodb/mongodb": "2.1.0", "php": ">=8.0" }, "require-dev": { "fakerphp/faker": "^1.14", "laravel/pint": "1.2.*", - "phpstan/phpstan": "1.8.*", + "phpstan/phpstan": "2.1.*", "phpunit/phpunit": "^9.4", "swoole/ide-helper": "4.8.0" }, @@ -2126,7 +2126,7 @@ "issues": "https://github.com/utopia-php/mongo/issues", "source": "https://github.com/utopia-php/mongo/tree/feat-mongo-transactions" }, - "time": "2025-07-21T10:12:18+00:00" + "time": "2025-07-24T20:15:02+00:00" }, { "name": "utopia-php/pools", diff --git a/docker-compose.yml b/docker-compose.yml index 26911a3af..993f78065 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -85,11 +85,17 @@ services: environment: MONGO_INITDB_DATABASE: utopia_testing MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: password - command: > - mongod --replSet rs0 - --auth - --keyFile /etc/mongo-keyfile + MONGO_INITDB_ROOT_PASSWORD: password + # Replica set + # MONGO_INITDB_REPLICA_SET: rs0 + # MONGO_INITDB_REPLICA_SET_NAME: rs0 + # MONGO_INITDB_REPLICA_SET_KEY: rs0 + # MONGO_INITDB_REPLICA_SET_KEY_FILE: /etc/mongo-keyfile + # MONGO_INITDB_REPLICA_SET_KEY_FILE_NAME: mongo-keyfile + # MONGO_INITDB_REPLICA_SET_KEY_FILE_PATH: /etc/mongo-keyfile + # MONGO_INITDB_REPLICA_SET_KEY_FILE_PATH_NAME: mongo-keyfile + # MONGO_INITDB_REPLICA_SET_KEY_FILE_PATH_NAME_NAME: mongo-keyfile + # Manyally initate the replica set #docker compose exec mongo mongosh admin -u root -p password --eval 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongo:27017"}]})' From 85748b7fbc79a4382d0e871ade20c27d1342df17 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 27 Jul 2025 16:16:53 +0300 Subject: [PATCH 12/22] sync with feat-mongo-2 --- composer.lock | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/composer.lock b/composer.lock index 7d0e83076..8598421d1 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "a3b0ff08e6addea30a6380a0b19a0529", + "content-hash": "f6dc7d44d9bb06432e3a2d2bf026022a", "packages": [ { "name": "brick/math", @@ -2070,16 +2070,16 @@ }, { "name": "utopia-php/mongo", - "version": "dev-feat-mongo-transactions", + "version": "0.5.0", "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "68dcddf5f266822f4a5b361a1aeafa5a9036d4bf" + "reference": "b7a4901f552f6383b274d5a6c84feba6357afa95" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/68dcddf5f266822f4a5b361a1aeafa5a9036d4bf", - "reference": "68dcddf5f266822f4a5b361a1aeafa5a9036d4bf", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/b7a4901f552f6383b274d5a6c84feba6357afa95", + "reference": "b7a4901f552f6383b274d5a6c84feba6357afa95", "shasum": "" }, "require": { @@ -2124,9 +2124,9 @@ ], "support": { "issues": "https://github.com/utopia-php/mongo/issues", - "source": "https://github.com/utopia-php/mongo/tree/feat-mongo-transactions" + "source": "https://github.com/utopia-php/mongo/tree/0.5.0" }, - "time": "2025-07-24T20:15:02+00:00" + "time": "2025-07-25T04:02:37+00:00" }, { "name": "utopia-php/pools", @@ -4335,18 +4335,9 @@ "time": "2022-10-09T10:19:07+00:00" } ], - "aliases": [ - { - "package": "utopia-php/mongo", - "version": "dev-feat-mongo-transactions", - "alias": "0.3.1", - "alias_normalized": "0.3.1.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/mongo": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 4b24c1391bd7895fbb6ad80c7cfb87e04439ea8c Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 27 Jul 2025 16:18:23 +0300 Subject: [PATCH 13/22] sync with feat-mongo-2 --- src/Database/Adapter/Mongo.php | 4 ++-- src/Database/Database.php | 4 ++-- tests/e2e/Adapter/Scopes/IndexTests.php | 2 +- tests/e2e/Adapter/Scopes/PermissionTests.php | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 93ff3ba3b..ea66f8e59 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -1322,7 +1322,7 @@ public function createOrUpdateDocuments(string $collection, string $attribute, a $tempDocuments[] = $change->getNew(); } } - + $sequences = $this->getSequences($collection, $tempDocuments); foreach ($changes as $change) { @@ -1354,7 +1354,7 @@ public function getSequences(string $collection, array $documents): array foreach ($documents as $document) { if (empty($document->getSequence())) { $documentIds[] = $document->getId(); - + if ($this->sharedTables) { $documentTenants[] = $document->getTenant(); } diff --git a/src/Database/Database.php b/src/Database/Database.php index 631fb83cd..dce84737a 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -5079,14 +5079,14 @@ public function createOrUpdateDocumentsWithIncrease( /** * @var array $chunk */ - + $batch = $this->withTransaction(fn () => Authorization::skip(fn () => $this->adapter->createOrUpdateDocuments( $collection->getId(), $attribute, $chunk ))); $batch = $this->adapter->getSequences($collection->getId(), $batch); - + foreach ($chunk as $change) { if ($change->getOld()->isEmpty()) { $created++; diff --git a/tests/e2e/Adapter/Scopes/IndexTests.php b/tests/e2e/Adapter/Scopes/IndexTests.php index 1d40c553e..df3207f35 100644 --- a/tests/e2e/Adapter/Scopes/IndexTests.php +++ b/tests/e2e/Adapter/Scopes/IndexTests.php @@ -304,7 +304,7 @@ public function testIndexLengthZero(): void $database = static::getDatabase(); $database->createCollection(__FUNCTION__); - + $database->createAttribute(__FUNCTION__, 'title1', Database::VAR_STRING, $database->getAdapter()->getMaxIndexLength() + 300, true); try { diff --git a/tests/e2e/Adapter/Scopes/PermissionTests.php b/tests/e2e/Adapter/Scopes/PermissionTests.php index 97ebc8de1..285ff2e4c 100644 --- a/tests/e2e/Adapter/Scopes/PermissionTests.php +++ b/tests/e2e/Adapter/Scopes/PermissionTests.php @@ -759,7 +759,7 @@ public function testCollectionPermissionsRelationshipsFindWorks(array $data): vo $this->expectNotToPerformAssertions(); return; } - + $documents = $database->find( $collection->getId() ); From 5bcf41ed990b36400663b6f301b255f176d98299 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 27 Jul 2025 16:58:07 +0300 Subject: [PATCH 14/22] Refactor MongoDB configuration in docker-compose and enhance transaction handling in Mongo adapter. Updated command for MongoDB to support replica sets and improved transaction callback handling in the adapter. --- docker-compose.yml | 12 ++---------- src/Database/Adapter/Mongo.php | 3 ++- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2f9b8301c..8b64c9822 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -87,16 +87,8 @@ services: environment: MONGO_INITDB_DATABASE: utopia_testing MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: password - # Replica set - # MONGO_INITDB_REPLICA_SET: rs0 - # MONGO_INITDB_REPLICA_SET_NAME: rs0 - # MONGO_INITDB_REPLICA_SET_KEY: rs0 - # MONGO_INITDB_REPLICA_SET_KEY_FILE: /etc/mongo-keyfile - # MONGO_INITDB_REPLICA_SET_KEY_FILE_NAME: mongo-keyfile - # MONGO_INITDB_REPLICA_SET_KEY_FILE_PATH: /etc/mongo-keyfile - # MONGO_INITDB_REPLICA_SET_KEY_FILE_PATH_NAME: mongo-keyfile - # MONGO_INITDB_REPLICA_SET_KEY_FILE_PATH_NAME_NAME: mongo-keyfile + MONGO_INITDB_ROOT_PASSWORD: password + command: mongod --replSet rs0 --bind_ip_all --keyFile /etc/mongo-keyfile # Manyally initate the replica set #docker compose exec mongo mongosh admin -u root -p password --eval 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongo:27017"}]})' diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index ea66f8e59..293322fa9 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -91,7 +91,8 @@ public function withTransaction(callable $callback): mixed { // If the database is not a replica set, we can't use transactions if (!$this->client->isReplicaSet()) { - return true; + $result = $callback(); + return $result; } // Removed the attmpts to retry the transaction. From f498894a1242aa8e0e4cb3b78f3a3e1ef003e692 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 29 Jul 2025 16:16:28 +0300 Subject: [PATCH 15/22] Update docker-compose.yml to add a note about manual initiation of the MongoDB replica set and user creation. --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 9c3d98bf7..f2da3f9b9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -91,6 +91,7 @@ services: command: mongod --replSet rs0 --bind_ip_all --keyFile /etc/mongo-keyfile # Manyally initate the replica set +# mongo users(!root) do not get created automatically!!! #docker compose exec mongo mongosh admin -u root -p password --eval 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongo:27017"}]})' mongo-express: From 2b65727a851a11671983faa22f6ebc582d99f26c Mon Sep 17 00:00:00 2001 From: Shimon Newman Date: Sun, 3 Aug 2025 10:11:52 +0300 Subject: [PATCH 16/22] Update src/Database/Adapter/Mongo.php Co-authored-by: Jake Barnby --- src/Database/Adapter/Mongo.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index d2af6b69f..7d80d7af0 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -225,7 +225,6 @@ public function rollbackTransaction(): bool */ private function addTransactionContext(array $options = []): array { - if ($this->inTransaction) { $options['lsid'] = ['id' => $this->sessionId]; $options['txnNumber'] = new \MongoDB\BSON\Int64($this->txnNumber); From 728e933a5cb360c5095240713ad97ab1394cb749 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 8 Sep 2025 21:53:28 +0300 Subject: [PATCH 17/22] Update Key validator to restrict length to 36 characters for MongoDB ID compliance --- src/Database/Validator/Key.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Database/Validator/Key.php b/src/Database/Validator/Key.php index 18ed1dd02..920ff7b92 100644 --- a/src/Database/Validator/Key.php +++ b/src/Database/Validator/Key.php @@ -76,8 +76,8 @@ public function isValid($value): bool if (\preg_match('/[^A-Za-z0-9\_\-\.]/', $value)) { return false; } - // Updated to 100 to suport mongodb ids - if (\mb_strlen($value) > 100) { + // At most 36 chars + if (\mb_strlen($value) > 36) { return false; } From c0e8f888521f537d1325756b80e50316f72d5002 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 8 Sep 2025 21:58:25 +0300 Subject: [PATCH 18/22] Enhance documentation for Mongo adapter by adding parameter type hints for addTransactionContext and related methods, improving code clarity and maintainability. --- src/Database/Adapter/Mongo.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 30cccad2c..d2f0181a8 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -221,6 +221,9 @@ public function rollbackTransaction(): bool /** * Helper to add transaction/session context to command options if in transaction + * + * @param array $options + * @return array */ private function addTransactionContext(array $options = []): array { @@ -1190,6 +1193,7 @@ public function createDocuments(Document $collection, array $documents): array * * @param string $name * @param array $document + * @param array $options * * @return array * @throws Duplicate From 771cc7eaa35a1bb39ed818e3ae6e610ceb15a9af Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 14 Sep 2025 16:40:34 +0300 Subject: [PATCH 19/22] linter --- src/Database/Adapter/Mongo.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index d2f0181a8..d765e8e3c 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -221,7 +221,7 @@ public function rollbackTransaction(): bool /** * Helper to add transaction/session context to command options if in transaction - * + * * @param array $options * @return array */ From 160c6f9cf32ae167188ae0b26c9205e2a825c0ac Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 14 Sep 2025 18:08:19 +0300 Subject: [PATCH 20/22] Update docker-compose.yml to enable MongoDB client mapping and enhance Mongo adapter with transaction context support in various methods. --- docker-compose.yml | 2 ++ src/Database/Adapter/Mongo.php | 20 +++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0b2eba586..a0c247beb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -104,6 +104,8 @@ services: # Manyally initate the replica set # mongo users(!root) do not get created automatically!!! +#sudo chmod 600 mongo-keyfile +#sudo chown mongodb:mongodb mongo-keyfile #docker compose exec mongo mongosh admin -u root -p password --eval 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongo:27017"}]})' mongo-express: diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 30bf7f4c2..6ee5fa7ed 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -1213,7 +1213,7 @@ private function insertDocument(string $name, array $document, array $options = $result = $this->client->find( $name, $filters, - ['limit' => 1] + array_merge(['limit' => 1], $options) )->cursor->firstBatch[0]; } catch (MongoException $e) { throw $this->processException($e); @@ -1435,7 +1435,8 @@ public function getSequences(string $collection, array $documents): array $filters['_tenant'] = $this->getTenantFilters($collection, $documentTenants); } try { - $results = $this->client->find($name, $filters, ['projection' => ['_uid' => 1, '_id' => 1]]); + $options = $this->addTransactionContext(['projection' => ['_uid' => 1, '_id' => 1]]); + $results = $this->client->find($name, $filters, $options); } catch (MongoException $e) { throw $this->processException($e); } @@ -1549,7 +1550,7 @@ public function deleteDocuments(string $collection, array $sequences, array $per $filters = $this->replaceInternalIdsKeys($filters, '$', '_', $this->operators); - $options = []; + $options = $this->addTransactionContext([]); try { $count = $this->client->delete( @@ -1660,6 +1661,9 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 $options['projection'] = $this->getAttributeProjection($selections); } + // Add transaction context to options + $options = $this->addTransactionContext($options); + $orFilters = []; foreach ($orderAttributes as $i => $originalAttribute) { @@ -1901,6 +1905,10 @@ public function count(Document $collection, array $queries = [], ?int $max = nul // Original count command (commented for reference and fallback) // Use this for single-instance MongoDB when performance is critical and accuracy is not a concern + + + + $options = $this->addTransactionContext([]); // return $this->client->count($name, $filters, $options); $pipeline = []; @@ -1933,7 +1941,8 @@ public function count(Document $collection, array $queries = [], ?int $max = nul } try { - $result = $this->client->aggregate($name, $pipeline); + + $result = $this->client->aggregate($name, $pipeline, $options); // Aggregation returns stdClass with cursor property containing firstBatch if (isset($result->cursor) && !empty($result->cursor->firstBatch)) { @@ -2004,7 +2013,8 @@ public function sum(Document $collection, string $attribute, array $queries = [] ], ]; - return $this->client->aggregate($name, $pipeline)->cursor->firstBatch[0]->total ?? 0; + $options = $this->addTransactionContext([]); + return $this->client->aggregate($name, $pipeline, $options)->cursor->firstBatch[0]->total ?? 0; } /** From 657f089c11ccc2ebb2a19fc05f49426d80a60dc1 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 26 Sep 2025 21:35:20 +1200 Subject: [PATCH 21/22] Fix tests --- composer.json | 2 +- composer.lock | 14 ++-- docker-compose.yml | 54 ++++++++------ src/Database/Adapter/Mongo.php | 70 +++++++------------ tests/resources/mongo/entrypoint.sh | 12 ++++ .../resources/mongo/mongo-keyfile | 0 6 files changed, 81 insertions(+), 71 deletions(-) create mode 100755 tests/resources/mongo/entrypoint.sh rename mongo-keyfile => tests/resources/mongo/mongo-keyfile (100%) diff --git a/composer.json b/composer.json index 4f3ff5b19..ad3bf6b8b 100755 --- a/composer.json +++ b/composer.json @@ -39,7 +39,7 @@ "utopia-php/framework": "0.33.*", "utopia-php/cache": "0.13.*", "utopia-php/pools": "0.8.*", - "utopia-php/mongo": "0.6.0" + "utopia-php/mongo": "0.7.*" }, "require-dev": { "fakerphp/faker": "1.23.*", diff --git a/composer.lock b/composer.lock index 07b1321b9..07fdebb32 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "e23429f4a3f7e66afaa960e249ee7525", + "content-hash": "0d7b7b4e8299046f6d9881d12a27708d", "packages": [ { "name": "brick/math", @@ -2166,16 +2166,16 @@ }, { "name": "utopia-php/mongo", - "version": "0.6.0", + "version": "0.7.0", "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "589e329a7fe4200e23ca87d65f3eb25a70ef0505" + "reference": "1363598f9f8e6c066f5821704be95e3e24ea66aa" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/589e329a7fe4200e23ca87d65f3eb25a70ef0505", - "reference": "589e329a7fe4200e23ca87d65f3eb25a70ef0505", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/1363598f9f8e6c066f5821704be95e3e24ea66aa", + "reference": "1363598f9f8e6c066f5821704be95e3e24ea66aa", "shasum": "" }, "require": { @@ -2221,9 +2221,9 @@ ], "support": { "issues": "https://github.com/utopia-php/mongo/issues", - "source": "https://github.com/utopia-php/mongo/tree/0.6.0" + "source": "https://github.com/utopia-php/mongo/tree/0.7.0" }, - "time": "2025-09-11T13:26:21+00:00" + "time": "2025-09-26T09:15:55+00:00" }, { "name": "utopia-php/pools", diff --git a/docker-compose.yml b/docker-compose.yml index a0c247beb..098d465ac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,6 @@ services: - ./dev/xdebug.ini:/usr/local/etc/php/conf.d/xdebug.ini - /var/run/docker.sock:/var/run/docker.sock - ./docker-compose.yml:/usr/src/code/docker-compose.yml - #- ./vendor/utopia-php/mongo/src/Client.php:/usr/src/code/vendor/utopia-php/mongo/src/Client.php environment: PHP_IDE_CONFIG: serverName=tests @@ -85,45 +84,54 @@ services: - MYSQL_ROOT_PASSWORD=password mongo: - image: mongo:latest + image: mongo:8.0.14 container_name: utopia-mongo + entrypoint: ["/entrypoint.sh"] networks: - database ports: - "9706:27017" volumes: - - ./mongo-keyfile:/etc/mongo-keyfile:ro - mongo-data:/data/db + - ./tests/resources/mongo/mongo-keyfile:/tmp/keyfile:ro + - ./tests/resources/mongo/entrypoint.sh:/entrypoint.sh:ro environment: - MONGO_INITDB_DATABASE: utopia_testing MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_PASSWORD: password - MONGO_INITDB_USERNAME: user - MONGO_INITDB_PASSWORD: paswword - command: mongod --replSet rs0 --bind_ip_all --keyFile /etc/mongo-keyfile - -# Manyally initate the replica set -# mongo users(!root) do not get created automatically!!! -#sudo chmod 600 mongo-keyfile -#sudo chown mongodb:mongodb mongo-keyfile -#docker compose exec mongo mongosh admin -u root -p password --eval 'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongo:27017"}]})' + MONGO_INITDB_DATABASE: utopia_testing + healthcheck: + test: | + bash -c " + if mongosh -u root -p password --authenticationDatabase admin --quiet --eval 'rs.status().ok' 2>/dev/null; then + exit 0 + else + mongosh -u root -p password --authenticationDatabase admin --quiet --eval \" + rs.initiate({_id: 'rs0', members: [{_id: 0, host: 'localhost:27017'}]}) + \" 2>/dev/null || exit 1 + fi + " + interval: 10s + timeout: 10s + retries: 10 + start_period: 30s mongo-express: image: mongo-express container_name: mongo-express + depends_on: + mongo: + condition: service_healthy networks: - database ports: - "8083:8081" environment: - ME_CONFIG_MONGODB_SERVER: mongo - ME_CONFIG_MONGODB_ADMINUSERNAME: root - ME_CONFIG_MONGODB_ADMINPASSWORD: password + ME_CONFIG_MONGODB_URL: mongodb://root:password@mongo:27017/?authSource=admin&directConnection=true ME_CONFIG_BASICAUTH_USERNAME: admin ME_CONFIG_BASICAUTH_PASSWORD: admin mysql: - image: mysql:8.0.41 + image: mysql:8.0.43 container_name: utopia-mysql networks: - database @@ -139,7 +147,7 @@ services: - SYS_NICE mysql-mirror: - image: mysql:8.0.41 + image: mysql:8.0.43 container_name: utopia-mysql-mirror networks: - database @@ -155,7 +163,7 @@ services: - SYS_NICE redis: - image: redis:7.4.1-alpine3.20 + image: redis:8.2.1-alpine3.22 container_name: utopia-redis ports: - "8708:6379" @@ -163,7 +171,7 @@ services: - database redis-mirror: - image: redis:7.4.1-alpine3.20 + image: redis:8.2.1-alpine3.22 container_name: utopia-redis-mirror ports: - "8709:6379" @@ -172,5 +180,11 @@ services: volumes: mongo-data: + networks: database: + +secrets: + mongo_keyfile: + file: ./tests/resources/mongo/mongo-keyfile + diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index de2e7ca7d..d69b3d994 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -3,6 +3,7 @@ namespace Utopia\Database\Adapter; use Exception; +use MongoDB\BSON\Int64; use MongoDB\BSON\Regex; use MongoDB\BSON\UTCDateTime; use Utopia\Database\Adapter; @@ -103,10 +104,6 @@ public function withTransaction(callable $callback): mixed return $result; } - // Removed the attmpts to retry the transaction. - //Unlike pdo if we run theabortTransaction more then once (same transactioId), - // it will throw an error the there is no transaction in progress. - try { $this->startTransaction(); $result = $callback(); @@ -115,18 +112,17 @@ public function withTransaction(callable $callback): mixed } catch (\Throwable $action) { try { $this->rollbackTransaction(); - } catch (\Throwable $rollback) { - $this->inTransaction = 0; + } catch (\Throwable) { // Throw the original exception, not the rollback one - // Since if it's a duplicate key error, the rollback will fail - //and we want to throw the original exception. + // Since if it's a duplicate key error, the rollback will fail, + // and we want to throw the original exception. } + $this->inTransaction = 0; throw $action; } } - public function startTransaction(): bool { try { @@ -171,16 +167,14 @@ public function commitTransaction(): bool } try { $result = $this->client->commitTransaction( - ['id' => $this->sessionId], // Pass raw id object - $this->txnNumber, - false + ['id' => $this->sessionId], + $this->txnNumber ); } catch (\Throwable $e) { throw new DatabaseException($e->getMessage(), $e->getCode(), $e); } - // Session is now closed by the client using endSessions, state is reseted - // TODO do we want session per transaction or to manage it on the connection level? + // Session is now closed by the client using endSessions, state is reset $this->sessionId = null; $this->txnNumber = null; @@ -194,7 +188,6 @@ public function commitTransaction(): bool public function rollbackTransaction(): bool { - try { if ($this->inTransaction === 0) { return false; @@ -208,8 +201,7 @@ public function rollbackTransaction(): bool try { $result = $this->client->abortTransaction( ['id' => $this->sessionId], // Pass raw id object - $this->txnNumber, - false + $this->txnNumber ); } catch (\Throwable $e) { throw new DatabaseException($e->getMessage(), $e->getCode(), $e); @@ -229,15 +221,15 @@ public function rollbackTransaction(): bool /** * Helper to add transaction/session context to command options if in transaction - * + * * @param array $options * @return array */ - private function addTransactionContext(array $options = []): array + private function getTransactionOptions(array $options = []): array { if ($this->inTransaction) { $options['lsid'] = ['id' => $this->sessionId]; - $options['txnNumber'] = new \MongoDB\BSON\Int64($this->txnNumber); + $options['txnNumber'] = new Int64($this->txnNumber); $options['autocommit'] = false; if ($this->firstOpInTransaction) { @@ -931,7 +923,7 @@ public function renameIndex(string $collection, string $old, string $new): bool try { $deletedindex = $this->deleteIndex($collection, $old); - $createdindex = $this->createIndex($collection, $new, $index['type'], $index['attributes'], $index['lengths'] ?? [], $index['orders'] ?? [], $indexAttributeTypes, []); + $createdindex = $this->createIndex($collection, $new, $index['type'], $index['attributes'], $index['lengths'] ?? [], $index['orders'] ?? [], $indexAttributeTypes); } catch (\Exception $e) { throw $this->processException($e); } @@ -1031,7 +1023,7 @@ public function createDocument(Document $collection, Document $document): Docume if (!empty($sequence)) { $record['_id'] = $sequence; } - $options = $this->addTransactionContext([]); + $options = $this->getTransactionOptions(); $result = $this->insertDocument($name, $this->removeNullKeys($record), $options); $result = $this->replaceChars('_', '$', $result); // in order to keep the original object refrence. @@ -1066,7 +1058,7 @@ public function castingAfter(Document $collection, Document $document): Document $key = $attribute['$id'] ?? ''; $type = $attribute['type'] ?? ''; $array = $attribute['array'] ?? false; - $value = $document->getAttribute($key, null); + $value = $document->getAttribute($key); if (is_null($value)) { continue; } @@ -1127,7 +1119,7 @@ public function castingBefore(Document $collection, Document $document): Documen $type = $attribute['type'] ?? ''; $array = $attribute['array'] ?? false; - $value = $document->getAttribute($key, null); + $value = $document->getAttribute($key); if (is_null($value)) { continue; } @@ -1172,7 +1164,7 @@ public function createDocuments(Document $collection, array $documents): array { $name = $this->getNamespace() . '_' . $this->filter($collection->getId()); - $options = $this->addTransactionContext([]); + $options = $this->getTransactionOptions(); $records = []; $hasSequence = null; $documents = \array_map(fn ($doc) => clone $doc, $documents); @@ -1271,7 +1263,7 @@ public function updateDocument(Document $collection, string $id, Document $docum try { unset($record['_id']); // Don't update _id - $options = $this->addTransactionContext([]); + $options = $this->getTransactionOptions(); $this->client->update($name, $filters, $record, $options); } catch (MongoException $e) { throw $this->processException($e); @@ -1298,7 +1290,7 @@ public function updateDocuments(Document $collection, Document $updates, array $ ; $name = $this->getNamespace() . '_' . $this->filter($collection->getId()); - $options = $this->addTransactionContext([]); + $options = $this->getTransactionOptions(); $queries = [ Query::equal('$sequence', \array_map(fn ($document) => $document->getSequence(), $documents)) ]; @@ -1402,7 +1394,7 @@ public function upsertDocuments(Document $collection, string $attribute, array $ ]; } - $options = $this->addTransactionContext([]); + $options = $this->getTransactionOptions(); $this->client->upsert( $name, @@ -1459,7 +1451,7 @@ public function getSequences(string $collection, array $documents): array 'batchSize' => self::DEFAULT_BATCH_SIZE ]; - $options = $this->addTransactionContext(['projection' => ['_uid' => 1, '_id' => 1]]); + $options = $this->getTransactionOptions(['projection' => ['_uid' => 1, '_id' => 1]]); $response = $this->client->find($name, $filters, $options); $results = $response->cursor->firstBatch ?? []; @@ -1532,7 +1524,7 @@ public function increaseDocumentAttribute(string $collection, string $id, string $filters[$attribute] = ['$gte' => $min]; } - $options = $this->addTransactionContext([]); + $options = $this->getTransactionOptions(); $this->client->update( $this->getNamespace() . '_' . $this->filter($collection), $filters, @@ -1566,7 +1558,7 @@ public function deleteDocument(string $collection, string $id): bool $filters['_tenant'] = $this->getTenantFilters($collection); } - $options = $this->addTransactionContext([]); + $options = $this->getTransactionOptions(); $result = $this->client->delete($name, $filters, 1, [], $options); return (!!$result); @@ -1596,14 +1588,13 @@ public function deleteDocuments(string $collection, array $sequences, array $per $filters = $this->replaceInternalIdsKeys($filters, '$', '_', $this->operators); - $options = $this->addTransactionContext([]); + $options = $this->getTransactionOptions(); try { $count = $this->client->delete( collection: $name, filters: $filters, limit: 0, - deleteOptions: [], options: $options ); } catch (MongoException $e) { @@ -1708,7 +1699,7 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 } // Add transaction context to options - $options = $this->addTransactionContext($options); + $options = $this->getTransactionOptions($options); $orFilters = []; @@ -1947,14 +1938,7 @@ public function count(Document $collection, array $queries = [], ?int $max = nul * https://www.mongodb.com/docs/manual/reference/command/count/#response **/ - // Original count command (commented for reference and fallback) - // Use this for single-instance MongoDB when performance is critical and accuracy is not a concern - - - - $options = $this->addTransactionContext([]); - // return $this->client->count($name, $filters, $options); - + $options = $this->getTransactionOptions(); $pipeline = []; // Add match stage if filters are provided @@ -2057,7 +2041,7 @@ public function sum(Document $collection, string $attribute, array $queries = [] ], ]; - $options = $this->addTransactionContext([]); + $options = $this->getTransactionOptions(); return $this->client->aggregate($name, $pipeline, $options)->cursor->firstBatch[0]->total ?? 0; } diff --git a/tests/resources/mongo/entrypoint.sh b/tests/resources/mongo/entrypoint.sh new file mode 100755 index 000000000..8119a4fa7 --- /dev/null +++ b/tests/resources/mongo/entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +# Fix keyfile permissions +if [ -f "/tmp/keyfile" ]; then + cp /tmp/keyfile /etc/mongo-keyfile + chmod 400 /etc/mongo-keyfile + chown mongodb:mongodb /etc/mongo-keyfile +fi + +# Use MongoDB's standard entrypoint with our command +exec docker-entrypoint.sh mongod --replSet rs0 --bind_ip_all --auth --keyFile /etc/mongo-keyfile \ No newline at end of file diff --git a/mongo-keyfile b/tests/resources/mongo/mongo-keyfile similarity index 100% rename from mongo-keyfile rename to tests/resources/mongo/mongo-keyfile From b05eb72a1dd93da76773a78f0dd85a466a72fe7b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 26 Sep 2025 21:52:09 +1200 Subject: [PATCH 22/22] Refactor name --- src/Database/Adapter.php | 8 ++++---- src/Database/Adapter/Mongo.php | 3 +-- src/Database/Adapter/Pool.php | 4 ++-- src/Database/Adapter/SQL.php | 2 +- src/Database/Database.php | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 930fc4584..172f7bd1b 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -1355,18 +1355,18 @@ abstract public function castingBefore(Document $collection, Document $document) abstract public function castingAfter(Document $collection, Document $document): Document; /** - * Is Mongo? + * Is internal casting supported? * * @return bool */ - abstract public function isMongo(): bool; + abstract public function getSupportForInternalCasting(): bool; /** - * Is internal casting supported? + * Is UTC casting supported? * * @return bool */ - abstract public function getSupportForInternalCasting(): bool; + abstract public function getSupportForUTCCasting(): bool; /** * Set UTC Datetime diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index d69b3d994..dc1ec1448 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -2411,8 +2411,7 @@ public function getSupportForInternalCasting(): bool return true; } - - public function isMongo(): bool + public function getSupportForUTCCasting(): bool { return true; } diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index 7da636fe1..f89c09873 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -580,12 +580,12 @@ public function castingAfter(Document $collection, Document $document): Document return $this->delegate(__FUNCTION__, \func_get_args()); } - public function isMongo(): bool + public function getSupportForInternalCasting(): bool { return $this->delegate(__FUNCTION__, \func_get_args()); } - public function getSupportForInternalCasting(): bool + public function getSupportForUTCCasting(): bool { return $this->delegate(__FUNCTION__, \func_get_args()); } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index d8387ec29..d8e21e5bf 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -1552,7 +1552,7 @@ public function getSupportForOrderRandom(): bool return true; } - public function isMongo(): bool + public function getSupportForUTCCasting(): bool { return false; } diff --git a/src/Database/Database.php b/src/Database/Database.php index cba41a844..d26af049e 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -7180,7 +7180,7 @@ public function convertQuery(Document $collection, Query $query): Query $values = $query->getValues(); foreach ($values as $valueIndex => $value) { try { - $values[$valueIndex] = $this->adapter->isMongo() + $values[$valueIndex] = $this->adapter->getSupportForUTCCasting() ? $this->adapter->setUTCDatetime($value) : DateTime::setTimezone($value); } catch (\Throwable $e) {