From 30a38c0c8ca2996bf4bc8cfdb7de97dc32264b83 Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 5 Nov 2023 19:53:50 +0200 Subject: [PATCH 01/45] Reproduce Many to One issue --- phpunit.xml | 2 +- src/Database/Database.php | 4 ++++ tests/Database/Base.php | 18 ++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/phpunit.xml b/phpunit.xml index 31b947dd6..3833748e0 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false"> + stopOnFailure="true"> ./tests/ diff --git a/src/Database/Database.php b/src/Database/Database.php index d8393ac8f..14f6e82bf 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -3550,6 +3550,10 @@ private function updateDocumentRelationships(Document $collection, Document $old $document->setAttribute($key, $value->getId()); } elseif (\is_null($value)) { break; + } elseif(empty($value)){ + var_dump('I am an empty array Is I think it is ok to throw an exeption'); + throw new DatabaseException('Invalid value for relationship'); + } else { throw new DatabaseException('Invalid value for relationship'); } diff --git a/tests/Database/Base.php b/tests/Database/Base.php index 31b0dc9df..4ec3f8566 100644 --- a/tests/Database/Base.php +++ b/tests/Database/Base.php @@ -6799,6 +6799,24 @@ public function testManyToOneOneWayRelationship(): void $review5 = static::getDatabase()->getDocument('review', 'review5'); $this->assertEquals('Movie 5', $review5->getAttribute('movie')->getAttribute('name')); + + // Todo: fix this is failing + static::getDatabase()->updateDocument('review', $review5->getId(), new Document([ + '$id' => 'review5', + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()), + ], + 'name' => 'Review 5', + 'movie' => [], // + ])); + + + $this->assertEquals(true, false); + + + // Update document with new related document static::getDatabase()->updateDocument( 'review', From 2d6a8c380d8eff01e62f122906f55d7d18c760e0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 7 Nov 2023 18:41:37 +1300 Subject: [PATCH 02/45] Add isolation mode constants and setters --- src/Database/Database.php | 54 ++++++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 14f6e82bf..0528120d5 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -212,6 +212,21 @@ class Database '$collection', ]; + /** + * Each database resource is created within its own schema. + */ + public const ISOLATION_MODE_SCHEMA = 'schema'; + + /** + * Each database resource is created within a shared schema + */ + public const ISOLATION_MODE_SHARED = 'shared'; + + /** + * Each database resource is created within shared tables in a shared schema. + */ + public const ISOLATION_MODE_TABLE = 'table'; + /** * Parent Collection * Defines the structure for both system and custom collections @@ -295,25 +310,27 @@ class Database protected ?\DateTime $timestamp = null; + protected string $isolationMode = self::ISOLATION_MODE_SHARED; + protected bool $resolveRelationships = true; - private int $relationshipFetchDepth = 1; + protected int $relationshipFetchDepth = 1; /** * Stack of collection IDs when creating or updating related documents * @var array */ - private array $relationshipWriteStack = []; + protected array $relationshipWriteStack = []; /** * @var array */ - private array $relationshipFetchStack = []; + protected array $relationshipFetchStack = []; /** * @var array */ - private array $relationshipDeleteStack = []; + protected array $relationshipDeleteStack = []; /** * @param Adapter $adapter @@ -624,12 +641,14 @@ public function resetMetadata(): void * * @param int $milliseconds * @param string $event - * @return void + * @return self * @throws Exception */ - public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): void + public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): self { $this->adapter->setTimeout($milliseconds, $event); + + return $this; } /** @@ -643,6 +662,24 @@ public function clearTimeout(string $event = Database::EVENT_ALL): void $this->adapter->clearTimeout($event); } + public function setIsolationMode(string $mode): self + { + if (!\in_array($mode, [ + self::ISOLATION_MODE_SCHEMA, + self::ISOLATION_MODE_SHARED, + self::ISOLATION_MODE_TABLE + ])) { + throw new DatabaseException('Invalid isolation mode'); + } + + return $this; + } + + public function getIsolationMode(): string + { + return $this->isolationMode; + } + /** * Ping Database * @@ -3550,9 +3587,8 @@ private function updateDocumentRelationships(Document $collection, Document $old $document->setAttribute($key, $value->getId()); } elseif (\is_null($value)) { break; - } elseif(empty($value)){ - var_dump('I am an empty array Is I think it is ok to throw an exeption'); - throw new DatabaseException('Invalid value for relationship'); + } elseif(empty($value)) { + throw new DatabaseException('Invalid value for relationship'); } else { throw new DatabaseException('Invalid value for relationship'); From ccff2ef5879c0d6cc3ed2162718836f479415e6b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 10 Nov 2023 19:06:04 +1300 Subject: [PATCH 03/45] Add tenant insertion + querying --- README.md | 4 +- bin/tasks/index.php | 2 +- bin/tasks/load.php | 16 +-- bin/tasks/query.php | 6 +- src/Database/Adapter.php | 25 +--- src/Database/Adapter/MariaDB.php | 100 +++++++------- src/Database/Adapter/Mongo.php | 67 +--------- src/Database/Adapter/MySQL.php | 4 +- src/Database/Adapter/Postgres.php | 13 +- src/Database/Adapter/SQL.php | 48 +++---- src/Database/Adapter/SQLite.php | 2 +- src/Database/Database.php | 129 ++++++++++++++----- src/Database/Document.php | 11 +- src/Database/Validator/Queries/Document.php | 10 +- src/Database/Validator/Queries/Documents.php | 6 + src/Database/Validator/Structure.php | 10 ++ tests/Database/Adapter/MariaDBTest.php | 6 +- tests/Database/Adapter/MongoDBTest.php | 4 +- tests/Database/Adapter/MySQLTest.php | 2 +- tests/Database/Adapter/PostgresTest.php | 2 +- tests/Database/Adapter/SQLiteTest.php | 2 +- tests/Database/Base.php | 85 +++++++++++- 22 files changed, 322 insertions(+), 232 deletions(-) diff --git a/README.md b/README.md index f9792d101..9c6b5bf55 100644 --- a/README.md +++ b/README.md @@ -244,10 +244,10 @@ $database->setNamespace( ); // Get default database -$database->getDefaultDatabase(); +$database->getDatabase(); // Sets default database -$database->setDefaultDatabase( +$database->setDatabase( name: 'dbName' ); diff --git a/bin/tasks/index.php b/bin/tasks/index.php index 66c919704..2ef8c4ef3 100644 --- a/bin/tasks/index.php +++ b/bin/tasks/index.php @@ -71,7 +71,7 @@ return; } - $database->setDefaultDatabase($name); + $database->setDatabase($name); $database->setNamespace($namespace); Console::info("For query: greaterThan(created, 2010-01-01 05:00:00)', 'equal(genre,travel)"); diff --git a/bin/tasks/load.php b/bin/tasks/load.php index 9362b100d..4658eadd6 100644 --- a/bin/tasks/load.php +++ b/bin/tasks/load.php @@ -56,7 +56,7 @@ $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MariaDB::getPDOAttributes()); $database = new Database(new MariaDB($pdo), $cache); - $database->setDefaultDatabase($name); + $database->setDatabase($name); $database->setNamespace($namespace); // Outline collection schema @@ -89,7 +89,7 @@ $pdo = $pool->get(); $database = new Database(new MariaDB($pdo), $cache); - $database->setDefaultDatabase($name); + $database->setDatabase($name); $database->setNamespace($namespace); // Each coroutine loads 1000 documents @@ -116,7 +116,7 @@ $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MySQL::getPDOAttributes()); $database = new Database(new MySQL($pdo), $cache); - $database->setDefaultDatabase($name); + $database->setDatabase($name); $database->setNamespace($namespace); // Outline collection schema @@ -150,7 +150,7 @@ $pdo = $pool->get(); $database = new Database(new MySQL($pdo), $cache); - $database->setDefaultDatabase($name); + $database->setDatabase($name); $database->setNamespace($namespace); // Each coroutine loads 1000 documents @@ -178,7 +178,7 @@ ); $database = new Database(new Mongo($client), $cache); - $database->setDefaultDatabase($name); + $database->setDatabase($name); $database->setNamespace($namespace); // Outline collection schema @@ -192,7 +192,7 @@ for ($i = 0; $i < $limit / 1000; $i++) { go(function () use ($client, $faker, $name, $namespace, $cache) { $database = new Database(new Mongo($client), $cache); - $database->setDefaultDatabase($name); + $database->setDatabase($name); $database->setNamespace($namespace); // Each coroutine loads 1000 documents @@ -227,8 +227,8 @@ function createSchema(Database $database): void { - if ($database->exists($database->getDefaultDatabase())) { - $database->delete($database->getDefaultDatabase()); + if ($database->exists($database->getDatabase())) { + $database->delete($database->getDatabase()); } $database->create(); diff --git a/bin/tasks/query.php b/bin/tasks/query.php index ab7115563..2fcd29143 100644 --- a/bin/tasks/query.php +++ b/bin/tasks/query.php @@ -45,7 +45,7 @@ ); $database = new Database(new Mongo($client), $cache); - $database->setDefaultDatabase($name); + $database->setDatabase($name); $database->setNamespace($namespace); break; @@ -58,7 +58,7 @@ $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MariaDB::getPDOAttributes()); $database = new Database(new MariaDB($pdo), $cache); - $database->setDefaultDatabase($name); + $database->setDatabase($name); $database->setNamespace($namespace); break; @@ -71,7 +71,7 @@ $pdo = new PDO("mysql:host={$dbHost};port={$dbPort};charset=utf8mb4", $dbUser, $dbPass, MySQL::getPDOAttributes()); $database = new Database(new MySQL($pdo), $cache); - $database->setDefaultDatabase($name); + $database->setDatabase($name); $database->setNamespace($namespace); break; diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index fa59aecd3..9dcb394b2 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -78,10 +78,6 @@ public function resetDebug(): self */ public function setNamespace(string $namespace): bool { - if (empty($namespace)) { - throw new DatabaseException('Missing namespace'); - } - $this->namespace = $this->filter($namespace); return true; @@ -98,10 +94,6 @@ public function setNamespace(string $namespace): bool */ public function getNamespace(): string { - if (empty($this->namespace)) { - throw new DatabaseException('Missing namespace'); - } - return $this->namespace; } @@ -111,18 +103,13 @@ public function getNamespace(): string * Set database to use for current scope * * @param string $name - * @param bool $reset * * @return bool - * @throws Exception + * @throws DatabaseException */ - public function setDefaultDatabase(string $name, bool $reset = false): bool + public function setDatabase(string $name): bool { - if (empty($name) && $reset === false) { - throw new DatabaseException('Missing database'); - } - - $this->defaultDatabase = ($reset) ? '' : $this->filter($name); + $this->defaultDatabase = $this->filter($name); return true; } @@ -133,10 +120,10 @@ public function setDefaultDatabase(string $name, bool $reset = false): bool * Get Database from current scope * * @return string - * @throws Exception + * @throws DatabaseException * */ - public function getDefaultDatabase(): string + public function getDatabase(): string { if (empty($this->defaultDatabase)) { throw new DatabaseException('Missing default database'); @@ -709,7 +696,7 @@ protected function getAttributeSelections(array $queries): array * * @param string $value * @return string - * @throws Exception + * @throws DatabaseException */ public function filter(string $value): string { diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 05d66ef60..0e8e66465 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -69,8 +69,6 @@ public function delete(string $name): bool */ public function createCollection(string $name, array $attributes = [], array $indexes = []): bool { - $database = $this->getDefaultDatabase(); - $namespace = $this->getNamespace(); $id = $this->filter($name); /** @var array $attributeStrings */ @@ -112,9 +110,10 @@ public function createCollection(string $name, array $attributes = [], array $in } $sql = " - CREATE TABLE IF NOT EXISTS `{$database}`.`{$namespace}_{$id}` ( + CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id)} ( `_id` int(11) unsigned NOT NULL AUTO_INCREMENT, `_uid` VARCHAR(255) NOT NULL, + `_tenant` CHAR(36) NOT NULL, `_createdAt` datetime(3) DEFAULT NULL, `_updatedAt` datetime(3) DEFAULT NULL, `_permissions` MEDIUMTEXT DEFAULT NULL, @@ -122,8 +121,10 @@ public function createCollection(string $name, array $attributes = [], array $in PRIMARY KEY (`_id`), " . \implode(' ', $indexStrings) . " UNIQUE KEY `_uid` (`_uid`), - KEY `_created_at` (`_createdAt`), - KEY `_updated_at` (`_updatedAt`) + KEY `_tenant` (`_tenant`), + KEY `_created_at` (`_createdAt`, `_tenant`), + KEY `_updated_at` (`_updatedAt`, `_tenant`), + KEY `_uid_tenant` (`_uid`, `_tenant`) ) "; @@ -135,14 +136,15 @@ public function createCollection(string $name, array $attributes = [], array $in ->execute(); $sql = " - CREATE TABLE IF NOT EXISTS `{$database}`.`{$namespace}_{$id}_perms` ( + CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id . '_perms')} ( `_id` int(11) unsigned NOT NULL AUTO_INCREMENT, + `_tenant` CHAR(36) NOT NULL, `_type` VARCHAR(12) NOT NULL, `_permission` VARCHAR(255) NOT NULL, `_document` VARCHAR(255) NOT NULL, PRIMARY KEY (`_id`), - UNIQUE INDEX `_index1` (`_document`,`_type`,`_permission`), - INDEX `_permission` (`_permission`,`_type`,`_document`) + UNIQUE INDEX `_index1` (`_document`, `_tenant`, `_type`,`_permission`), + INDEX `_permission` (`_permission`, `_type`, `_tenant`, `_document`) ) "; @@ -162,7 +164,8 @@ public function createCollection(string $name, array $attributes = [], array $in } /** - * Get Collection Size + * Get collection size + * * @param string $collection * @return int * @throws DatabaseException @@ -171,7 +174,7 @@ public function getSizeOfCollection(string $collection): int { $collection = $this->filter($collection); $collection = $this->getNamespace() . '_' . $collection; - $database = $this->getDefaultDatabase(); + $database = $this->getDatabase(); $name = $database . '/' . $collection; $permissions = $database . '/' . $collection . '_perms'; @@ -202,7 +205,8 @@ public function getSizeOfCollection(string $collection): int } /** - * Delete Collection + * Delete collection + * * @param string $id * @return bool * @throws Exception @@ -341,7 +345,7 @@ public function renameAttribute(string $collection, string $old, string $new): b * @param bool $twoWay * @param string $twoWayKey * @return bool - * @throws Exception + * @throws DatabaseException */ public function createRelationship( string $collection, @@ -471,6 +475,17 @@ public function updateRelationship( ->execute(); } + /** + * @param string $collection + * @param string $relatedCollection + * @param string $type + * @param bool $twoWay + * @param string $key + * @param string $twoWayKey + * @param string $side + * @return bool + * @throws DatabaseException + */ public function deleteRelationship( string $collection, string $relatedCollection, @@ -646,9 +661,10 @@ public function deleteIndex(string $collection, string $id): bool public function createDocument(string $collection, Document $document): Document { $attributes = $document->getAttributes(); + $attributes['_tenant'] = $document->getAttribute('$tenant'); $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); - $attributes['_permissions'] = json_encode($document->getPermissions()); + $attributes['_permissions'] = \json_encode($document->getPermissions()); $name = $this->filter($collection); $columns = ''; @@ -660,11 +676,8 @@ public function createDocument(string $collection, Document $document): Document $this->getPDO()->rollBack(); } - /** - * Insert Attributes - */ $bindIndex = 0; - foreach ($attributes as $attribute => $value) { // Parse statement + foreach ($attributes as $attribute => $value) { $column = $this->filter($attribute); $bindKey = 'key_' . $bindIndex; $columns .= "`{$column}`, "; @@ -672,7 +685,7 @@ public function createDocument(string $collection, Document $document): Document $bindIndex++; } - // Insert manual id if set + // Insert internal ID if set if (!empty($document->getInternalId())) { $bindKey = '_id'; $columns .= "_id, "; @@ -690,19 +703,17 @@ public function createDocument(string $collection, Document $document): Document $stmt->bindValue(':_uid', $document->getId(), PDO::PARAM_STR); - // Bind manual internal id if set if (!empty($document->getInternalId())) { $stmt->bindValue(':_id', $document->getInternalId(), PDO::PARAM_STR); } $attributeIndex = 0; - foreach ($attributes as $attribute => $value) { - if (is_array($value)) { // arrays & objects should be saved as strings + foreach ($attributes as $value) { + if (is_array($value)) { $value = json_encode($value); } $bindKey = 'key_' . $attributeIndex; - $attribute = $this->filter($attribute); $value = (is_bool($value)) ? (int)$value : $value; $stmt->bindValue(':' . $bindKey, $value, $this->getPDOType($value)); $attributeIndex++; @@ -712,7 +723,7 @@ public function createDocument(string $collection, Document $document): Document foreach (Database::PERMISSIONS as $type) { foreach ($document->getPermissionsByType($type) as $permission) { $permission = \str_replace('"', '', $permission); - $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}')"; + $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}', '{$document->getAttribute('$tenant')}')"; } } @@ -720,7 +731,7 @@ public function createDocument(string $collection, Document $document): Document $strPermissions = \implode(', ', $permissions); $sqlPermissions = " - INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document) + INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document, _tenant) VALUES {$strPermissions} "; $sqlPermissions = $this->trigger(Database::EVENT_PERMISSIONS_CREATE, $sqlPermissions); @@ -737,14 +748,11 @@ public function createDocument(string $collection, Document $document): Document } } catch (PDOException $e) { $this->getPDO()->rollBack(); - switch ($e->getCode()) { - case 1062: - case 23000: - throw new DuplicateException('Duplicated document: ' . $e->getMessage()); - - default: - throw $e; - } + throw match ($e->getCode()) { + 1062, + 23000 => new DuplicateException('Duplicated document: ' . $e->getMessage()), + default => $e, + }; } if (!$this->getPDO()->commit()) { @@ -842,7 +850,7 @@ public function createDocuments(string $collection, array $documents, int $batch } if (!$this->getPDO()->commit()) { - throw new Exception('Failed to commit transaction'); + throw new DatabaseException('Failed to commit transaction'); } return $documents; @@ -1002,7 +1010,6 @@ public function updateDocument(string $collection, Document $document): Document /** * Update Attributes */ - $bindIndex = 0; foreach ($attributes as $attribute => $value) { $column = $this->filter($attribute); @@ -1025,7 +1032,7 @@ public function updateDocument(string $collection, Document $document): Document $attributeIndex = 0; foreach ($attributes as $attribute => $value) { - if (is_array($value)) { // arrays & objects should be saved as strings + if (is_array($value)) { $value = json_encode($value); } @@ -1047,14 +1054,11 @@ public function updateDocument(string $collection, Document $document): Document } } catch (PDOException $e) { $this->getPDO()->rollBack(); - switch ($e->getCode()) { - case 1062: - case 23000: - throw new DuplicateException('Duplicated document: ' . $e->getMessage()); - - default: - throw $e; - } + throw match ($e->getCode()) { + 1062, + 23000 => new DuplicateException('Duplicated document: ' . $e->getMessage()), + default => $e, + }; } if (!$this->getPDO()->commit()) { @@ -1265,7 +1269,7 @@ public function updateDocuments(string $collection, array $documents, int $batch } if (!$this->getPDO()->commit()) { - throw new Exception('Failed to commit transaction'); + throw new DatabaseException('Failed to commit transaction'); } return $documents; @@ -1273,7 +1277,8 @@ public function updateDocuments(string $collection, array $documents, int $batch $this->getPDO()->rollBack(); throw match ($e->getCode()) { - 1062, 23000 => new DuplicateException('Duplicated document: ' . $e->getMessage()), + 1062, + 23000 => new DuplicateException('Duplicated document: ' . $e->getMessage()), default => $e, }; } @@ -1757,6 +1762,7 @@ protected function getSQLCondition(Query $query): string $query->setAttribute(match ($query->getAttribute()) { '$id' => '_uid', '$internalId' => '_id', + '$tenant' => '_tenant', '$createdAt' => '_createdAt', '$updatedAt' => '_updatedAt', default => $query->getAttribute() @@ -1861,7 +1867,9 @@ protected function getSQLIndex(string $collection, string $id, string $type, arr default => throw new DatabaseException('Unknown index type: ' . $type . '. Must be one of ' . Database::INDEX_KEY . ', ' . Database::INDEX_UNIQUE . ', ' . Database::INDEX_ARRAY . ', ' . Database::INDEX_FULLTEXT), }; - return "CREATE {$type} `{$id}` ON {$this->getSQLTable($collection)} ( " . implode(', ', $attributes) . " )"; + $attributes = \implode(', ', $attributes); + + return "CREATE {$type} `{$id}` ON {$this->getSQLTable($collection)} ({$attributes}, _tenant)"; } /** diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 451913461..01479c145 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -612,8 +612,7 @@ public function deleteIndex(string $collection, string $id): bool { $name = $this->getNamespace() . '_' . $this->filter($collection); $id = $this->filter($id); - $collection = $this->getDatabase(); - $collection->dropIndexes($name, [$id]); + $this->getClient()->dropIndexes($name, [$id]); return true; } @@ -1219,17 +1218,6 @@ public function sum(string $collection, string $attribute, array $queries = [], return $this->client->aggregate($name, $pipeline)->cursor->firstBatch[0]->total ?? 0; } - /** - * @param string|null $name - * @return Client - * - * @throws Exception - */ - protected function getDatabase(string $name = null): Client - { - return $this->getClient()->selectDatabase(); - } - /** * @return Client * @@ -1647,58 +1635,6 @@ public function getSupportForCasting(): bool return true; } - /** - * Return set namespace. - * - * @return string - * @throws Exception - */ - public function getNamespace(): string - { - if (empty($this->namespace)) { - throw new DatabaseException('Missing namespace'); - } - - return $this->namespace; - } - - /** - * Set's default database. - * - * @param string $name - * @param bool $reset - * @return bool - * @throws Exception - */ - public function setDefaultDatabase(string $name, bool $reset = false): bool - { - if (empty($name) && $reset === false) { - throw new DatabaseException('Missing database'); - } - - $this->defaultDatabase = ($reset) ? '' : $this->filter($name); - - return true; - } - - /** - * Set's the namespace. - * - * @param string $namespace - * @return bool - * @throws Exception - */ - public function setNamespace(string $namespace): bool - { - if (empty($namespace)) { - throw new DatabaseException('Missing namespace'); - } - - $this->namespace = $this->filter($namespace); - - return true; - } - /** * Flattens the array. * @@ -1773,6 +1709,7 @@ public function setTimeout(int $milliseconds, string $event = Database::EVENT_AL if (!$this->getSupportForTimeouts()) { return; } + $this->timeout = $milliseconds; } diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php index da6bf6dd7..23d98cadc 100644 --- a/src/Database/Adapter/MySQL.php +++ b/src/Database/Adapter/MySQL.php @@ -47,7 +47,7 @@ protected function getSQLIndex(string $collection, string $id, string $type, arr throw new DatabaseException('Unknown index type: ' . $type . '. Must be one of ' . Database::INDEX_KEY . ', ' . Database::INDEX_UNIQUE . ', ' . Database::INDEX_ARRAY . ', ' . Database::INDEX_FULLTEXT); } - return 'CREATE '.$type.' `'.$id.'` ON `'.$this->getDefaultDatabase().'`.`'.$this->getNamespace().'_'.$collection.'` ( '.implode(', ', $attributes).' );'; + return 'CREATE '.$type.' `'.$id.'` ON `'.$this->getDatabase().'`.`'.$this->getNamespace().'_'.$collection.'` ( '.implode(', ', $attributes).' );'; } /** @@ -103,7 +103,7 @@ public function getSizeOfCollection(string $collection): int { $collection = $this->filter($collection); $collection = $this->getNamespace() . '_' . $collection; - $database = $this->getDefaultDatabase(); + $database = $this->getDatabase(); $name = $database . '/' . $collection; $permissions = $database . '/' . $collection . '_perms'; diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 6b9e3795c..eb137918c 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -74,7 +74,6 @@ public function delete(string $name): bool */ public function createCollection(string $name, array $attributes = [], array $indexes = []): bool { - $database = $this->getDefaultDatabase(); $namespace = $this->getNamespace(); $id = $this->filter($name); @@ -98,6 +97,7 @@ public function createCollection(string $name, array $attributes = [], array $in CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id)} ( \"_id\" SERIAL NOT NULL, \"_uid\" VARCHAR(255) NOT NULL, + \"_tenant\" SERIAL NOT NULL, \"_createdAt\" TIMESTAMP(3) DEFAULT NULL, \"_updatedAt\" TIMESTAMP(3) DEFAULT NULL, \"_permissions\" TEXT DEFAULT NULL, @@ -120,6 +120,7 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = " CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id . '_perms')} ( \"_id\" SERIAL NOT NULL, + \"_tenant\" SERIAL NOT NULL, \"_type\" VARCHAR(12) NOT NULL, \"_permission\" VARCHAR(255) NOT NULL, \"_document\" VARCHAR(255) NOT NULL, @@ -604,7 +605,7 @@ public function deleteIndex(string $collection, string $id): bool { $name = $this->filter($collection); $id = $this->filter($id); - $schemaName = $this->getDefaultDatabase(); + $schemaName = $this->getDatabase(); $sql = "DROP INDEX IF EXISTS \"{$schemaName}\".{$id}"; $sql = $this->trigger(Database::EVENT_INDEX_DELETE, $sql); @@ -844,7 +845,7 @@ public function createDocuments(string $collection, array $documents, int $batch } if (!$this->getPDO()->commit()) { - throw new Exception('Failed to commit transaction'); + throw new DatabaseException('Failed to commit transaction'); } return $documents; @@ -1256,7 +1257,7 @@ public function updateDocuments(string $collection, array $documents, int $batch } if (!$this->getPDO()->commit()) { - throw new Exception('Failed to commit transaction'); + throw new DatabaseException('Failed to commit transaction'); } return $documents; @@ -1858,7 +1859,7 @@ protected function getSQLSchema(): string return ''; } - return "\"{$this->getDefaultDatabase()}\"."; + return "\"{$this->getDatabase()}\"."; } /** @@ -1869,7 +1870,7 @@ protected function getSQLSchema(): string */ protected function getSQLTable(string $name): string { - return "\"{$this->getDefaultDatabase()}\".\"{$this->getNamespace()}_{$name}\""; + return "\"{$this->getDatabase()}\".\"{$this->getNamespace()}_{$name}\""; } /** diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 39729e8b6..bddff52c2 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -68,10 +68,11 @@ public function exists(string $database, ?string $collection): bool $match = $database; } - $stmt = $this->getPDO() - ->prepare("SELECT {$select} - FROM {$from} - WHERE {$where};"); + $stmt = $this->getPDO()->prepare(" + SELECT {$select} + FROM {$from} + WHERE {$where} + "); $stmt->bindValue(':schema', $database, PDO::PARAM_STR); @@ -83,11 +84,12 @@ public function exists(string $database, ?string $collection): bool $document = $stmt->fetchAll(); $stmt->closeCursor(); + if (!empty($document)) { $document = $document[0]; } - return (($document[$select] ?? '') === $match) || // case insensitive check + return (($document[$select] ?? '') === $match) || // case-insensitive check (($document[strtolower($select)] ?? '') === $match); } @@ -113,16 +115,32 @@ public function list(): array public function getDocument(string $collection, string $id, array $queries = []): Document { $name = $this->filter($collection); - $selections = $this->getAttributeSelections($queries); + $filters = Query::groupByType($queries)['filters']; + + $tenantQuery = null; + $tenantWhere = ''; + + foreach ($filters as $query) { + if ($query->getAttribute() === '$tenant') { + $tenantQuery = $query; + $tenantWhere = 'AND _tenant = :_tenant'; + } + } $stmt = $this->getPDO()->prepare(" SELECT {$this->getAttributeProjection($selections)} FROM {$this->getSQLTable($name)} - WHERE _uid = :_uid; + WHERE _uid = :_uid + {$tenantWhere} "); $stmt->bindValue(':_uid', $id); + + if (!empty($tenantQuery)) { + $stmt->bindValue(':_tenant', $tenantQuery->getValue()); + } + $stmt->execute(); $document = $stmt->fetchAll(); @@ -851,20 +869,6 @@ protected function getSQLPermissionsCondition(string $collection, array $roles): )"; } - /** - * Get SQL schema - * - * @return string - */ - protected function getSQLSchema(): string - { - if (!$this->getSupportForSchemas()) { - return ''; - } - - return "`{$this->getDefaultDatabase()}`."; - } - /** * Get SQL table * @@ -873,7 +877,7 @@ protected function getSQLSchema(): string */ protected function getSQLTable(string $name): string { - return "{$this->getSQLSchema()}`{$this->getNamespace()}_{$name}`"; + return "`{$this->getDatabase()}`.`{$this->getNamespace()}_{$name}`"; } /** diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index c4afe6810..94ea465f5 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -931,7 +931,7 @@ public function updateDocuments(string $collection, array $documents, int $batch } if (!$this->getPDO()->commit()) { - throw new Exception('Failed to commit transaction'); + throw new DatabaseException('Failed to commit transaction'); } return $documents; diff --git a/src/Database/Database.php b/src/Database/Database.php index 0528120d5..8db4d34f0 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -142,20 +142,11 @@ class Database protected array $map = []; /** - * @var array - */ - protected array $primitives = [ - self::VAR_STRING => true, - self::VAR_INTEGER => true, - self::VAR_FLOAT => true, - self::VAR_BOOLEAN => true, - ]; - - /** - * List of Internal Ids + * List of Internal attributes + * * @var array> */ - protected static array $attributes = [ + protected const ATTRIBUTES = [ [ '$id' => '$id', 'type' => self::VAR_STRING, @@ -174,6 +165,16 @@ class Database 'array' => false, 'filters' => [], ], + [ + '$id' => '$tenant', + 'type' => self::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => false, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], [ '$id' => '$createdAt', 'type' => Database::VAR_DATETIME, @@ -195,7 +196,7 @@ class Database 'default' => null, 'array' => false, 'filters' => ['datetime'] - ] + ], ]; /** @@ -206,10 +207,11 @@ class Database public const INTERNAL_ATTRIBUTES = [ '$id', '$internalId', + '$collection', + '$tenant', '$createdAt', '$updatedAt', '$permissions', - '$collection', ]; /** @@ -312,6 +314,8 @@ class Database protected string $isolationMode = self::ISOLATION_MODE_SHARED; + protected string $tenant = ''; + protected bool $resolveRelationships = true; protected int $relationshipFetchDepth = 1; @@ -578,14 +582,15 @@ public function getNamespace(): string * Set database to use for current scope * * @param string $name - * @param bool $reset * - * @return bool + * @return self * @throws Exception */ - public function setDefaultDatabase(string $name, bool $reset = false): bool + public function setDatabase(string $name): self { - return $this->adapter->setDefaultDatabase($name, $reset); + $this->adapter->setDatabase($name); + + return $this; } /** @@ -597,9 +602,9 @@ public function setDefaultDatabase(string $name, bool $reset = false): bool * * @return string */ - public function getDefaultDatabase(): string + public function getDatabase(): string { - return $this->adapter->getDefaultDatabase(); + return $this->adapter->getDatabase(); } /** @@ -662,22 +667,33 @@ public function clearTimeout(string $event = Database::EVENT_ALL): void $this->adapter->clearTimeout($event); } + /** + * Set the isolation mode used when creating a new database resource + * + * @param string $mode + * @return $this + * @throws DatabaseException + */ public function setIsolationMode(string $mode): self { if (!\in_array($mode, [ - self::ISOLATION_MODE_SCHEMA, - self::ISOLATION_MODE_SHARED, - self::ISOLATION_MODE_TABLE - ])) { + self::ISOLATION_MODE_SCHEMA, + self::ISOLATION_MODE_SHARED, + self::ISOLATION_MODE_TABLE + ])) { throw new DatabaseException('Invalid isolation mode'); } + $this->isolationMode = $mode; + return $this; } - public function getIsolationMode(): string + public function setTenant(string $tenant): self { - return $this->isolationMode; + $this->tenant = $tenant; + + return $this; } /** @@ -699,7 +715,7 @@ public function ping(): bool */ public function create(): bool { - $name = $this->adapter->getDefaultDatabase(); + $name = $this->adapter->getDatabase(); $this->adapter->create($name); /** @@ -2288,6 +2304,13 @@ public function deleteIndex(string $collection, string $id): bool */ public function getDocument(string $collection, string $id, array $queries = []): Document { + if ($this->isolationMode === self::ISOLATION_MODE_TABLE) { + if (empty($this->tenant)) { + throw new DatabaseException('Missing tenant. Tenant must be set when isolation mode is set to "table".'); + } + $queries[] = Query::equal('$tenant', [$this->tenant]); + } + if ($collection === self::METADATA && $id === self::METADATA) { return new Document($this->collection); } @@ -2713,6 +2736,13 @@ private function populateDocumentRelationships(Document $collection, Document $d */ public function createDocument(string $collection, Document $document): Document { + if ($this->isolationMode === self::ISOLATION_MODE_TABLE) { + if (empty($this->tenant)) { + throw new DatabaseException('Missing tenant. Tenant must be set when isolation mode is set to "table".'); + } + $document->setAttribute('$tenant', $this->tenant); + } + $collection = $this->silent(fn () => $this->getCollection($collection)); if ($collection->getId() !== self::METADATA) { @@ -2783,6 +2813,12 @@ public function createDocuments(string $collection, array $documents, int $batch $time = DateTime::now(); foreach ($documents as $key => $document) { + if ($this->isolationMode === self::ISOLATION_MODE_TABLE) { + if (empty($document->getAttribute('$tenant'))) { + throw new DatabaseException('Missing tenant. Tenant must be included when isolation mode is set to "table".'); + } + } + $document ->setAttribute('$id', empty($document->getId()) ? ID::unique() : $document->getId()) ->setAttribute('$collection', $collection->getId()) @@ -3101,6 +3137,12 @@ private function relateDocumentsById( */ public function updateDocument(string $collection, string $id, Document $document): Document { + if ($this->isolationMode === self::ISOLATION_MODE_TABLE) { + if (empty($document->getAttribute('$tenant'))) { + throw new DatabaseException('Missing tenant. Tenant must be included when isolation mode is set to "table".'); + } + } + if (!$document->getId() || !$id) { throw new DatabaseException('Must define $id attribute'); } @@ -3109,6 +3151,7 @@ public function updateDocument(string $collection, string $id, Document $documen $old = Authorization::skip(fn () => $this->silent(fn () => $this->getDocument($collection, $id))); // Skip ensures user does not need read permission for this $document = \array_merge($old->getArrayCopy(), $document->getArrayCopy()); $document['$collection'] = $old->getAttribute('$collection'); // Make sure user doesn't switch collectionID + $document['$tenant'] = $old->getAttribute('$tenant'); // Make sure user doesn't switch tenant $document['$createdAt'] = $old->getCreatedAt(); // Make sure user doesn't switch createdAt $document = new Document($document); @@ -3274,12 +3317,15 @@ public function updateDocuments(string $collection, array $documents, int $batch $collection = $this->silent(fn () => $this->getCollection($collection)); foreach ($documents as $document) { - if (!$document->getId()) { - throw new Exception('Must define $id attribute for each document'); + if ($this->isolationMode === self::ISOLATION_MODE_TABLE) { + if (empty($document->getAttribute('$tenant'))) { + throw new DatabaseException('Missing tenant. Tenant must be included when isolation mode is set to "table".'); + } } - $document->setAttribute('$updatedAt', $time); - $document = $this->encode($collection, $document); + if (!$document->getId()) { + throw new DatabaseException('Must define $id attribute for each document'); + } $old = Authorization::skip(fn () => $this->silent( fn () => $this->getDocument( @@ -3288,6 +3334,9 @@ public function updateDocuments(string $collection, array $documents, int $batch ) )); + $document->setAttribute('$updatedAt', $time); + $document = $this->encode($collection, $document); + $validator = new Authorization(self::PERMISSION_UPDATE); if ($collection->getId() !== self::METADATA && !$validator->isValid($old->getUpdate())) { @@ -3844,7 +3893,14 @@ public function decreaseDocumentAttribute(string $collection, string $id, string */ public function deleteDocument(string $collection, string $id): bool { - $document = Authorization::skip(fn () => $this->silent(fn () => $this->getDocument($collection, $id))); // Skip ensures user does not need read permission for this + $document = Authorization::skip(fn () => $this->silent(fn () => $this->getDocument($collection, $id))); + + if ($this->isolationMode === self::ISOLATION_MODE_TABLE) { + if (empty($document->getAttribute('$tenant'))) { + throw new DatabaseException('Missing tenant. Tenant must be included when isolation mode is set to "table".'); + } + } + $collection = $this->silent(fn () => $this->getCollection($collection)); $validator = new Authorization(self::PERMISSION_DELETE); @@ -4290,6 +4346,13 @@ public function deleteCachedDocument(string $collection, string $id): bool */ public function find(string $collection, array $queries = []): array { + if ($this->isolationMode === self::ISOLATION_MODE_TABLE) { + if (empty($this->tenant)) { + throw new DatabaseException('Missing tenant. Tenant must be set when isolation mode is set to "table".'); + } + $queries[] = Query::equal('$tenant', [$this->tenant]); + } + $originalName = $collection; $collection = $this->silent(fn () => $this->getCollection($collection)); @@ -4554,7 +4617,7 @@ public static function addFilter(string $name, callable $encode, callable $decod public static function getInternalAttributes(): array { $attributes = []; - foreach (self::$attributes as $internal) { + foreach (Database::ATTRIBUTES as $internal) { $attributes[] = new Document($internal); } return $attributes; diff --git a/src/Database/Document.php b/src/Database/Document.php index 6311841b4..ba7846bf6 100644 --- a/src/Database/Document.php +++ b/src/Database/Document.php @@ -4,8 +4,6 @@ use ArrayObject; use Utopia\Database\Exception as DatabaseException; -use Utopia\Database\Helpers\Permission; -use Utopia\Database\Helpers\Role; /** * @extends ArrayObject @@ -169,14 +167,7 @@ public function getAttributes(): array $attributes = []; foreach ($this as $attribute => $value) { - if (\array_key_exists($attribute, [ - '$id' => true, - '$internalId' => true, - '$collection' => true, - '$permissions' => [Permission::read(Role::any())], - '$createdAt' => true, - '$updatedAt' => true, - ])) { + if (\in_array($attribute, Database::INTERNAL_ATTRIBUTES)) { continue; } diff --git a/src/Database/Validator/Queries/Document.php b/src/Database/Validator/Queries/Document.php index fbb164d9d..91f57a0f0 100644 --- a/src/Database/Validator/Queries/Document.php +++ b/src/Database/Validator/Queries/Document.php @@ -5,6 +5,7 @@ use Exception; use Utopia\Database\Database; use Utopia\Database\Validator\Queries; +use Utopia\Database\Validator\Query\Filter; use Utopia\Database\Validator\Query\Select; class Document extends Queries @@ -21,14 +22,18 @@ public function __construct(array $attributes) 'type' => Database::VAR_STRING, 'array' => false, ]); - + $attributes[] = new \Utopia\Database\Document([ + '$id' => '$tenant', + 'key' => '$tenant', + 'type' => Database::VAR_DATETIME, + 'array' => false, + ]); $attributes[] = new \Utopia\Database\Document([ '$id' => '$createdAt', 'key' => '$createdAt', 'type' => Database::VAR_DATETIME, 'array' => false, ]); - $attributes[] = new \Utopia\Database\Document([ '$id' => '$updatedAt', 'key' => '$updatedAt', @@ -38,6 +43,7 @@ public function __construct(array $attributes) $validators = [ new Select($attributes), + new Filter($attributes) ]; parent::__construct($validators); diff --git a/src/Database/Validator/Queries/Documents.php b/src/Database/Validator/Queries/Documents.php index 0d1dc2384..a3211d6a3 100644 --- a/src/Database/Validator/Queries/Documents.php +++ b/src/Database/Validator/Queries/Documents.php @@ -36,6 +36,12 @@ public function __construct(array $attributes, array $indexes) 'type' => Database::VAR_STRING, 'array' => false, ]); + $attributes[] = new Document([ + '$id' => '$tenant', + 'key' => '$tenant', + 'type' => Database::VAR_STRING, + 'array' => false, + ]); $attributes[] = new Document([ '$id' => '$createdAt', 'key' => '$createdAt', diff --git a/src/Database/Validator/Structure.php b/src/Database/Validator/Structure.php index 14223b1a5..e8efbea94 100644 --- a/src/Database/Validator/Structure.php +++ b/src/Database/Validator/Structure.php @@ -52,6 +52,16 @@ class Structure extends Validator 'array' => false, 'filters' => [], ], + [ + '$id' => '$tenant', + 'type' => Database::VAR_STRING, + 'size' => 36, + 'required' => false, + 'default' => null, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], [ '$id' => '$permissions', 'type' => Database::VAR_STRING, diff --git a/tests/Database/Adapter/MariaDBTest.php b/tests/Database/Adapter/MariaDBTest.php index d5014432e..e88a60bd2 100644 --- a/tests/Database/Adapter/MariaDBTest.php +++ b/tests/Database/Adapter/MariaDBTest.php @@ -29,9 +29,9 @@ public static function getAdapterName(): string /** * @return Database */ - public static function getDatabase(): Database + public static function getDatabase(bool $fresh = false): Database { - if (!is_null(self::$database)) { + if (!is_null(self::$database) && !$fresh) { return self::$database; } @@ -47,7 +47,7 @@ public static function getDatabase(): Database $cache = new Cache(new RedisAdapter($redis)); $database = new Database(new MariaDB($pdo), $cache); - $database->setDefaultDatabase('utopiaTests'); + $database->setDatabase('utopiaTests'); $database->setNamespace('myapp_' . uniqid()); if ($database->exists('utopiaTests')) { diff --git a/tests/Database/Adapter/MongoDBTest.php b/tests/Database/Adapter/MongoDBTest.php index 62c48ae37..fced06b4b 100644 --- a/tests/Database/Adapter/MongoDBTest.php +++ b/tests/Database/Adapter/MongoDBTest.php @@ -52,7 +52,7 @@ public static function getDatabase(): Database ); $database = new Database(new Mongo($client), $cache); - $database->setDefaultDatabase($schema); + $database->setDatabase($schema); $database->setNamespace('myapp_' . uniqid()); if ($database->exists('utopiaTests')) { @@ -73,7 +73,7 @@ public function testCreateExistsDelete(): void $this->assertNotNull(static::getDatabase()->create()); $this->assertEquals(true, static::getDatabase()->delete($this->testDatabase)); $this->assertEquals(true, static::getDatabase()->create()); - $this->assertEquals(true, static::getDatabase()->setDefaultDatabase($this->testDatabase)); + $this->assertEquals(true, static::getDatabase()->setDatabase($this->testDatabase)); } public function testRenameAttribute(): void diff --git a/tests/Database/Adapter/MySQLTest.php b/tests/Database/Adapter/MySQLTest.php index 0faefca61..d527b75cd 100644 --- a/tests/Database/Adapter/MySQLTest.php +++ b/tests/Database/Adapter/MySQLTest.php @@ -58,7 +58,7 @@ public static function getDatabase(): Database $cache = new Cache(new RedisAdapter($redis)); $database = new Database(new MySQL($pdo), $cache); - $database->setDefaultDatabase('utopiaTests'); + $database->setDatabase('utopiaTests'); $database->setNamespace('myapp_'.uniqid()); if ($database->exists('utopiaTests')) { diff --git a/tests/Database/Adapter/PostgresTest.php b/tests/Database/Adapter/PostgresTest.php index 9965a8684..91ebc84e6 100644 --- a/tests/Database/Adapter/PostgresTest.php +++ b/tests/Database/Adapter/PostgresTest.php @@ -45,7 +45,7 @@ public static function getDatabase(): Database $cache = new Cache(new RedisAdapter($redis)); $database = new Database(new Postgres($pdo), $cache); - $database->setDefaultDatabase('utopiaTests'); + $database->setDatabase('utopiaTests'); $database->setNamespace('myapp_'.uniqid()); if ($database->exists('utopiaTests')) { diff --git a/tests/Database/Adapter/SQLiteTest.php b/tests/Database/Adapter/SQLiteTest.php index 8a8d24199..d82ed52f6 100644 --- a/tests/Database/Adapter/SQLiteTest.php +++ b/tests/Database/Adapter/SQLiteTest.php @@ -61,7 +61,7 @@ public static function getDatabase(): Database $cache = new Cache(new RedisAdapter($redis)); $database = new Database(new SQLite($pdo), $cache); - $database->setDefaultDatabase('utopiaTests'); + $database->setDatabase('utopiaTests'); $database->setNamespace('myapp_'.uniqid()); return self::$database = $database; diff --git a/tests/Database/Base.php b/tests/Database/Base.php index 4ec3f8566..60061607e 100644 --- a/tests/Database/Base.php +++ b/tests/Database/Base.php @@ -60,7 +60,7 @@ public function testCreateExistsDelete(): void { $schemaSupport = $this->getDatabase()->getAdapter()->getSupportForSchemas(); if (!$schemaSupport) { - $this->assertEquals(true, static::getDatabase()->setDefaultDatabase($this->testDatabase)); + $this->assertEquals(true, static::getDatabase()->setDatabase($this->testDatabase)); $this->assertEquals(true, static::getDatabase()->create()); return; } @@ -71,7 +71,7 @@ public function testCreateExistsDelete(): void $this->assertEquals(true, static::getDatabase()->exists($this->testDatabase)); $this->assertEquals(true, static::getDatabase()->delete($this->testDatabase)); $this->assertEquals(false, static::getDatabase()->exists($this->testDatabase)); - $this->assertEquals(true, static::getDatabase()->setDefaultDatabase($this->testDatabase)); + $this->assertEquals(true, static::getDatabase()->setDatabase($this->testDatabase)); $this->assertEquals(true, static::getDatabase()->create()); } @@ -12529,6 +12529,83 @@ public function testEmptyOperatorValues(): void } } + public function testIsolationModes(): void + { + /** + * Default mode already tested, we'll test 'schema' and 'table' isolation here + */ + $database = static::getDatabase(); + + /** + * Schema + */ + $database + ->setIsolationMode(Database::ISOLATION_MODE_SCHEMA) // Do we even need this for schema? + ->setDatabase('schema1') + ->setNamespace('') + ->create(); + + $this->assertEquals(true, $database->exists('schema1')); + + $database + ->setIsolationMode(Database::ISOLATION_MODE_SCHEMA) // Do we even need this for schema? + ->setDatabase('schema2') + ->setNamespace('') + ->create(); + + $this->assertEquals(true, $database->exists('schema2')); + + /** + * Table + */ + + $tenant1 = ID::unique(); + $tenant2 = ID::unique(); + + $database + ->setIsolationMode(Database::ISOLATION_MODE_TABLE) + ->setDatabase('sharedTables') + ->setNamespace('') + ->setTenant($tenant1) + ->create(); + + $this->assertEquals(true, $database->exists('sharedTables')); + + $database->createCollection('people', [ + new Document([ + '$id' => 'name', + 'type' => Database::VAR_STRING, + 'size' => 128, + 'required' => true, + ]) + ]); + + $docId = ID::unique(); + + $database->createDocument('people', new Document([ + '$id' => $docId, + '$permissions' => [ + Permission::read(Role::any()), + ], + 'name' => $tenant1, + ])); + + $doc = $database->getDocument('people', $docId); + $this->assertEquals($tenant1, $doc['name']); + + $docs = $database->find('people'); + $this->assertEquals(1, \count($docs)); + + $database->setTenant($tenant2); + + $doc = $database->getDocument('people', $docId); + $this->assertEmpty($doc); + + $docs = $database->find('people'); + $this->assertEquals(0, \count($docs)); + + } + public function testTransformations(): void { static::getDatabase()->createCollection('docs', attributes: [ @@ -12591,7 +12668,7 @@ public function testEvents(): void }); if ($this->getDatabase()->getAdapter()->getSupportForSchemas()) { - $database->setDefaultDatabase('hellodb'); + $database->setDatabase('hellodb'); $database->create(); } else { array_shift($events); @@ -12599,7 +12676,7 @@ public function testEvents(): void $database->list(); - $database->setDefaultDatabase($this->testDatabase); + $database->setDatabase($this->testDatabase); $collectionId = ID::unique(); $database->createCollection($collectionId); From 9da5b28900f61fa33ecba5713acfe039b8edb110 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 13 Nov 2023 18:48:13 +1300 Subject: [PATCH 04/45] Delete isolation mode DBs --- src/Database/Adapter.php | 21 +++++++-------------- src/Database/Database.php | 25 ++++++++++++------------- tests/Database/Adapter/SQLiteTest.php | 6 ++++++ tests/Database/Base.php | 19 +++++++++++++++++-- 4 files changed, 42 insertions(+), 29 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 9dcb394b2..9c263ac5f 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -7,15 +7,9 @@ abstract class Adapter { - /** - * @var string - */ - protected string $namespace = ''; + protected string $database = ''; - /** - * @var string - */ - protected string $defaultDatabase = ''; + protected string $namespace = ''; /** * @var array @@ -73,7 +67,7 @@ public function resetDebug(): self * @param string $namespace * * @return bool - * @throws Exception + * @throws DatabaseException * */ public function setNamespace(string $namespace): bool @@ -89,7 +83,6 @@ public function setNamespace(string $namespace): bool * Get namespace of current set scope * * @return string - * @throws DatabaseException * */ public function getNamespace(): string @@ -109,7 +102,7 @@ public function getNamespace(): string */ public function setDatabase(string $name): bool { - $this->defaultDatabase = $this->filter($name); + $this->database = $this->filter($name); return true; } @@ -125,11 +118,11 @@ public function setDatabase(string $name): bool */ public function getDatabase(): string { - if (empty($this->defaultDatabase)) { - throw new DatabaseException('Missing default database'); + if (empty($this->database)) { + throw new DatabaseException('Missing database. Database must be set before use.'); } - return $this->defaultDatabase; + return $this->database; } /** diff --git a/src/Database/Database.php b/src/Database/Database.php index 8db4d34f0..4ad37b055 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -555,7 +555,7 @@ public function withRequestTimestamp(?\DateTime $requestTimestamp, callable $cal * * @return $this * - * @throws Exception + * @throws DatabaseException */ public function setNamespace(string $namespace): self { @@ -584,7 +584,7 @@ public function getNamespace(): string * @param string $name * * @return self - * @throws Exception + * @throws DatabaseException */ public function setDatabase(string $name): self { @@ -593,15 +593,14 @@ public function setDatabase(string $name): self return $this; } - /** - * Get Database. - * - * Get Database from current scope - * - * @throws Exception - * - * @return string - */ + /** + * Get Database. + * + * Get Database from current scope + * + * @return string + * @throws DatabaseException + */ public function getDatabase(): string { return $this->adapter->getDatabase(); @@ -707,9 +706,9 @@ public function ping(): bool } /** - * Create the Default Database + * Create the database * - * @throws Exception + * @throws DatabaseException * * @return bool */ diff --git a/tests/Database/Adapter/SQLiteTest.php b/tests/Database/Adapter/SQLiteTest.php index d82ed52f6..bc4c78488 100644 --- a/tests/Database/Adapter/SQLiteTest.php +++ b/tests/Database/Adapter/SQLiteTest.php @@ -64,6 +64,12 @@ public static function getDatabase(): Database $database->setDatabase('utopiaTests'); $database->setNamespace('myapp_'.uniqid()); + if ($database->exists()) { + $database->delete(); + } + + $database->create(); + return self::$database = $database; } } diff --git a/tests/Database/Base.php b/tests/Database/Base.php index 60061607e..35a3ca615 100644 --- a/tests/Database/Base.php +++ b/tests/Database/Base.php @@ -10,11 +10,15 @@ use Utopia\Database\Database; use Utopia\Database\DateTime; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Authorization as AuthorizationException; use Utopia\Database\Exception\Conflict as ConflictException; use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\Limit as LimitException; +use Utopia\Database\Exception\Query as QueryException; use Utopia\Database\Exception\Restricted as RestrictedException; +use Utopia\Database\Exception\Structure as StructureException; +use Utopia\Database\Exception\Timeout as TimeoutException; use Utopia\Database\Helpers\ID; use Utopia\Database\Helpers\Permission; use Utopia\Database\Exception\Timeout; @@ -25,7 +29,6 @@ use Utopia\Database\Validator\Index; use Utopia\Database\Validator\Structure; use Utopia\Validator\Range; -use Utopia\Database\Exception\Structure as StructureException; abstract class Base extends TestCase { @@ -12529,7 +12532,16 @@ public function testEmptyOperatorValues(): void } } - public function testIsolationModes(): void + /** + * @throws AuthorizationException + * @throws DatabaseException + * @throws DuplicateException + * @throws LimitException + * @throws QueryException + * @throws StructureException + * @throws TimeoutException + */ + public function testIsolationModes(): void { /** * Default mode already tested, we'll test 'schema' and 'table' isolation here @@ -12604,6 +12616,9 @@ public function testIsolationModes(): void $docs = $database->find('people'); $this->assertEquals(0, \count($docs)); + $database->setDatabase('schema1')->delete(); + $database->setDatabase('schema2')->delete(); + $database->setDatabase('sharedTables')->delete(); } public function testTransformations(): void From 18a0485ebd6c59ef6c0159a6e93dfc2b6ecabbc6 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 13 Nov 2023 19:02:24 +1300 Subject: [PATCH 05/45] Optional database for create/exists/delete --- src/Database/Database.php | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 4ad37b055..a9d60e16e 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -712,10 +712,10 @@ public function ping(): bool * * @return bool */ - public function create(): bool + public function create(?string $database = null): bool { - $name = $this->adapter->getDatabase(); - $this->adapter->create($name); + $database = $database ?? $this->adapter->getDatabase(); + $this->adapter->create($database); /** * Create array of attribute documents @@ -737,7 +737,7 @@ public function create(): bool $this->silent(fn () => $this->createCollection(self::METADATA, $attributes)); - $this->trigger(self::EVENT_DATABASE_CREATE, $name); + $this->trigger(self::EVENT_DATABASE_CREATE, $database); return true; } @@ -746,13 +746,15 @@ public function create(): bool * Check if database exists * Optionally check if collection exists in database * - * @param string $database database name + * @param string|null $database (optional) database name * @param string|null $collection (optional) collection name * * @return bool */ - public function exists(string $database, string $collection = null): bool + public function exists(?string $database = null, ?string $collection = null): bool { + $database = $database ?? $this->adapter->getDatabase(); + return $this->adapter->exists($database, $collection); } @@ -773,15 +775,19 @@ public function list(): array /** * Delete Database * - * @param string $name - * + * @param string|null $database * @return bool */ - public function delete(string $name): bool + public function delete(?string $database = null): bool { - $deleted = $this->adapter->delete($name); + $database = $database ?? $this->adapter->getDatabase(); - $this->trigger(self::EVENT_DATABASE_DELETE, ['name' => $name, 'deleted' => $deleted]); + $deleted = $this->adapter->delete($database); + + $this->trigger(self::EVENT_DATABASE_DELETE, [ + 'name' => $database, + 'deleted' => $deleted + ]); return $deleted; } From 0cda563634a7d7fa696f080d230f495482c8afd7 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 13 Nov 2023 19:03:01 +1300 Subject: [PATCH 06/45] Format --- src/Database/Adapter.php | 2 +- src/Database/Database.php | 16 ++++++++-------- tests/Database/Adapter/SQLiteTest.php | 8 ++++---- tests/Database/Base.php | 26 +++++++++++++------------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 9c263ac5f..984e9fe44 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -7,7 +7,7 @@ abstract class Adapter { - protected string $database = ''; + protected string $database = ''; protected string $namespace = ''; diff --git a/src/Database/Database.php b/src/Database/Database.php index a9d60e16e..a5e41fe7e 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -593,14 +593,14 @@ public function setDatabase(string $name): self return $this; } - /** - * Get Database. - * - * Get Database from current scope - * - * @return string - * @throws DatabaseException - */ + /** + * Get Database. + * + * Get Database from current scope + * + * @return string + * @throws DatabaseException + */ public function getDatabase(): string { return $this->adapter->getDatabase(); diff --git a/tests/Database/Adapter/SQLiteTest.php b/tests/Database/Adapter/SQLiteTest.php index bc4c78488..18f120f23 100644 --- a/tests/Database/Adapter/SQLiteTest.php +++ b/tests/Database/Adapter/SQLiteTest.php @@ -64,11 +64,11 @@ public static function getDatabase(): Database $database->setDatabase('utopiaTests'); $database->setNamespace('myapp_'.uniqid()); - if ($database->exists()) { - $database->delete(); - } + if ($database->exists()) { + $database->delete(); + } - $database->create(); + $database->create(); return self::$database = $database; } diff --git a/tests/Database/Base.php b/tests/Database/Base.php index 35a3ca615..3a08b3f92 100644 --- a/tests/Database/Base.php +++ b/tests/Database/Base.php @@ -12532,16 +12532,16 @@ public function testEmptyOperatorValues(): void } } - /** - * @throws AuthorizationException - * @throws DatabaseException - * @throws DuplicateException - * @throws LimitException - * @throws QueryException - * @throws StructureException - * @throws TimeoutException - */ - public function testIsolationModes(): void + /** + * @throws AuthorizationException + * @throws DatabaseException + * @throws DuplicateException + * @throws LimitException + * @throws QueryException + * @throws StructureException + * @throws TimeoutException + */ + public function testIsolationModes(): void { /** * Default mode already tested, we'll test 'schema' and 'table' isolation here @@ -12616,9 +12616,9 @@ public function testIsolationModes(): void $docs = $database->find('people'); $this->assertEquals(0, \count($docs)); - $database->setDatabase('schema1')->delete(); - $database->setDatabase('schema2')->delete(); - $database->setDatabase('sharedTables')->delete(); + $database->setDatabase('schema1')->delete(); + $database->setDatabase('schema2')->delete(); + $database->setDatabase('sharedTables')->delete(); } public function testTransformations(): void From 9be6cb023340fb320a0cccc71fbd23a613dc445d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 14 Nov 2023 15:16:34 +1300 Subject: [PATCH 07/45] Simply table sharing setters --- src/Database/Database.php | 35 +++++++++++++---------------------- tests/Database/Base.php | 18 +++++++++++------- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index a5e41fe7e..787703c30 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -312,7 +312,7 @@ class Database protected ?\DateTime $timestamp = null; - protected string $isolationMode = self::ISOLATION_MODE_SHARED; + protected bool $shareTables = false; protected string $tenant = ''; @@ -667,23 +667,14 @@ public function clearTimeout(string $event = Database::EVENT_ALL): void } /** - * Set the isolation mode used when creating a new database resource + * Should tables be shared between users, segmented by tenant. * - * @param string $mode - * @return $this - * @throws DatabaseException + * @param bool $enabled + * @return self */ - public function setIsolationMode(string $mode): self + public function setShareTables(bool $enabled): self { - if (!\in_array($mode, [ - self::ISOLATION_MODE_SCHEMA, - self::ISOLATION_MODE_SHARED, - self::ISOLATION_MODE_TABLE - ])) { - throw new DatabaseException('Invalid isolation mode'); - } - - $this->isolationMode = $mode; + $this->shareTables = $enabled; return $this; } @@ -2309,7 +2300,7 @@ public function deleteIndex(string $collection, string $id): bool */ public function getDocument(string $collection, string $id, array $queries = []): Document { - if ($this->isolationMode === self::ISOLATION_MODE_TABLE) { + if ($this->shareTables) { if (empty($this->tenant)) { throw new DatabaseException('Missing tenant. Tenant must be set when isolation mode is set to "table".'); } @@ -2741,7 +2732,7 @@ private function populateDocumentRelationships(Document $collection, Document $d */ public function createDocument(string $collection, Document $document): Document { - if ($this->isolationMode === self::ISOLATION_MODE_TABLE) { + if ($this->shareTables) { if (empty($this->tenant)) { throw new DatabaseException('Missing tenant. Tenant must be set when isolation mode is set to "table".'); } @@ -2818,7 +2809,7 @@ public function createDocuments(string $collection, array $documents, int $batch $time = DateTime::now(); foreach ($documents as $key => $document) { - if ($this->isolationMode === self::ISOLATION_MODE_TABLE) { + if ($this->shareTables) { if (empty($document->getAttribute('$tenant'))) { throw new DatabaseException('Missing tenant. Tenant must be included when isolation mode is set to "table".'); } @@ -3142,7 +3133,7 @@ private function relateDocumentsById( */ public function updateDocument(string $collection, string $id, Document $document): Document { - if ($this->isolationMode === self::ISOLATION_MODE_TABLE) { + if ($this->shareTables) { if (empty($document->getAttribute('$tenant'))) { throw new DatabaseException('Missing tenant. Tenant must be included when isolation mode is set to "table".'); } @@ -3322,7 +3313,7 @@ public function updateDocuments(string $collection, array $documents, int $batch $collection = $this->silent(fn () => $this->getCollection($collection)); foreach ($documents as $document) { - if ($this->isolationMode === self::ISOLATION_MODE_TABLE) { + if ($this->shareTables) { if (empty($document->getAttribute('$tenant'))) { throw new DatabaseException('Missing tenant. Tenant must be included when isolation mode is set to "table".'); } @@ -3900,7 +3891,7 @@ public function deleteDocument(string $collection, string $id): bool { $document = Authorization::skip(fn () => $this->silent(fn () => $this->getDocument($collection, $id))); - if ($this->isolationMode === self::ISOLATION_MODE_TABLE) { + if ($this->shareTables) { if (empty($document->getAttribute('$tenant'))) { throw new DatabaseException('Missing tenant. Tenant must be included when isolation mode is set to "table".'); } @@ -4351,7 +4342,7 @@ public function deleteCachedDocument(string $collection, string $id): bool */ public function find(string $collection, array $queries = []): array { - if ($this->isolationMode === self::ISOLATION_MODE_TABLE) { + if ($this->shareTables) { if (empty($this->tenant)) { throw new DatabaseException('Missing tenant. Tenant must be set when isolation mode is set to "table".'); } diff --git a/tests/Database/Base.php b/tests/Database/Base.php index 3a08b3f92..4b8071cab 100644 --- a/tests/Database/Base.php +++ b/tests/Database/Base.php @@ -12548,11 +12548,20 @@ public function testIsolationModes(): void */ $database = static::getDatabase(); + if ($database->exists('schema1')) { + $database->setDatabase('schema1')->delete(); + } + if ($database->exists('schema2')) { + $database->setDatabase('schema2')->delete(); + } + if ($database->exists('sharedTables')) { + $database->setDatabase('sharedTables')->delete(); + } + /** * Schema */ $database - ->setIsolationMode(Database::ISOLATION_MODE_SCHEMA) // Do we even need this for schema? ->setDatabase('schema1') ->setNamespace('') ->create(); @@ -12560,7 +12569,6 @@ public function testIsolationModes(): void $this->assertEquals(true, $database->exists('schema1')); $database - ->setIsolationMode(Database::ISOLATION_MODE_SCHEMA) // Do we even need this for schema? ->setDatabase('schema2') ->setNamespace('') ->create(); @@ -12575,9 +12583,9 @@ public function testIsolationModes(): void $tenant2 = ID::unique(); $database - ->setIsolationMode(Database::ISOLATION_MODE_TABLE) ->setDatabase('sharedTables') ->setNamespace('') + ->setShareTables(true) ->setTenant($tenant1) ->create(); @@ -12615,10 +12623,6 @@ public function testIsolationModes(): void $docs = $database->find('people'); $this->assertEquals(0, \count($docs)); - - $database->setDatabase('schema1')->delete(); - $database->setDatabase('schema2')->delete(); - $database->setDatabase('sharedTables')->delete(); } public function testTransformations(): void From f9da8fc02dbe8336eb2d3500bc30f6fb426417c4 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 15 Nov 2023 23:55:25 +1300 Subject: [PATCH 08/45] Cache fixes --- src/Database/Adapter/MariaDB.php | 13 ++++++++++--- src/Database/Database.php | 6 +++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 0e8e66465..91173590d 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -111,11 +111,11 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = " CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id)} ( - `_id` int(11) unsigned NOT NULL AUTO_INCREMENT, + `_id` INT(11) UNSIGNED NOT NULL AUTO_INCREMENT, `_uid` VARCHAR(255) NOT NULL, `_tenant` CHAR(36) NOT NULL, - `_createdAt` datetime(3) DEFAULT NULL, - `_updatedAt` datetime(3) DEFAULT NULL, + `_createdAt` DATETIME(3) DEFAULT NULL, + `_updatedAt` DATETIME(3) DEFAULT NULL, `_permissions` MEDIUMTEXT DEFAULT NULL, " . \implode(' ', $attributeStrings) . " PRIMARY KEY (`_id`), @@ -794,6 +794,7 @@ public function createDocuments(string $collection, array $documents, int $batch foreach ($batch as $document) { $attributes = $document->getAttributes(); $attributes['_uid'] = $document->getId(); + $attributes['_tenant'] = $document->getAttribute('$tenant'); $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = \json_encode($document->getPermissions()); @@ -878,6 +879,7 @@ public function createDocuments(string $collection, array $documents, int $batch public function updateDocument(string $collection, Document $document): Document { $attributes = $document->getAttributes(); + $attributes['_tenant'] = $document->getAttribute('$tenant'); $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = json_encode($document->getPermissions()); @@ -1105,6 +1107,7 @@ public function updateDocuments(string $collection, array $documents, int $batch foreach ($batch as $index => $document) { $attributes = $document->getAttributes(); $attributes['_uid'] = $document->getId(); + $attributes['_tenant'] = $document->getAttribute('$tenant'); $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = json_encode($document->getPermissions()); @@ -1555,6 +1558,10 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $results[$index]['$internalId'] = $document['_id']; unset($results[$index]['_id']); } + if (\array_key_exists('_tenant', $document)) { + $results[$index]['$tenant'] = $document['_tenant']; + unset($results[$index]['_tenant']); + } if (\array_key_exists('_createdAt', $document)) { $results[$index]['$createdAt'] = $document['_createdAt']; unset($results[$index]['_createdAt']); diff --git a/src/Database/Database.php b/src/Database/Database.php index 787703c30..6185ccdde 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -2379,7 +2379,7 @@ public function getDocument(string $collection, string $id, array $queries = []) $validator = new Authorization(self::PERMISSION_READ); $documentSecurity = $collection->getAttribute('documentSecurity', false); - $cacheKey = 'cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $id; + $cacheKey = 'cache-' . $this->getNamespace() . ':' . $this->tenant . ':' . $collection->getId() . ':' . $id; if (!empty($selections)) { $cacheKey .= ':' . \md5(\implode($selections)); @@ -2449,13 +2449,13 @@ public function getDocument(string $collection, string $id, array $queries = []) * @phpstan-ignore-next-line */ foreach ($this->map as $key => $value) { - list($k, $v) = explode('=>', $key); + [$k, $v] = \explode('=>', $key); $ck = 'cache-' . $this->getNamespace() . ':map:' . $k; $cache = $this->cache->load($ck, self::TTL); if (empty($cache)) { $cache = []; } - if (!in_array($v, $cache)) { + if (!\in_array($v, $cache)) { $cache[] = $v; $this->cache->save($ck, $cache); } From bbeecdebb3d3299682d489ba482eb33d4983e327 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 17 Nov 2023 12:50:31 +1300 Subject: [PATCH 09/45] Test fixes --- src/Database/Database.php | 8 +++---- src/Database/Validator/Structure.php | 2 +- tests/Database/Base.php | 28 ++++++++++++++++++---- tests/Database/Validator/StructureTest.php | 2 +- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 6185ccdde..f53144dd2 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -2302,7 +2302,7 @@ public function getDocument(string $collection, string $id, array $queries = []) { if ($this->shareTables) { if (empty($this->tenant)) { - throw new DatabaseException('Missing tenant. Tenant must be set when isolation mode is set to "table".'); + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } $queries[] = Query::equal('$tenant', [$this->tenant]); } @@ -2734,7 +2734,7 @@ public function createDocument(string $collection, Document $document): Document { if ($this->shareTables) { if (empty($this->tenant)) { - throw new DatabaseException('Missing tenant. Tenant must be set when isolation mode is set to "table".'); + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } $document->setAttribute('$tenant', $this->tenant); } @@ -4344,7 +4344,7 @@ public function find(string $collection, array $queries = []): array { if ($this->shareTables) { if (empty($this->tenant)) { - throw new DatabaseException('Missing tenant. Tenant must be set when isolation mode is set to "table".'); + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } $queries[] = Query::equal('$tenant', [$this->tenant]); } @@ -4353,7 +4353,7 @@ public function find(string $collection, array $queries = []): array $collection = $this->silent(fn () => $this->getCollection($collection)); if ($collection->isEmpty()) { - throw new DatabaseException('Collection "'. $originalName .'" not found'); + throw new DatabaseException('Collection not found'); } $attributes = $collection->getAttribute('attributes', []); diff --git a/src/Database/Validator/Structure.php b/src/Database/Validator/Structure.php index e8efbea94..3d7d2ade7 100644 --- a/src/Database/Validator/Structure.php +++ b/src/Database/Validator/Structure.php @@ -218,7 +218,7 @@ public function isValid($document): bool } if (empty($this->collection->getId()) || Database::METADATA !== $this->collection->getCollection()) { - $this->message = 'Collection "'.$this->collection->getCollection().'" not found'; + $this->message = 'Collection not found'; return false; } diff --git a/tests/Database/Base.php b/tests/Database/Base.php index 4b8071cab..1c4d22d27 100644 --- a/tests/Database/Base.php +++ b/tests/Database/Base.php @@ -526,7 +526,7 @@ public function testCollectionNotFound(): void static::getDatabase()->find('not_exist', []); $this->fail('Failed to throw Exception'); } catch (Exception $e) { - $this->assertEquals('Collection "not_exist" not found', $e->getMessage()); + $this->assertEquals('Collection not found', $e->getMessage()); } } @@ -12616,13 +12616,31 @@ public function testIsolationModes(): void $docs = $database->find('people'); $this->assertEquals(1, \count($docs)); + // Swap to tenant 2, no access $database->setTenant($tenant2); - $doc = $database->getDocument('people', $docId); - $this->assertEmpty($doc); + try { + $database->getDocument('people', $docId); + $this->fail('Failed to throw exception'); + } catch (Exception $e) { + $this->assertEquals('Collection not found', $e->getMessage()); + } - $docs = $database->find('people'); - $this->assertEquals(0, \count($docs)); + try { + $database->find('people'); + $this->fail('Failed to throw exception'); + } catch (Exception $e) { + $this->assertEquals('Collection not found', $e->getMessage()); + } + + // Swap back to tenant 1, allowed + $database->setTenant($tenant1); + + $doc = $database->getDocument('people', $docId); + $this->assertEquals($tenant1, $doc['name']); + + $docs = $database->find('people'); + $this->assertEquals(1, \count($docs)); } public function testTransformations(): void diff --git a/tests/Database/Validator/StructureTest.php b/tests/Database/Validator/StructureTest.php index 6f10f2bf4..da9a2cff2 100644 --- a/tests/Database/Validator/StructureTest.php +++ b/tests/Database/Validator/StructureTest.php @@ -154,7 +154,7 @@ public function testCollection(): void 'feedback' => 'team@appwrite.io', ]))); - $this->assertEquals('Invalid document structure: Collection "" not found', $validator->getDescription()); + $this->assertEquals('Invalid document structure: Collection not found', $validator->getDescription()); } public function testRequiredKeys(): void From f250d073689009dfd81555be0f88e2113251a5dd Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 17 Nov 2023 13:59:45 +1300 Subject: [PATCH 10/45] Format --- src/Database/Adapter/MariaDB.php | 14 ++++++------ tests/Database/Base.php | 38 ++++++++++++++++---------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index e24973caa..d2c6796ca 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -801,7 +801,7 @@ public function createDocuments(string $collection, array $documents, int $batch foreach ($batch as $document) { $attributes = $document->getAttributes(); $attributes['_uid'] = $document->getId(); - $attributes['_tenant'] = $document->getAttribute('$tenant'); + $attributes['_tenant'] = $document->getAttribute('$tenant'); $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = \json_encode($document->getPermissions()); @@ -889,7 +889,7 @@ public function createDocuments(string $collection, array $documents, int $batch public function updateDocument(string $collection, Document $document): Document { $attributes = $document->getAttributes(); - $attributes['_tenant'] = $document->getAttribute('$tenant'); + $attributes['_tenant'] = $document->getAttribute('$tenant'); $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = json_encode($document->getPermissions()); @@ -1117,7 +1117,7 @@ public function updateDocuments(string $collection, array $documents, int $batch foreach ($batch as $index => $document) { $attributes = $document->getAttributes(); $attributes['_uid'] = $document->getId(); - $attributes['_tenant'] = $document->getAttribute('$tenant'); + $attributes['_tenant'] = $document->getAttribute('$tenant'); $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = json_encode($document->getPermissions()); @@ -1564,10 +1564,10 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $results[$index]['$internalId'] = $document['_id']; unset($results[$index]['_id']); } - if (\array_key_exists('_tenant', $document)) { - $results[$index]['$tenant'] = $document['_tenant']; - unset($results[$index]['_tenant']); - } + if (\array_key_exists('_tenant', $document)) { + $results[$index]['$tenant'] = $document['_tenant']; + unset($results[$index]['_tenant']); + } if (\array_key_exists('_createdAt', $document)) { $results[$index]['$createdAt'] = $document['_createdAt']; unset($results[$index]['_createdAt']); diff --git a/tests/Database/Base.php b/tests/Database/Base.php index 1c4d22d27..801f65256 100644 --- a/tests/Database/Base.php +++ b/tests/Database/Base.php @@ -12616,31 +12616,31 @@ public function testIsolationModes(): void $docs = $database->find('people'); $this->assertEquals(1, \count($docs)); - // Swap to tenant 2, no access + // Swap to tenant 2, no access $database->setTenant($tenant2); - try { - $database->getDocument('people', $docId); - $this->fail('Failed to throw exception'); - } catch (Exception $e) { - $this->assertEquals('Collection not found', $e->getMessage()); - } + try { + $database->getDocument('people', $docId); + $this->fail('Failed to throw exception'); + } catch (Exception $e) { + $this->assertEquals('Collection not found', $e->getMessage()); + } - try { - $database->find('people'); - $this->fail('Failed to throw exception'); - } catch (Exception $e) { - $this->assertEquals('Collection not found', $e->getMessage()); - } + try { + $database->find('people'); + $this->fail('Failed to throw exception'); + } catch (Exception $e) { + $this->assertEquals('Collection not found', $e->getMessage()); + } - // Swap back to tenant 1, allowed - $database->setTenant($tenant1); + // Swap back to tenant 1, allowed + $database->setTenant($tenant1); - $doc = $database->getDocument('people', $docId); - $this->assertEquals($tenant1, $doc['name']); + $doc = $database->getDocument('people', $docId); + $this->assertEquals($tenant1, $doc['name']); - $docs = $database->find('people'); - $this->assertEquals(1, \count($docs)); + $docs = $database->find('people'); + $this->assertEquals(1, \count($docs)); } public function testTransformations(): void From 2902e7079e392480621b2ea5653815ecc84b073c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 17 Nov 2023 14:54:37 +1300 Subject: [PATCH 11/45] Default null tenant --- src/Database/Adapter/MariaDB.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index d2c6796ca..74703b73f 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -117,7 +117,7 @@ public function createCollection(string $name, array $attributes = [], array $in CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id)} ( `_id` INT(11) UNSIGNED NOT NULL AUTO_INCREMENT, `_uid` VARCHAR(255) NOT NULL, - `_tenant` CHAR(36) NOT NULL, + `_tenant` CHAR(36) DEFAULT NULL, `_createdAt` DATETIME(3) DEFAULT NULL, `_updatedAt` DATETIME(3) DEFAULT NULL, `_permissions` MEDIUMTEXT DEFAULT NULL, @@ -142,7 +142,7 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = " CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id . '_perms')} ( `_id` int(11) unsigned NOT NULL AUTO_INCREMENT, - `_tenant` CHAR(36) NOT NULL, + `_tenant` CHAR(36) DEFAULT NULL, `_type` VARCHAR(12) NOT NULL, `_permission` VARCHAR(255) NOT NULL, `_document` VARCHAR(255) NOT NULL, From b7bf0c5d082cd82b59f53c0c034c893ca1ff1a34 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 17 Nov 2023 21:11:47 +1300 Subject: [PATCH 12/45] Fix isolation modes + parallel workflow --- .github/workflows/tests.yml | 126 ++++++--- composer.json | 6 +- phpunit.xml | 8 +- src/Database/Adapter.php | 60 ++++ src/Database/Adapter/MariaDB.php | 173 +++++++++--- src/Database/Adapter/Postgres.php | 2 +- src/Database/Adapter/SQL.php | 78 +++--- src/Database/Adapter/SQLite.php | 2 +- src/Database/Database.php | 260 +++++++++++++----- src/Database/Validator/Queries/Document.php | 8 - src/Database/Validator/Queries/Documents.php | 6 - tests/{Database => e2e/Adapter}/Base.php | 45 ++- .../{Database => e2e}/Adapter/MariaDBTest.php | 15 +- .../{Database => e2e}/Adapter/MongoDBTest.php | 9 +- tests/{Database => e2e}/Adapter/MySQLTest.php | 11 +- .../Adapter/PostgresTest.php | 12 +- .../{Database => e2e}/Adapter/SQLiteTest.php | 11 +- tests/{Database => unit}/DocumentTest.php | 4 +- tests/{Database => unit}/Format.php | 3 +- tests/{Database => unit}/IDTest.php | 2 +- tests/{Database => unit}/PermissionTest.php | 2 +- tests/{Database => unit}/QueryTest.php | 4 +- tests/{Database => unit}/RoleTest.php | 2 +- .../Validator/AuthorizationTest.php | 4 +- .../Validator/DateTimeTest.php | 4 +- .../Validator/DocumentQueriesTest.php | 2 +- .../Validator/DocumentsQueriesTest.php | 2 +- .../Validator/IndexTest.php | 4 +- .../Validator/IndexedQueriesTest.php | 2 +- .../{Database => unit}/Validator/KeyTest.php | 4 +- .../Validator/LabelTest.php | 4 +- .../Validator/PermissionsTest.php | 2 +- .../Validator/QueriesTest.php | 2 +- .../Validator/Query/CursorTest.php | 2 +- .../Validator/Query/FilterTest.php | 2 +- .../Validator/Query/LimitTest.php | 2 +- .../Validator/Query/OffsetTest.php | 2 +- .../Validator/Query/OrderTest.php | 2 +- .../Validator/Query/SelectTest.php | 2 +- .../Validator/QueryTest.php | 2 +- .../Validator/RolesTest.php | 4 +- .../Validator/StructureTest.php | 8 +- .../{Database => unit}/Validator/UIDTest.php | 2 +- 43 files changed, 604 insertions(+), 303 deletions(-) rename tests/{Database => e2e/Adapter}/Base.php (99%) rename tests/{Database => e2e}/Adapter/MariaDBTest.php (85%) rename tests/{Database => e2e}/Adapter/MongoDBTest.php (93%) rename tests/{Database => e2e}/Adapter/MySQLTest.php (89%) rename tests/{Database => e2e}/Adapter/PostgresTest.php (89%) rename tests/{Database => e2e}/Adapter/SQLiteTest.php (89%) rename tests/{Database => unit}/DocumentTest.php (99%) rename tests/{Database => unit}/Format.php (94%) rename tests/{Database => unit}/IDTest.php (94%) rename tests/{Database => unit}/PermissionTest.php (99%) rename tests/{Database => unit}/QueryTest.php (99%) rename tests/{Database => unit}/RoleTest.php (99%) rename tests/{Database => unit}/Validator/AuthorizationTest.php (99%) rename tests/{Database => unit}/Validator/DateTimeTest.php (98%) rename tests/{Database => unit}/Validator/DocumentQueriesTest.php (98%) rename tests/{Database => unit}/Validator/DocumentsQueriesTest.php (99%) rename tests/{Database => unit}/Validator/IndexTest.php (99%) rename tests/{Database => unit}/Validator/IndexedQueriesTest.php (99%) rename tests/{Database => unit}/Validator/KeyTest.php (99%) rename tests/{Database => unit}/Validator/LabelTest.php (98%) rename tests/{Database => unit}/Validator/PermissionsTest.php (99%) rename tests/{Database => unit}/Validator/QueriesTest.php (98%) rename tests/{Database => unit}/Validator/Query/CursorTest.php (96%) rename tests/{Database => unit}/Validator/Query/FilterTest.php (99%) rename tests/{Database => unit}/Validator/Query/LimitTest.php (95%) rename tests/{Database => unit}/Validator/Query/OffsetTest.php (96%) rename tests/{Database => unit}/Validator/Query/OrderTest.php (97%) rename tests/{Database => unit}/Validator/Query/SelectTest.php (97%) rename tests/{Database => unit}/Validator/QueryTest.php (99%) rename tests/{Database => unit}/Validator/RolesTest.php (98%) rename tests/{Database => unit}/Validator/StructureTest.php (99%) rename tests/{Database => unit}/Validator/UIDTest.php (55%) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 970fc22de..7a8e79a97 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,40 +1,100 @@ name: "Tests" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + IMAGE: databases-dev + CACHE_KEY: databases-dev-${{ github.event.pull_request.head.sha }} + on: [pull_request] + jobs: - tests: - name: Unit & E2E + setup: + name: Setup & Build Docker Image + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build Docker Image + uses: docker/build-push-action@v3 + with: + context: . + push: false + tags: ${{ env.IMAGE }} + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + outputs: type=docker,dest=/tmp/${{ env.IMAGE }}.tar + + - name: Cache Docker Image + uses: actions/cache@v3 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + + unit_test: + name: Unit Test runs-on: ubuntu-latest + needs: setup steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - fetch-depth: 2 - submodules: recursive - - - run: git checkout HEAD^2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Build image - uses: docker/build-push-action@v3 - with: - context: . - push: false - tags: database-dev - load: true - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Start Databases - run: | - docker compose up -d - sleep 10 - - - name: Run Tests - run: docker compose exec -T tests vendor/bin/phpunit --configuration phpunit.xml - - - name: Check Coverage - run: docker compose exec -T tests vendor/bin/coverage-check tmp/clover.xml 90 \ No newline at end of file + - name: checkout + uses: actions/checkout@v2 + + - name: Load Cache + uses: actions/cache@v3 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + fail-on-cache-miss: true + + - name: Load and Start Services + run: | + docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose up -d + sleep 10 + + - name: Run Unit Tests + run: docker compose exec appwrite test /usr/src/code/tests/unit + + e2e_test: + name: E2E Tests + runs-on: ubuntu-latest + needs: setup + strategy: + fail-fast: false + matrix: + adapter: + [ + MariaDB, + MySQL, + Postgres, + SQLite, + MongoDB, + ] + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Load Cache + uses: actions/cache@v3 + with: + key: ${{ env.CACHE_KEY }} + path: /tmp/${{ env.IMAGE }}.tar + fail-on-cache-miss: true + + - name: Load and Start Services + run: | + docker load --input /tmp/${{ env.IMAGE }}.tar + docker compose up -d + sleep 10 + + - name: Run ${{matrix.service}} Tests + run: docker compose exec -T tests vendor/bin/phpunit /usr/src/code/tests/Database/Adapter/${{matrix.service}}Test.php --debug \ No newline at end of file diff --git a/composer.json b/composer.json index eda86eba2..497e464e1 100755 --- a/composer.json +++ b/composer.json @@ -9,7 +9,11 @@ "psr-4": {"Utopia\\Database\\": "src/Database"} }, "autoload-dev": { - "psr-4": {"Utopia\\Tests\\": "tests/Database"} + "psr-4": { + "Tests\\E2E\\": "tests/e2e", + "Tests\\Unit\\": "tests/unit", + "Appwrite\\Tests\\": "tests/extensions" + } }, "scripts": { "build": [ diff --git a/phpunit.xml b/phpunit.xml index 3833748e0..350af5c34 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -9,8 +9,12 @@ processIsolation="false" stopOnFailure="true"> - - ./tests/ + + ./tests/unit + + + ./tests/e2e/Client.php + ./tests/e2e/Adapter diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index a06600cd8..e879ea499 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -11,6 +11,10 @@ abstract class Adapter protected string $namespace = ''; + protected bool $shareTables = false; + + protected ?string $tenant = null; + /** * @var array */ @@ -125,6 +129,62 @@ public function getDatabase(): string return $this->database; } + /** + * Set Share Tables. + * + * Set whether to share tables between tenants + * + * @param bool $shareTables + * + * @return bool + */ + public function setShareTables(bool $shareTables): bool + { + $this->shareTables = $shareTables; + + return true; + } + + /** + * Get Share Tables. + * + * Get whether to share tables between tenants + * + * @return bool + */ + public function getShareTables(): bool + { + return $this->shareTables; + } + + /** + * Set Tenant. + * + * Set tenant to use for current scope + * + * @param string $tenant + * + * @return bool + */ + public function setTenant(string $tenant): bool + { + $this->tenant = $tenant; + + return true; + } + + /** + * Get Tenant. + * + * Get Tenant from current scope + * + * @return string + */ + public function getTenant(): ?string + { + return $this->tenant; + } + /** * Set metadata for query comments * diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 74703b73f..9ea324e2f 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -141,7 +141,7 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = " CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id . '_perms')} ( - `_id` int(11) unsigned NOT NULL AUTO_INCREMENT, + `_id` int(11) UNSIGNED NOT NULL AUTO_INCREMENT, `_tenant` CHAR(36) DEFAULT NULL, `_type` VARCHAR(12) NOT NULL, `_permission` VARCHAR(255) NOT NULL, @@ -666,7 +666,7 @@ public function deleteIndex(string $collection, string $id): bool public function createDocument(string $collection, Document $document): Document { $attributes = $document->getAttributes(); - $attributes['_tenant'] = $document->getAttribute('$tenant'); + $attributes['_tenant'] = $this->tenant; $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = \json_encode($document->getPermissions()); @@ -695,7 +695,7 @@ public function createDocument(string $collection, Document $document): Document } $sql = " - INSERT INTO {$this->getSQLTable($name)}({$columns} _uid) + INSERT INTO {$this->getSQLTable($name)} ({$columns} _uid) VALUES ({$columnNames} :_uid) "; @@ -703,10 +703,10 @@ public function createDocument(string $collection, Document $document): Document $stmt = $this->getPDO()->prepare($sql); - $stmt->bindValue(':_uid', $document->getId(), PDO::PARAM_STR); + $stmt->bindValue(':_uid', $document->getId()); if (!empty($document->getInternalId())) { - $stmt->bindValue(':_id', $document->getInternalId(), PDO::PARAM_STR); + $stmt->bindValue(':_id', $document->getInternalId()); } $attributeIndex = 0; @@ -725,19 +725,20 @@ public function createDocument(string $collection, Document $document): Document foreach (Database::PERMISSIONS as $type) { foreach ($document->getPermissionsByType($type) as $permission) { $permission = \str_replace('"', '', $permission); - $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}', '{$document->getAttribute('$tenant')}')"; + $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}', :_tenant)"; } } if (!empty($permissions)) { - $strPermissions = \implode(', ', $permissions); + $permissions = \implode(', ', $permissions); $sqlPermissions = " INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document, _tenant) - VALUES {$strPermissions} + VALUES {$permissions} "; $sqlPermissions = $this->trigger(Database::EVENT_PERMISSIONS_CREATE, $sqlPermissions); $stmtPermissions = $this->getPDO()->prepare($sqlPermissions); + $stmtPermissions->bindValue(':_tenant', $this->tenant); } try { @@ -801,7 +802,7 @@ public function createDocuments(string $collection, array $documents, int $batch foreach ($batch as $document) { $attributes = $document->getAttributes(); $attributes['_uid'] = $document->getId(); - $attributes['_tenant'] = $document->getAttribute('$tenant'); + $attributes['_tenant'] = $this->tenant; $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = \json_encode($document->getPermissions()); @@ -830,7 +831,7 @@ public function createDocuments(string $collection, array $documents, int $batch foreach (Database::PERMISSIONS as $type) { foreach ($document->getPermissionsByType($type) as $permission) { $permission = \str_replace('"', '', $permission); - $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}')"; + $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}', :_tenant)"; } } } @@ -850,9 +851,10 @@ public function createDocuments(string $collection, array $documents, int $batch if (!empty($permissions)) { $stmtPermissions = $this->getPDO()->prepare( " - INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document) + INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document, _tenant) VALUES " . \implode(', ', $permissions) ); + $stmtPermissions->bindValue(':_tenant', $this->tenant); $stmtPermissions?->execute(); } } @@ -889,7 +891,7 @@ public function createDocuments(string $collection, array $documents, int $batch public function updateDocument(string $collection, Document $document): Document { $attributes = $document->getAttributes(); - $attributes['_tenant'] = $document->getAttribute('$tenant'); + $attributes['_tenant'] = $this->tenant; $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = json_encode($document->getPermissions()); @@ -899,10 +901,14 @@ public function updateDocument(string $collection, Document $document): Document $sql = " SELECT _type, _permission - FROM {$this->getSQLTable($name . '_perms')} p - WHERE p._document = :_uid + FROM {$this->getSQLTable($name . '_perms')} + WHERE _document = :_uid "; + if ($this->shareTables) { + $sql .= ' AND _tenant = :_tenant'; + } + $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql); /** @@ -910,6 +916,11 @@ public function updateDocument(string $collection, Document $document): Document */ $sqlPermissions = $this->getPDO()->prepare($sql); $sqlPermissions->bindValue(':_uid', $document->getId()); + + if ($this->shareTables) { + $sqlPermissions->bindValue(':_tenant', $this->tenant); + } + $sqlPermissions->execute(); $permissions = $sqlPermissions->fetchAll(); $sqlPermissions->closeCursor(); @@ -965,19 +976,27 @@ public function updateDocument(string $collection, Document $document): Document } if (!empty($removeQuery)) { $removeQuery .= ')'; - $removeQuery = " + $sql = " DELETE FROM {$this->getSQLTable($name . '_perms')} - WHERE - _document = :_uid - {$removeQuery} + WHERE _document = :_uid "; + if ($this->shareTables) { + $sql .= ' AND _tenant = :_tenant'; + } + + $removeQuery = $sql . $removeQuery; + $removeQuery = $this->trigger(Database::EVENT_PERMISSIONS_DELETE, $removeQuery); $stmtRemovePermissions = $this->getPDO()->prepare($removeQuery); $stmtRemovePermissions->bindValue(':_uid', $document->getId()); + if ($this->shareTables) { + $stmtRemovePermissions->bindValue(':_tenant', $this->tenant); + } + foreach ($removals as $type => $permissions) { foreach ($permissions as $i => $permission) { $stmtRemovePermissions->bindValue(":_remove_{$type}_{$i}", $permission); @@ -992,20 +1011,20 @@ public function updateDocument(string $collection, Document $document): Document $values = []; foreach ($additions as $type => $permissions) { foreach ($permissions as $i => $_) { - $values[] = "( :_uid, '{$type}', :_add_{$type}_{$i} )"; + $values[] = "( :_uid, '{$type}', :_add_{$type}_{$i}, :_tenant)"; } } $sql = " - INSERT INTO {$this->getSQLTable($name . '_perms')} - (_document, _type, _permission) VALUES " . \implode(', ', $values) - ; + INSERT INTO {$this->getSQLTable($name . '_perms')} (_document, _type, _permission, _tenant) + VALUES " . \implode(', ', $values); $sql = $this->trigger(Database::EVENT_PERMISSIONS_CREATE, $sql); $stmtAddPermissions = $this->getPDO()->prepare($sql); $stmtAddPermissions->bindValue(":_uid", $document->getId()); + $stmtAddPermissions->bindValue(":_tenant", $this->tenant); foreach ($additions as $type => $permissions) { foreach ($permissions as $i => $permission) { $stmtAddPermissions->bindValue(":_add_{$type}_{$i}", $permission); @@ -1030,12 +1049,20 @@ public function updateDocument(string $collection, Document $document): Document WHERE _uid = :_uid "; + if ($this->shareTables) { + $sql .= ' AND _tenant = :_tenant'; + } + $sql = $this->trigger(Database::EVENT_DOCUMENT_UPDATE, $sql); $stmt = $this->getPDO()->prepare($sql); $stmt->bindValue(':_uid', $document->getId()); + if ($this->shareTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } + $attributeIndex = 0; foreach ($attributes as $attribute => $value) { if (is_array($value)) { @@ -1043,7 +1070,6 @@ public function updateDocument(string $collection, Document $document): Document } $bindKey = 'key_' . $attributeIndex; - $attribute = $this->filter($attribute); $value = (is_bool($value)) ? (int)$value : $value; $stmt->bindValue(':' . $bindKey, $value, $this->getPDOType($value)); $attributeIndex++; @@ -1117,7 +1143,7 @@ public function updateDocuments(string $collection, array $documents, int $batch foreach ($batch as $index => $document) { $attributes = $document->getAttributes(); $attributes['_uid'] = $document->getId(); - $attributes['_tenant'] = $document->getAttribute('$tenant'); + $attributes['_tenant'] = $this->tenant; $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = json_encode($document->getPermissions()); @@ -1142,14 +1168,27 @@ public function updateDocuments(string $collection, array $documents, int $batch $batchKeys[] = '(' . implode(', ', $bindKeys) . ')'; // Permissions logic - $permissionsStmt = $this->getPDO()->prepare(" + $sql = " SELECT _type, _permission FROM {$this->getSQLTable($name . '_perms')} WHERE _document = :_uid - "); + "; + + if ($this->shareTables) { + $sql .= ' AND _tenant = :_tenant'; + } + + $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql); + + $permissionsStmt = $this->getPDO()->prepare($sql); $permissionsStmt->bindValue(':_uid', $document->getId()); + + if ($this->shareTables) { + $permissionsStmt->bindValue(':_tenant', $this->tenant); + } + $permissionsStmt->execute(); - $permissions = $permissionsStmt->fetchAll(PDO::FETCH_ASSOC); + $permissions = $permissionsStmt->fetchAll(); $initial = []; foreach (Database::PERMISSIONS as $type) { @@ -1177,10 +1216,16 @@ public function updateDocuments(string $collection, array $documents, int $batch $removeBindKeys[] = ':uid_' . $index; $removeBindValues[$bindKey] = $document->getId(); + $tenantQuery = ''; + if ($this->shareTables) { + $tenantQuery = ' AND _tenant = :_tenant'; + } + $removeQuery .= "( _document = :uid_{$index} + {$tenantQuery} AND _type = '{$type}' - AND _permission IN (" . implode(', ', \array_map(function (string $i) use ($permissionsToRemove, $index, $type, &$removeBindKeys, &$removeBindValues) { + AND _permission IN (" . \implode(', ', \array_map(function (string $i) use ($permissionsToRemove, $index, $type, &$removeBindKeys, &$removeBindValues) { $bindKey = 'remove_' . $type . '_' . $index . '_' . $i; $removeBindKeys[] = ':' . $bindKey; $removeBindValues[$bindKey] = $permissionsToRemove[$i]; @@ -1219,7 +1264,7 @@ public function updateDocuments(string $collection, array $documents, int $batch $bindKey = 'add_' . $type . '_' . $index . '_' . $i; $addBindValues[$bindKey] = $permission; - $addQuery .= "(:uid_{$index}, '{$type}', :{$bindKey})"; + $addQuery .= "(:uid_{$index}, '{$type}', :{$bindKey}, :_tenant)"; if ($i !== \array_key_last($permissionsToAdd) || $type !== \array_key_last($additions)) { $addQuery .= ', '; @@ -1263,20 +1308,22 @@ public function updateDocuments(string $collection, array $documents, int $batch foreach ($removeBindValues as $key => $value) { $stmtRemovePermissions->bindValue($key, $value, $this->getPDOType($value)); } - + if ($this->shareTables) { + $stmtRemovePermissions->bindValue(':_tenant', $this->tenant); + } $stmtRemovePermissions->execute(); } if (!empty($addQuery)) { $stmtAddPermissions = $this->getPDO()->prepare(" - INSERT INTO {$this->getSQLTable($name . '_perms')} (`_document`, `_type`, `_permission`) + INSERT INTO {$this->getSQLTable($name . '_perms')} (`_document`, `_type`, `_permission`, `_tenant`) VALUES {$addQuery} "); foreach ($addBindValues as $key => $value) { $stmtAddPermissions->bindValue($key, $value, $this->getPDOType($value)); } - + $stmtAddPermissions->bindValue(':_tenant', $this->tenant); $stmtAddPermissions->execute(); } } @@ -1322,18 +1369,25 @@ public function increaseDocumentAttribute(string $collection, string $id, string $sql = " UPDATE {$this->getSQLTable($name)} SET `{$attribute}` = `{$attribute}` + :val - WHERE - _uid = :_uid - {$sqlMax} - {$sqlMin} + WHERE _uid = :_uid "; + if ($this->shareTables) { + $sql .= ' AND _tenant = :_tenant'; + } + + $sql .= $sqlMax . $sqlMin; + $sql = $this->trigger(Database::EVENT_DOCUMENT_UPDATE, $sql); $stmt = $this->getPDO()->prepare($sql); $stmt->bindValue(':_uid', $id); $stmt->bindValue(':val', $value); + if ($this->shareTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } + $stmt->execute() || throw new DatabaseException('Failed to update attribute'); return true; } @@ -1356,22 +1410,38 @@ public function deleteDocument(string $collection, string $id): bool WHERE _uid = :_uid "; + if ($this->shareTables) { + $sql .= ' AND _tenant = :_tenant'; + } + $sql = $this->trigger(Database::EVENT_DOCUMENT_DELETE, $sql); $stmt = $this->getPDO()->prepare($sql); $stmt->bindValue(':_uid', $id); + if ($this->shareTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } + $sql = " DELETE FROM {$this->getSQLTable($name . '_perms')} WHERE _document = :_uid "; + if ($this->shareTables) { + $sql .= ' AND _tenant = :_tenant'; + } + $sql = $this->trigger(Database::EVENT_PERMISSIONS_DELETE, $sql); $stmtPermissions = $this->getPDO()->prepare($sql); $stmtPermissions->bindValue(':_uid', $id); + if ($this->shareTables) { + $stmtPermissions->bindValue(':_tenant', $this->tenant); + } + try { $this->getPDO()->beginTransaction(); @@ -1379,7 +1449,7 @@ public function deleteDocument(string $collection, string $id): bool throw new DatabaseException('Failed to delete document'); } if (!$stmtPermissions->execute()) { - throw new DatabaseException('Failed to clean permissions'); + throw new DatabaseException('Failed to delete permissions'); } $this->getPDO()->commit(); @@ -1416,6 +1486,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $orderAttributes = \array_map(fn ($orderAttribute) => match ($orderAttribute) { '$id' => '_uid', '$internalId' => '_id', + '$tenant' => '_tenant', '$createdAt' => '_createdAt', '$updatedAt' => '_updatedAt', default => $orderAttribute @@ -1499,6 +1570,10 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $where[] = $this->getSQLPermissionsCondition($name, $roles); } + if ($this->shareTables) { + $where[] = "table_main._tenant = :_tenant"; + } + $sqlWhere = !empty($where) ? 'WHERE ' . implode(' AND ', $where) : ''; $sqlOrder = 'ORDER BY ' . implode(', ', $orders); $sqlLimit = \is_null($limit) ? '' : 'LIMIT :limit'; @@ -1521,6 +1596,9 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, foreach ($queries as $query) { $this->bindConditionValue($stmt, $query); } + if ($this->shareTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } if (!empty($cursor) && !empty($orderAttributes) && array_key_exists(0, $orderAttributes)) { $attribute = $orderAttributes[0]; @@ -1528,6 +1606,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $attribute = match ($attribute) { '_uid' => '$id', '_id' => '$internalId', + '_tenant' => '$tenant', '_createdAt' => '$createdAt', '_updatedAt' => '$updatedAt', default => $attribute @@ -1616,6 +1695,10 @@ public function count(string $collection, array $queries = [], ?int $max = null) $where[] = $this->getSQLPermissionsCondition($name, $roles); } + if ($this->shareTables) { + $where[] = "table_main._tenant = :_tenant"; + } + $sqlWhere = !empty($where) ? 'WHERE ' . \implode(' AND ', $where) : ''; @@ -1624,7 +1707,7 @@ public function count(string $collection, array $queries = [], ?int $max = null) SELECT COUNT(1) as sum FROM ( SELECT 1 FROM {$this->getSQLTable($name)} table_main - " . $sqlWhere . " + {$sqlWhere} {$limit} ) table_count "; @@ -1637,6 +1720,10 @@ public function count(string $collection, array $queries = [], ?int $max = null) $this->bindConditionValue($stmt, $query); } + if ($this->shareTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } + if (!\is_null($max)) { $stmt->bindValue(':max', $max, PDO::PARAM_INT); } @@ -1678,6 +1765,10 @@ public function sum(string $collection, string $attribute, array $queries = [], $where[] = $this->getSQLPermissionsCondition($name, $roles); } + if ($this->shareTables) { + $where[] = "table_main._tenant = :_tenant"; + } + $sqlWhere = !empty($where) ? 'WHERE ' . \implode(' AND ', $where) : ''; @@ -1699,6 +1790,10 @@ public function sum(string $collection, string $attribute, array $queries = [], $this->bindConditionValue($stmt, $query); } + if ($this->shareTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } + if (!\is_null($max)) { $stmt->bindValue(':max', $max, PDO::PARAM_INT); } @@ -1882,7 +1977,7 @@ protected function getSQLIndex(string $collection, string $id, string $type, arr $attributes = \implode(', ', $attributes); - return "CREATE {$type} `{$id}` ON {$this->getSQLTable($collection)} ({$attributes}, _tenant)"; + return "CREATE {$type} `{$id}` ON {$this->getSQLTable($collection)} ({$attributes})"; } /** diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 98d9508da..d840919f8 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1128,7 +1128,7 @@ public function updateDocuments(string $collection, array $documents, int $batch "); $permissionsStmt->bindValue(':_uid', $document->getId()); $permissionsStmt->execute(); - $permissions = $permissionsStmt->fetchAll(PDO::FETCH_ASSOC); + $permissions = $permissionsStmt->fetchAll(); $initial = []; foreach (Database::PERMISSIONS as $type) { diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 24276e6ae..b4d90697c 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -108,29 +108,23 @@ public function getDocument(string $collection, string $id, array $queries = []) { $name = $this->filter($collection); $selections = $this->getAttributeSelections($queries); - $filters = Query::groupByType($queries)['filters']; - $tenantQuery = null; - $tenantWhere = ''; + $sql = " + SELECT {$this->getAttributeProjection($selections)} + FROM {$this->getSQLTable($name)} + WHERE _uid = :_uid + "; - foreach ($filters as $query) { - if ($query->getAttribute() === '$tenant') { - $tenantQuery = $query; - $tenantWhere = 'AND _tenant = :_tenant'; - } + if ($this->shareTables) { + $sql .= "AND _tenant = :_tenant"; } - $stmt = $this->getPDO()->prepare(" - SELECT {$this->getAttributeProjection($selections)} - FROM {$this->getSQLTable($name)} - WHERE _uid = :_uid - {$tenantWhere} - "); + $stmt = $this->getPDO()->prepare($sql); $stmt->bindValue(':_uid', $id); - if (!empty($tenantQuery)) { - $stmt->bindValue(':_tenant', $tenantQuery->getValue()); + if ($this->shareTables) { + $stmt->bindValue(':_tenant', $this->getTenant()); } $stmt->execute(); @@ -152,6 +146,10 @@ public function getDocument(string $collection, string $id, array $queries = []) $document['$id'] = $document['_uid']; unset($document['_uid']); } + if (\array_key_exists('_tenant', $document)) { + $document['$tenant'] = $document['_tenant']; + unset($document['_tenant']); + } if (\array_key_exists('_createdAt', $document)) { $document['$createdAt'] = $document['_createdAt']; unset($document['_createdAt']); @@ -284,7 +282,7 @@ public function getCountOfIndexes(Document $collection): int */ public static function getCountOfDefaultAttributes(): int { - return 4; + return 5; } /** @@ -294,7 +292,7 @@ public static function getCountOfDefaultAttributes(): int */ public static function getCountOfDefaultIndexes(): int { - return 5; + return 6; } /** @@ -329,34 +327,13 @@ public function getAttributeWidth(Document $collection): int foreach ($attributes as $attribute) { switch ($attribute['type']) { case Database::VAR_STRING: - switch (true) { - case ($attribute['size'] > 16777215): - // 8 bytes length + 4 bytes for LONGTEXT - $total += 12; - break; - - case ($attribute['size'] > 65535): - // 8 bytes length + 3 bytes for MEDIUMTEXT - $total += 11; - break; - - case ($attribute['size'] > $this->getMaxVarcharLength()): - // 8 bytes length + 2 bytes for TEXT - $total += 10; - break; - - case ($attribute['size'] > 255): - // $size = $size * 4; // utf8mb4 up to 4 bytes per char - // 8 bytes length + 2 bytes for VARCHAR(>255) - $total += ($attribute['size'] * 4) + 2; - break; - - default: - // $size = $size * 4; // utf8mb4 up to 4 bytes per char - // 8 bytes length + 1 bytes for VARCHAR(<=255) - $total += ($attribute['size'] * 4) + 1; - break; - } + $total += match (true) { + $attribute['size'] > 16777215 => 12, + $attribute['size'] > 65535 => 11, + $attribute['size'] > $this->getMaxVarcharLength() => 10, + $attribute['size'] > 255 => ($attribute['size'] * 4) + 2, + default => ($attribute['size'] * 4) + 1, + }; break; case Database::VAR_INTEGER: @@ -853,11 +830,18 @@ protected function getSQLIndexType(string $type): string protected function getSQLPermissionsCondition(string $collection, array $roles): string { $roles = array_map(fn (string $role) => $this->getPDO()->quote($role), $roles); + + $tenantQuery = ''; + if ($this->shareTables) { + $tenantQuery = 'AND _tenant = :_tenant'; + } + return "table_main._uid IN ( SELECT _document FROM {$this->getSQLTable($collection . '_perms')} WHERE _permission IN (" . implode(', ', $roles) . ") - AND _type = 'read' + AND _type = 'read' + {$tenantQuery} )"; } diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 37efef424..e10ae73fb 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -797,7 +797,7 @@ public function updateDocuments(string $collection, array $documents, int $batch "); $permissionsStmt->bindValue(':_uid', $document->getId()); $permissionsStmt->execute(); - $permissions = $permissionsStmt->fetchAll(PDO::FETCH_ASSOC); + $permissions = $permissionsStmt->fetchAll(); $initial = []; foreach (Database::PERMISSIONS as $type) { diff --git a/src/Database/Database.php b/src/Database/Database.php index 7a0a050ec..dfa17b139 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -3,7 +3,6 @@ namespace Utopia\Database; use Exception; -use InvalidArgumentException; use Utopia\Cache\Cache; use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Authorization as AuthorizationException; @@ -312,10 +311,6 @@ class Database protected ?\DateTime $timestamp = null; - protected bool $shareTables = false; - - protected string $tenant = ''; - protected bool $resolveRelationships = true; protected int $relationshipFetchDepth = 1; @@ -669,19 +664,19 @@ public function clearTimeout(string $event = Database::EVENT_ALL): void /** * Should tables be shared between users, segmented by tenant. * - * @param bool $enabled + * @param bool $share * @return self */ - public function setShareTables(bool $enabled): self + public function setShareTables(bool $share): self { - $this->shareTables = $enabled; + $this->adapter->setShareTables($share); return $this; } public function setTenant(string $tenant): self { - $this->tenant = $tenant; + $this->adapter->setTenant($tenant); return $this; } @@ -705,6 +700,10 @@ public function ping(): bool */ public function create(?string $database = null): bool { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $database = $database ?? $this->adapter->getDatabase(); $this->adapter->create($database); @@ -744,6 +743,10 @@ public function create(?string $database = null): bool */ public function exists(?string $database = null, ?string $collection = null): bool { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $database = $database ?? $this->adapter->getDatabase(); return $this->adapter->exists($database, $collection); @@ -756,6 +759,10 @@ public function exists(?string $database = null, ?string $collection = null): bo */ public function list(): array { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $databases = $this->adapter->list(); $this->trigger(self::EVENT_DATABASE_LIST, $databases); @@ -771,6 +778,10 @@ public function list(): array */ public function delete(?string $database = null): bool { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $database = $database ?? $this->adapter->getDatabase(); $deleted = $this->adapter->delete($database); @@ -794,18 +805,21 @@ public function delete(?string $database = null): bool * @return Document * @throws DatabaseException * @throws DuplicateException - * @throws InvalidArgumentException * @throws LimitException */ public function createCollection(string $id, array $attributes = [], array $indexes = [], array $permissions = null, bool $documentSecurity = true): Document { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $permissions ??= [ Permission::create(Role::any()), ]; $validator = new Permissions(); if (!$validator->isValid($permissions)) { - throw new InvalidArgumentException($validator->getDescription()); + throw new DatabaseException($validator->getDescription()); } $collection = $this->silent(fn () => $this->getCollection($id)); @@ -876,20 +890,30 @@ public function createCollection(string $id, array $attributes = [], array $inde * @param bool $documentSecurity * * @return Document - * @throws InvalidArgumentException * @throws ConflictException * @throws DatabaseException - * @throws InvalidArgumentException */ public function updateCollection(string $id, array $permissions, bool $documentSecurity): Document { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $validator = new Permissions(); if (!$validator->isValid($permissions)) { - throw new InvalidArgumentException($validator->getDescription()); + throw new DatabaseException($validator->getDescription()); } $collection = $this->silent(fn () => $this->getCollection($id)); + if ($collection->isEmpty()) { + throw new DatabaseException('Collection not found'); + } + + if ($this->adapter->getShareTables() && $collection->getAttribute('$tenant') !== $this->adapter->getTenant()) { + throw new DatabaseException('Collection not found'); + } + $collection ->setAttribute('$permissions', $permissions) ->setAttribute('documentSecurity', $documentSecurity); @@ -911,8 +935,18 @@ public function updateCollection(string $id, array $permissions, bool $documentS */ public function getCollection(string $id): Document { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $collection = $this->silent(fn () => $this->getDocument(self::METADATA, $id)); + if ($id !== self::METADATA + && $this->adapter->getShareTables() + && $collection->getAttribute('$tenant') !== $this->adapter->getTenant()) { + return new Document(); + } + $this->trigger(self::EVENT_COLLECTION_READ, $collection); return $collection; @@ -929,11 +963,21 @@ public function getCollection(string $id): Document */ public function listCollections(int $limit = 25, int $offset = 0): array { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $result = $this->silent(fn () => $this->find(self::METADATA, [ Query::limit($limit), Query::offset($offset) ])); + if ($this->adapter->getShareTables()) { + $result = \array_filter($result, function ($collection) { + return $collection->getAttribute('$tenant') === $this->adapter->getTenant(); + }); + } + $this->trigger(self::EVENT_COLLECTION_LIST, $result); return $result; @@ -948,7 +992,21 @@ public function listCollections(int $limit = 25, int $offset = 0): array */ public function getSizeOfCollection(string $collection): int { - return $this->adapter->getSizeOfCollection($collection); + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + + $collection = $this->silent(fn () => $this->getCollection($collection)); + + if ($collection->isEmpty()) { + throw new DatabaseException('Collection not found'); + } + + if ($this->adapter->getShareTables() && $collection->getAttribute('$tenant') !== $this->adapter->getTenant()) { + throw new DatabaseException('Collection not found'); + } + + return $this->adapter->getSizeOfCollection($collection->getId()); } /** @@ -960,8 +1018,20 @@ public function getSizeOfCollection(string $collection): int */ public function deleteCollection(string $id): bool { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $collection = $this->silent(fn () => $this->getDocument(self::METADATA, $id)); + if ($collection->isEmpty()) { + throw new DatabaseException('Collection not found'); + } + + if ($this->adapter->getShareTables() && $collection->getAttribute('$tenant') !== $this->adapter->getTenant()) { + throw new DatabaseException('Collection not found'); + } + $relationships = \array_filter( $collection->getAttribute('attributes'), fn ($attribute) => @@ -1006,12 +1076,20 @@ public function deleteCollection(string $id): bool */ public function createAttribute(string $collection, string $id, string $type, int $size, bool $required, mixed $default = null, bool $signed = true, bool $array = false, string $format = null, array $formatOptions = [], array $filters = []): bool { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $collection = $this->silent(fn () => $this->getCollection($collection)); if ($collection->isEmpty()) { throw new DatabaseException('Collection not found'); } + if ($this->adapter->getShareTables() && $collection->getAttribute('$tenant') !== $this->adapter->getTenant()) { + throw new DatabaseException('Collection not found'); + } + // attribute IDs are case insensitive $attributes = $collection->getAttribute('attributes', []); /** @var array $attributes */ @@ -1179,11 +1257,16 @@ protected function validateDefaultTypes(string $type, mixed $default): void * @throws ConflictException * @throws DatabaseException */ - private function updateIndexMeta(string $collection, string $id, callable $updateCallback): Document + protected function updateIndexMeta(string $collection, string $id, callable $updateCallback): Document { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $collection = $this->silent(fn () => $this->getCollection($collection)); + if ($collection->getId() === self::METADATA) { - throw new DatabaseException('Cannot update metadata attributes'); + throw new DatabaseException('Cannot update metadata indexes'); } $indexes = $collection->getAttribute('indexes', []); @@ -1217,9 +1300,14 @@ private function updateIndexMeta(string $collection, string $id, callable $updat * @throws ConflictException * @throws DatabaseException */ - private function updateAttributeMeta(string $collection, string $id, callable $updateCallback): Document + protected function updateAttributeMeta(string $collection, string $id, callable $updateCallback): Document { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $collection = $this->silent(fn () => $this->getCollection($collection)); + if ($collection->getId() === self::METADATA) { throw new DatabaseException('Cannot update metadata attributes'); } @@ -1474,6 +1562,10 @@ public function updateAttribute(string $collection, string $id, string $type = n */ public function checkAttribute(Document $collection, Document $attribute): bool { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $collection = clone $collection; $collection->setAttribute('attributes', $attribute, Document::SET_TYPE_APPEND); @@ -1507,6 +1599,10 @@ public function checkAttribute(Document $collection, Document $attribute): bool */ public function deleteAttribute(string $collection, string $id): bool { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $collection = $this->silent(fn () => $this->getCollection($collection)); $attributes = $collection->getAttribute('attributes', []); $indexes = $collection->getAttribute('indexes', []); @@ -1574,6 +1670,10 @@ public function deleteAttribute(string $collection, string $id): bool */ public function renameAttribute(string $collection, string $old, string $new): bool { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $collection = $this->silent(fn () => $this->getCollection($collection)); $attributes = $collection->getAttribute('attributes', []); $indexes = $collection->getAttribute('indexes', []); @@ -1648,6 +1748,10 @@ public function createRelationship( ?string $twoWayKey = null, string $onDelete = Database::RELATION_MUTATE_RESTRICT ): bool { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $collection = $this->silent(fn () => $this->getCollection($collection)); if ($collection->isEmpty()) { @@ -1835,6 +1939,10 @@ public function updateRelationship( ?bool $twoWay = null, ?string $onDelete = null ): bool { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + if ( \is_null($newKey) && \is_null($newTwoWayKey) @@ -2000,6 +2108,10 @@ public function updateRelationship( */ public function deleteRelationship(string $collection, string $id): bool { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $collection = $this->silent(fn () => $this->getCollection($collection)); $attributes = $collection->getAttribute('attributes', []); $relationship = null; @@ -2116,6 +2228,10 @@ public function deleteRelationship(string $collection, string $id): bool */ public function renameIndex(string $collection, string $old, string $new): bool { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $collection = $this->silent(fn () => $this->getCollection($collection)); $indexes = $collection->getAttribute('indexes', []); @@ -2175,6 +2291,10 @@ public function renameIndex(string $collection, string $old, string $new): bool */ public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = []): bool { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + if (empty($attributes)) { throw new DatabaseException('Missing attributes'); } @@ -2263,6 +2383,10 @@ public function createIndex(string $collection, string $id, string $type, array */ public function deleteIndex(string $collection, string $id): bool { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + $collection = $this->silent(fn () => $this->getCollection($collection)); $indexes = $collection->getAttribute('indexes', []); @@ -2300,11 +2424,8 @@ public function deleteIndex(string $collection, string $id): bool */ public function getDocument(string $collection, string $id, array $queries = []): Document { - if ($this->shareTables) { - if (empty($this->tenant)) { - throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); - } - $queries[] = Query::equal('$tenant', [$this->tenant]); + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } if ($collection === self::METADATA && $id === self::METADATA) { @@ -2379,7 +2500,7 @@ public function getDocument(string $collection, string $id, array $queries = []) $validator = new Authorization(self::PERMISSION_READ); $documentSecurity = $collection->getAttribute('documentSecurity', false); - $cacheKey = 'cache-' . $this->getNamespace() . ':' . $this->tenant . ':' . $collection->getId() . ':' . $id; + $cacheKey = 'cache-' . $this->getNamespace() . ':' . $this->adapter->getTenant() . ':' . $collection->getId() . ':' . $id; if (!empty($selections)) { $cacheKey .= ':' . \md5(\implode($selections)); @@ -2732,11 +2853,8 @@ private function populateDocumentRelationships(Document $collection, Document $d */ public function createDocument(string $collection, Document $document): Document { - if ($this->shareTables) { - if (empty($this->tenant)) { - throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); - } - $document->setAttribute('$tenant', $this->tenant); + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } $collection = $this->silent(fn () => $this->getCollection($collection)); @@ -2760,7 +2878,7 @@ public function createDocument(string $collection, Document $document): Document $validator = new Permissions(); if (!$validator->isValid($document->getPermissions())) { - throw new InvalidArgumentException($validator->getDescription()); + throw new DatabaseException($validator->getDescription()); } $structure = new Structure($collection); @@ -2800,6 +2918,10 @@ public function createDocument(string $collection, Document $document): Document */ public function createDocuments(string $collection, array $documents, int $batchSize = self::INSERT_BATCH_SIZE): array { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + if (empty($documents)) { return []; } @@ -2809,7 +2931,7 @@ public function createDocuments(string $collection, array $documents, int $batch $time = DateTime::now(); foreach ($documents as $key => $document) { - if ($this->shareTables) { + if ($this->adapter->getShareTables()) { if (empty($document->getAttribute('$tenant'))) { throw new DatabaseException('Missing tenant. Tenant must be included when isolation mode is set to "table".'); } @@ -3133,10 +3255,8 @@ private function relateDocumentsById( */ public function updateDocument(string $collection, string $id, Document $document): Document { - if ($this->shareTables) { - if (empty($document->getAttribute('$tenant'))) { - throw new DatabaseException('Missing tenant. Tenant must be included when isolation mode is set to "table".'); - } + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } if (!$document->getId() || !$id) { @@ -3146,12 +3266,13 @@ public function updateDocument(string $collection, string $id, Document $documen $time = DateTime::now(); $old = Authorization::skip(fn () => $this->silent(fn () => $this->getDocument($collection, $id))); // Skip ensures user does not need read permission for this $document = \array_merge($old->getArrayCopy(), $document->getArrayCopy()); - $document['$collection'] = $old->getAttribute('$collection'); // Make sure user doesn't switch collectionID + $document['$collection'] = $old->getAttribute('$collection'); // Make sure user doesn't switch collection ID $document['$tenant'] = $old->getAttribute('$tenant'); // Make sure user doesn't switch tenant $document['$createdAt'] = $old->getCreatedAt(); // Make sure user doesn't switch createdAt $document = new Document($document); $collection = $this->silent(fn () => $this->getCollection($collection)); + $relationships = \array_filter($collection->getAttribute('attributes', []), function ($attribute) { return $attribute['type'] === Database::VAR_RELATIONSHIP; }); @@ -3283,7 +3404,7 @@ public function updateDocument(string $collection, string $id, Document $documen $this->purgeRelatedDocuments($collection, $id); - $this->cache->purge('cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $id . ':*'); + $this->deleteCachedDocument($collection->getId(), $id); $this->trigger(self::EVENT_DOCUMENT_UPDATE, $document); @@ -3305,6 +3426,10 @@ public function updateDocument(string $collection, string $id, Document $documen */ public function updateDocuments(string $collection, array $documents, int $batchSize = self::INSERT_BATCH_SIZE): array { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + if (empty($documents)) { return []; } @@ -3313,10 +3438,8 @@ public function updateDocuments(string $collection, array $documents, int $batch $collection = $this->silent(fn () => $this->getCollection($collection)); foreach ($documents as $document) { - if ($this->shareTables) { - if (empty($document->getAttribute('$tenant'))) { - throw new DatabaseException('Missing tenant. Tenant must be included when isolation mode is set to "table".'); - } + if ($this->adapter->getShareTables() && empty($document->getAttribute('$tenant'))) { + throw new DatabaseException('Missing tenant. Tenant must be included when isolation mode is set to "table".'); } if (!$document->getId()) { @@ -3350,7 +3473,7 @@ public function updateDocuments(string $collection, array $documents, int $batch foreach ($documents as $key => $document) { $documents[$key] = $this->decode($collection, $document); - $this->cache->purge('cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $document->getId() . ':*'); + $this->deleteCachedDocument($collection->getId(), $document->getId()); } $this->trigger(self::EVENT_DOCUMENTS_UPDATE, $documents); @@ -3756,6 +3879,10 @@ private function getJunctionCollection(Document $collection, Document $relatedCo */ public function increaseDocumentAttribute(string $collection, string $id, string $attribute, int|float $value = 1, int|float|null $max = null): bool { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + if ($value <= 0) { // Can be a float throw new DatabaseException('Value must be numeric and greater than 0'); } @@ -3800,7 +3927,8 @@ public function increaseDocumentAttribute(string $collection, string $id, string $max = $max ? $max - $value : null; $result = $this->adapter->increaseDocumentAttribute($collection->getId(), $id, $attribute, $value, null, $max); - $this->cache->purge('cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $id . ':*'); + + $this->deleteCachedDocument($collection->getId(), $id); $this->trigger(self::EVENT_DOCUMENT_INCREASE, $document); @@ -3823,6 +3951,10 @@ public function increaseDocumentAttribute(string $collection, string $id, string */ public function decreaseDocumentAttribute(string $collection, string $id, string $attribute, int|float $value = 1, int|float|null $min = null): bool { + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); + } + if ($value <= 0) { // Can be a float throw new DatabaseException('Value must be numeric and greater than 0'); } @@ -3867,7 +3999,7 @@ public function decreaseDocumentAttribute(string $collection, string $id, string $min = $min ? $min + $value : null; $result = $this->adapter->increaseDocumentAttribute($collection->getId(), $id, $attribute, $value * -1, $min); - $this->cache->purge('cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $id . ':*'); + $this->deleteCachedDocument($collection->getId(), $id); $this->trigger(self::EVENT_DOCUMENT_DECREASE, $document); return $result; @@ -3889,14 +4021,12 @@ public function decreaseDocumentAttribute(string $collection, string $id, string */ public function deleteDocument(string $collection, string $id): bool { - $document = Authorization::skip(fn () => $this->silent(fn () => $this->getDocument($collection, $id))); - - if ($this->shareTables) { - if (empty($document->getAttribute('$tenant'))) { - throw new DatabaseException('Missing tenant. Tenant must be included when isolation mode is set to "table".'); - } + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } + $document = Authorization::skip(fn () => $this->silent(fn () => $this->getDocument($collection, $id))); + $collection = $this->silent(fn () => $this->getCollection($collection)); $validator = new Authorization(self::PERMISSION_DELETE); @@ -3929,7 +4059,7 @@ public function deleteDocument(string $collection, string $id): bool $deleted = $this->adapter->deleteDocument($collection->getId(), $id); $this->purgeRelatedDocuments($collection, $id); - $this->cache->purge('cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $id . ':*'); + $this->deleteCachedDocument($collection->getId(), $id); $this->trigger(self::EVENT_DOCUMENT_DELETE, $document); @@ -4312,7 +4442,7 @@ private function deleteCascade(Document $collection, Document $relatedCollection */ public function deleteCachedCollection(string $collection): bool { - return $this->cache->purge('cache-' . $this->getNamespace() . ':' . $collection . ':*'); + return $this->cache->purge('cache-' . $this->getNamespace() . ':' . $this->adapter->getTenant() . ':' . $collection . ':*'); } /** @@ -4326,7 +4456,7 @@ public function deleteCachedCollection(string $collection): bool */ public function deleteCachedDocument(string $collection, string $id): bool { - return $this->cache->purge('cache-' . $this->getNamespace() . ':' . $collection . ':' . $id . ':*'); + return $this->cache->purge('cache-' . $this->getNamespace() . ':' . $this->adapter->getTenant() . ':' . $collection . ':' . $id . ':*'); } /** @@ -4342,14 +4472,10 @@ public function deleteCachedDocument(string $collection, string $id): bool */ public function find(string $collection, array $queries = []): array { - if ($this->shareTables) { - if (empty($this->tenant)) { - throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); - } - $queries[] = Query::equal('$tenant', [$this->tenant]); + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } - $originalName = $collection; $collection = $this->silent(fn () => $this->getCollection($collection)); if ($collection->isEmpty()) { @@ -4469,6 +4595,8 @@ public function find(string $collection, array $queries = []): array } } + unset($query); + // Remove internal attributes which are not queried foreach ($queries as $query) { if ($query->getMethod() === Query::TYPE_SELECT) { @@ -4521,12 +4649,11 @@ public function findOne(string $collection, array $queries = []): false|Document */ public function count(string $collection, array $queries = [], ?int $max = null): int { - $collection = $this->silent(fn () => $this->getCollection($collection)); - - if ($collection->isEmpty()) { - throw new DatabaseException("Collection not found"); + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } + $collection = $this->silent(fn () => $this->getCollection($collection)); $attributes = $collection->getAttribute('attributes', []); $indexes = $collection->getAttribute('indexes', []); @@ -4566,12 +4693,11 @@ public function count(string $collection, array $queries = [], ?int $max = null) */ public function sum(string $collection, string $attribute, array $queries = [], ?int $max = null): float|int { - $collection = $this->silent(fn () => $this->getCollection($collection)); - - if ($collection->isEmpty()) { - throw new DatabaseException("Collection not found"); + if ($this->adapter->getShareTables() && empty($this->adapter->getTenant())) { + throw new DatabaseException('Missing tenant. Tenant must be set when table sharing is enabled.'); } + $collection = $this->silent(fn () => $this->getCollection($collection)); $attributes = $collection->getAttribute('attributes', []); $indexes = $collection->getAttribute('indexes', []); diff --git a/src/Database/Validator/Queries/Document.php b/src/Database/Validator/Queries/Document.php index 91f57a0f0..41c9f3f9b 100644 --- a/src/Database/Validator/Queries/Document.php +++ b/src/Database/Validator/Queries/Document.php @@ -5,7 +5,6 @@ use Exception; use Utopia\Database\Database; use Utopia\Database\Validator\Queries; -use Utopia\Database\Validator\Query\Filter; use Utopia\Database\Validator\Query\Select; class Document extends Queries @@ -22,12 +21,6 @@ public function __construct(array $attributes) 'type' => Database::VAR_STRING, 'array' => false, ]); - $attributes[] = new \Utopia\Database\Document([ - '$id' => '$tenant', - 'key' => '$tenant', - 'type' => Database::VAR_DATETIME, - 'array' => false, - ]); $attributes[] = new \Utopia\Database\Document([ '$id' => '$createdAt', 'key' => '$createdAt', @@ -43,7 +36,6 @@ public function __construct(array $attributes) $validators = [ new Select($attributes), - new Filter($attributes) ]; parent::__construct($validators); diff --git a/src/Database/Validator/Queries/Documents.php b/src/Database/Validator/Queries/Documents.php index a3211d6a3..0d1dc2384 100644 --- a/src/Database/Validator/Queries/Documents.php +++ b/src/Database/Validator/Queries/Documents.php @@ -36,12 +36,6 @@ public function __construct(array $attributes, array $indexes) 'type' => Database::VAR_STRING, 'array' => false, ]); - $attributes[] = new Document([ - '$id' => '$tenant', - 'key' => '$tenant', - 'type' => Database::VAR_STRING, - 'array' => false, - ]); $attributes[] = new Document([ '$id' => '$createdAt', 'key' => '$createdAt', diff --git a/tests/Database/Base.php b/tests/e2e/Adapter/Base.php similarity index 99% rename from tests/Database/Base.php rename to tests/e2e/Adapter/Base.php index 801f65256..3ce5e206f 100644 --- a/tests/Database/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -1,9 +1,8 @@ assertEquals(true, static::getDatabase()->exists($this->testDatabase)); $this->assertEquals(true, static::getDatabase()->delete($this->testDatabase)); $this->assertEquals(false, static::getDatabase()->exists($this->testDatabase)); - $this->assertEquals(true, static::getDatabase()->setDatabase($this->testDatabase)); + $this->assertEquals(static::getDatabase(), static::getDatabase()->setDatabase($this->testDatabase)); $this->assertEquals(true, static::getDatabase()->create()); } @@ -236,7 +236,7 @@ public function testQueryTimeout(): void for ($i = 0 ; $i <= 20 ; $i++) { static::getDatabase()->createDocument('global-timeouts', new Document([ - 'longtext' => file_get_contents(__DIR__ . '/../resources/longtext.txt'), + 'longtext' => file_get_contents(__DIR__ . '/../../resources/longtext.txt'), '$permissions' => [ Permission::read(Role::any()), Permission::update(Role::any()), @@ -245,7 +245,7 @@ public function testQueryTimeout(): void ])); } - $this->expectException(Timeout::class); + $this->expectException(TimeoutException::class); static::getDatabase()->setTimeout(1); @@ -253,7 +253,7 @@ public function testQueryTimeout(): void static::getDatabase()->find('global-timeouts', [ Query::notEqual('longtext', 'appwrite'), ]); - } catch(Timeout $ex) { + } catch(TimeoutException $ex) { static::getDatabase()->clearTimeout(); static::getDatabase()->deleteCollection('global-timeouts'); throw $ex; @@ -3921,7 +3921,7 @@ public function testGetAttributeLimit(): void public function testGetIndexLimit(): void { - $this->assertEquals(59, $this->getDatabase()->getLimitForIndexes()); + $this->assertEquals(58, $this->getDatabase()->getLimitForIndexes()); } public function testGetId(): void @@ -6802,24 +6802,6 @@ public function testManyToOneOneWayRelationship(): void $review5 = static::getDatabase()->getDocument('review', 'review5'); $this->assertEquals('Movie 5', $review5->getAttribute('movie')->getAttribute('name')); - - // Todo: fix this is failing - static::getDatabase()->updateDocument('review', $review5->getId(), new Document([ - '$id' => 'review5', - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()), - ], - 'name' => 'Review 5', - 'movie' => [], // - ])); - - - $this->assertEquals(true, false); - - - // Update document with new related document static::getDatabase()->updateDocument( 'review', @@ -11398,7 +11380,7 @@ public function testCollectionUpdate(): Document */ public function testCollectionUpdatePermissionsThrowException(Document $collection): void { - $this->expectException(InvalidArgumentException::class); + $this->expectException(DatabaseException::class); static::getDatabase()->updateCollection($collection->getId(), permissions: [ 'i dont work' ], documentSecurity: false); @@ -11428,7 +11410,7 @@ public function testCollectionPermissions(): Document public function testCollectionPermissionsExceptions(): void { - $this->expectException(InvalidArgumentException::class); + $this->expectException(DatabaseException::class); static::getDatabase()->createCollection('collectionSecurity', permissions: [ 'i dont work' ]); @@ -12641,6 +12623,11 @@ public function testIsolationModes(): void $docs = $database->find('people'); $this->assertEquals(1, \count($docs)); + + // Reset state + $database->setShareTables(false); + $database->setNamespace(static::$namespace); + $database->setDatabase($this->testDatabase); } public function testTransformations(): void diff --git a/tests/Database/Adapter/MariaDBTest.php b/tests/e2e/Adapter/MariaDBTest.php similarity index 85% rename from tests/Database/Adapter/MariaDBTest.php rename to tests/e2e/Adapter/MariaDBTest.php index e88a60bd2..e26ca69c2 100644 --- a/tests/Database/Adapter/MariaDBTest.php +++ b/tests/e2e/Adapter/MariaDBTest.php @@ -1,20 +1,19 @@ setDatabase('utopiaTests'); - $database->setNamespace('myapp_' . uniqid()); + $database->setNamespace(static::$namespace = 'myapp_' . uniqid()); if ($database->exists('utopiaTests')) { $database->delete('utopiaTests'); diff --git a/tests/Database/Adapter/MongoDBTest.php b/tests/e2e/Adapter/MongoDBTest.php similarity index 93% rename from tests/Database/Adapter/MongoDBTest.php rename to tests/e2e/Adapter/MongoDBTest.php index fced06b4b..edba83f6f 100644 --- a/tests/Database/Adapter/MongoDBTest.php +++ b/tests/e2e/Adapter/MongoDBTest.php @@ -1,20 +1,19 @@ setDatabase($schema); - $database->setNamespace('myapp_' . uniqid()); + $database->setNamespace(static::$namespace = 'myapp_' . uniqid()); if ($database->exists('utopiaTests')) { $database->delete('utopiaTests'); diff --git a/tests/Database/Adapter/MySQLTest.php b/tests/e2e/Adapter/MySQLTest.php similarity index 89% rename from tests/Database/Adapter/MySQLTest.php rename to tests/e2e/Adapter/MySQLTest.php index d527b75cd..8704b1567 100644 --- a/tests/Database/Adapter/MySQLTest.php +++ b/tests/e2e/Adapter/MySQLTest.php @@ -1,20 +1,19 @@ setDatabase('utopiaTests'); - $database->setNamespace('myapp_'.uniqid()); + $database->setNamespace(static::$namespace = 'myapp_' . uniqid()); if ($database->exists('utopiaTests')) { $database->delete('utopiaTests'); diff --git a/tests/Database/Adapter/PostgresTest.php b/tests/e2e/Adapter/PostgresTest.php similarity index 89% rename from tests/Database/Adapter/PostgresTest.php rename to tests/e2e/Adapter/PostgresTest.php index 91ebc84e6..6a12ecd8c 100644 --- a/tests/Database/Adapter/PostgresTest.php +++ b/tests/e2e/Adapter/PostgresTest.php @@ -1,18 +1,18 @@ setDatabase('utopiaTests'); - $database->setNamespace('myapp_'.uniqid()); + $database->setNamespace(static::$namespace = 'myapp_' . uniqid()); if ($database->exists('utopiaTests')) { $database->delete('utopiaTests'); diff --git a/tests/Database/Adapter/SQLiteTest.php b/tests/e2e/Adapter/SQLiteTest.php similarity index 89% rename from tests/Database/Adapter/SQLiteTest.php rename to tests/e2e/Adapter/SQLiteTest.php index 18f120f23..a3dd72b09 100644 --- a/tests/Database/Adapter/SQLiteTest.php +++ b/tests/e2e/Adapter/SQLiteTest.php @@ -1,20 +1,19 @@ setDatabase('utopiaTests'); - $database->setNamespace('myapp_'.uniqid()); + $database->setNamespace(static::$namespace = 'myapp_' . uniqid()); if ($database->exists()) { $database->delete(); diff --git a/tests/Database/DocumentTest.php b/tests/unit/DocumentTest.php similarity index 99% rename from tests/Database/DocumentTest.php rename to tests/unit/DocumentTest.php index acb9e7be0..4d2e517fc 100644 --- a/tests/Database/DocumentTest.php +++ b/tests/unit/DocumentTest.php @@ -1,10 +1,10 @@ Date: Fri, 17 Nov 2023 21:25:33 +1300 Subject: [PATCH 13/45] Fix test workflow --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7a8e79a97..e16d6a95c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -63,8 +63,8 @@ jobs: - name: Run Unit Tests run: docker compose exec appwrite test /usr/src/code/tests/unit - e2e_test: - name: E2E Tests + adapter_test: + name: Adapter Tests runs-on: ubuntu-latest needs: setup strategy: @@ -97,4 +97,4 @@ jobs: sleep 10 - name: Run ${{matrix.service}} Tests - run: docker compose exec -T tests vendor/bin/phpunit /usr/src/code/tests/Database/Adapter/${{matrix.service}}Test.php --debug \ No newline at end of file + run: docker compose exec -T tests vendor/bin/phpunit /usr/src/code/tests/e2e/Adapter/${{matrix.adapter}}Test.php --debug \ No newline at end of file From 4b825ec4e75df15cdf974594ec29a44874254fd4 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 17 Nov 2023 21:36:39 +1300 Subject: [PATCH 14/45] Fix unit test workflow --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e16d6a95c..f82d92b3d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,7 +61,7 @@ jobs: sleep 10 - name: Run Unit Tests - run: docker compose exec appwrite test /usr/src/code/tests/unit + run: docker compose exec tests vendor/bin/phpunit /usr/src/code/tests/unit adapter_test: name: Adapter Tests From 025eb5e64bf3bc7ff694125e8a21fae70e24656c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 21 Nov 2023 01:10:57 +1300 Subject: [PATCH 15/45] Postgres isolation modes --- composer.json | 3 +- phpunit.xml | 3 +- src/Database/Adapter.php | 4 +- src/Database/Adapter/MariaDB.php | 4 +- src/Database/Adapter/Postgres.php | 208 ++++++++++++++++++++++-------- src/Database/Adapter/SQL.php | 2 +- src/Database/Database.php | 33 ++--- 7 files changed, 177 insertions(+), 80 deletions(-) diff --git a/composer.json b/composer.json index 497e464e1..a94a99477 100755 --- a/composer.json +++ b/composer.json @@ -11,8 +11,7 @@ "autoload-dev": { "psr-4": { "Tests\\E2E\\": "tests/e2e", - "Tests\\Unit\\": "tests/unit", - "Appwrite\\Tests\\": "tests/extensions" + "Tests\\Unit\\": "tests/unit" } }, "scripts": { diff --git a/phpunit.xml b/phpunit.xml index 350af5c34..ccdaa969e 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,13 +7,12 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="true"> + stopOnFailure="false"> ./tests/unit - ./tests/e2e/Client.php ./tests/e2e/Adapter diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index e879ea499..d6c0ce588 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -160,7 +160,7 @@ public function getShareTables(): bool /** * Set Tenant. * - * Set tenant to use for current scope + * Set tenant to use if tables are shared * * @param string $tenant * @@ -176,7 +176,7 @@ public function setTenant(string $tenant): bool /** * Get Tenant. * - * Get Tenant from current scope + * Get tenant to use for shared tables * * @return string */ diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 9ea324e2f..8d0294a10 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -117,7 +117,7 @@ public function createCollection(string $name, array $attributes = [], array $in CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id)} ( `_id` INT(11) UNSIGNED NOT NULL AUTO_INCREMENT, `_uid` VARCHAR(255) NOT NULL, - `_tenant` CHAR(36) DEFAULT NULL, + `_tenant` VARCHAR(36) DEFAULT NULL, `_createdAt` DATETIME(3) DEFAULT NULL, `_updatedAt` DATETIME(3) DEFAULT NULL, `_permissions` MEDIUMTEXT DEFAULT NULL, @@ -142,7 +142,7 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = " CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id . '_perms')} ( `_id` int(11) UNSIGNED NOT NULL AUTO_INCREMENT, - `_tenant` CHAR(36) DEFAULT NULL, + `_tenant` VARCHAR(36) DEFAULT NULL, `_type` VARCHAR(12) NOT NULL, `_permission` VARCHAR(255) NOT NULL, `_document` VARCHAR(255) NOT NULL, diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index d840919f8..9871564a1 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -31,6 +31,7 @@ class Postgres extends SQL * @param string $name * * @return bool + * @throws DatabaseException */ public function create(string $name): bool { @@ -101,7 +102,7 @@ public function createCollection(string $name, array $attributes = [], array $in CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id)} ( \"_id\" SERIAL NOT NULL, \"_uid\" VARCHAR(255) NOT NULL, - \"_tenant\" SERIAL NOT NULL, + \"_tenant\" VARCHAR(36) DEFAULT NULL, \"_createdAt\" TIMESTAMP(3) DEFAULT NULL, \"_updatedAt\" TIMESTAMP(3) DEFAULT NULL, \"_permissions\" TEXT DEFAULT NULL, @@ -109,9 +110,9 @@ public function createCollection(string $name, array $attributes = [], array $in PRIMARY KEY (\"_id\") ); - CREATE UNIQUE INDEX \"{$namespace}_{$id}_uid\" on {$this->getSQLTable($id)} (LOWER(\"_uid\")); - CREATE INDEX \"{$namespace}_{$id}_created\" ON {$this->getSQLTable($id)} (\"_createdAt\"); - CREATE INDEX \"{$namespace}_{$id}_updated\" ON {$this->getSQLTable($id)} (\"_updatedAt\"); + CREATE UNIQUE INDEX \"{$namespace}_{$id}_uid\" ON {$this->getSQLTable($id)} (LOWER(\"_uid\"), \"_tenant\"); + CREATE INDEX \"{$namespace}_{$id}_created\" ON {$this->getSQLTable($id)} (\"_createdAt\", \"_tenant\"); + CREATE INDEX \"{$namespace}_{$id}_updated\" ON {$this->getSQLTable($id)} (\"_updatedAt\", \"_tenant\"); "; $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql); @@ -124,16 +125,16 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = " CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id . '_perms')} ( \"_id\" SERIAL NOT NULL, - \"_tenant\" SERIAL NOT NULL, + \"_tenant\" VARCHAR(36) DEFAULT NULL, \"_type\" VARCHAR(12) NOT NULL, \"_permission\" VARCHAR(255) NOT NULL, \"_document\" VARCHAR(255) NOT NULL, PRIMARY KEY (\"_id\") ); CREATE UNIQUE INDEX \"index_{$namespace}_{$id}_ukey\" - ON {$this->getSQLTable($id. '_perms')} USING btree (\"_document\",\"_type\",\"_permission\"); + ON {$this->getSQLTable($id. '_perms')} USING btree (\"_document\",\"_tenant\",\"_type\",\"_permission\"); CREATE INDEX \"index_{$namespace}_{$id}_permission\" - ON {$this->getSQLTable($id. '_perms')} USING btree (\"_permission\",\"_type\",\"_document\"); + ON {$this->getSQLTable($id. '_perms')} USING btree (\"_permission\",\"_type\",\"_tenant\",\"_document\"); "; $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql); @@ -657,9 +658,10 @@ public function renameIndex(string $collection, string $old, string $new): bool public function createDocument(string $collection, Document $document): Document { $attributes = $document->getAttributes(); + $attributes['_tenant'] = $this->tenant; $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); - $attributes['_permissions'] = json_encode($document->getPermissions()); + $attributes['_permissions'] = \json_encode($document->getPermissions()); $name = $this->filter($collection); $columns = ''; @@ -671,7 +673,7 @@ public function createDocument(string $collection, Document $document): Document * Insert Attributes */ - // Insert manual id if set + // Insert internal id if set if (!empty($document->getInternalId())) { $bindKey = '_id'; $columns .= "\"_id\", "; @@ -679,7 +681,7 @@ public function createDocument(string $collection, Document $document): Document } $bindIndex = 0; - foreach ($attributes as $attribute => $value) { // Parse statement + foreach ($attributes as $attribute => $value) { $column = $this->filter($attribute); $bindKey = 'key_' . $bindIndex; $columns .= "\"{$column}\", "; @@ -688,8 +690,9 @@ public function createDocument(string $collection, Document $document): Document } $sql = " - INSERT INTO {$this->getSQLTable($name)} ({$columns}\"_uid\") - VALUES ({$columnNames}:_uid) RETURNING _id + INSERT INTO {$this->getSQLTable($name)} ({$columns} \"_uid\") + VALUES ({$columnNames} :_uid) + RETURNING _id "; $sql = $this->trigger(Database::EVENT_DOCUMENT_CREATE, $sql); @@ -698,19 +701,17 @@ public function createDocument(string $collection, Document $document): Document $stmt->bindValue(':_uid', $document->getId(), PDO::PARAM_STR); - // Bind manual internal id if set if (!empty($document->getInternalId())) { $stmt->bindValue(':_id', $document->getInternalId(), PDO::PARAM_STR); } $attributeIndex = 0; - foreach ($attributes as $attribute => $value) { - if (is_array($value)) { // arrays & objects should be saved as strings - $value = json_encode($value); + foreach ($attributes as $value) { + if (is_array($value)) { + $value = \json_encode($value); } $bindKey = 'key_' . $attributeIndex; - $attribute = $this->filter($attribute); $value = (is_bool($value)) ? ($value ? "true" : "false") : $value; $stmt->bindValue(':' . $bindKey, $value, $this->getPDOType($value)); $attributeIndex++; @@ -720,19 +721,22 @@ public function createDocument(string $collection, Document $document): Document foreach (Database::PERMISSIONS as $type) { foreach ($document->getPermissionsByType($type) as $permission) { $permission = \str_replace('"', '', $permission); - $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}')"; + $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}', :_tenant)"; } } if (!empty($permissions)) { + $permissions = \implode(', ', $permissions); + $queryPermissions = " - INSERT INTO {$this->getSQLTable($name . '_perms')} - (_type, _permission, _document) VALUES " . implode(', ', $permissions); + INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document, _tenant) + VALUES {$permissions} + "; $queryPermissions = $this->trigger(Database::EVENT_PERMISSIONS_CREATE, $queryPermissions); - $stmtPermissions = $this->getPDO()->prepare($queryPermissions); + $stmtPermissions->bindValue(':_tenant', $this->tenant); } try { @@ -748,7 +752,6 @@ public function createDocument(string $collection, Document $document): Document case 23505: $this->getPDO()->rollBack(); throw new Duplicate('Duplicated document: ' . $e->getMessage()); - default: throw $e; } @@ -792,6 +795,7 @@ public function createDocuments(string $collection, array $documents, int $batch foreach ($batch as $document) { $attributes = $document->getAttributes(); + $attributes['_tenant'] = $this->tenant; $attributes['_uid'] = $document->getId(); $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); @@ -821,7 +825,7 @@ public function createDocuments(string $collection, array $documents, int $batch foreach (Database::PERMISSIONS as $type) { foreach ($document->getPermissionsByType($type) as $permission) { $permission = \str_replace('"', '', $permission); - $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}')"; + $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}', :_tenant)"; } } } @@ -841,9 +845,10 @@ public function createDocuments(string $collection, array $documents, int $batch if (!empty($permissions)) { $stmtPermissions = $this->getPDO()->prepare( " - INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document) + INSERT INTO {$this->getSQLTable($name . '_perms')} (_type, _permission, _document, _tenant) VALUES " . \implode(', ', $permissions) ); + $stmtPermissions->bindValue(':_tenant', $this->tenant); $stmtPermissions?->execute(); } } @@ -875,6 +880,7 @@ public function createDocuments(string $collection, array $documents, int $batch public function updateDocument(string $collection, Document $document): Document { $attributes = $document->getAttributes(); + $attributes['_tenant'] = $this->tenant; $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = json_encode($document->getPermissions()); @@ -884,10 +890,14 @@ public function updateDocument(string $collection, Document $document): Document $sql = " SELECT _type, _permission - FROM {$this->getSQLTable($name . '_perms')} p - WHERE p._document = :_uid + FROM {$this->getSQLTable($name . '_perms')} + WHERE _document = :_uid "; + if ($this->shareTables) { + $sql .= ' AND _tenant = :_tenant'; + } + $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql); /** @@ -895,6 +905,11 @@ public function updateDocument(string $collection, Document $document): Document */ $permissionsStmt = $this->getPDO()->prepare($sql); $permissionsStmt->bindValue(':_uid', $document->getId()); + + if ($this->shareTables) { + $permissionsStmt->bindValue(':_tenant', $this->tenant); + } + $permissionsStmt->execute(); $permissions = $permissionsStmt->fetchAll(); $permissionsStmt->closeCursor(); @@ -952,17 +967,27 @@ public function updateDocument(string $collection, Document $document): Document } if (!empty($removeQuery)) { $removeQuery .= ')'; - $removeQuery = " + + $sql = " DELETE FROM {$this->getSQLTable($name . '_perms')} - WHERE - _document = :_uid - {$removeQuery} + WHERE _document = :_uid "; + + if ($this->shareTables) { + $sql .= ' AND _tenant = :_tenant'; + } + + $removeQuery = $sql . $removeQuery; + $removeQuery = $this->trigger(Database::EVENT_PERMISSIONS_DELETE, $removeQuery); $stmtRemovePermissions = $this->getPDO()->prepare($removeQuery); $stmtRemovePermissions->bindValue(':_uid', $document->getId()); + if ($this->shareTables) { + $stmtRemovePermissions->bindValue(':_tenant', $this->tenant); + } + foreach ($removals as $type => $permissions) { foreach ($permissions as $i => $permission) { $stmtRemovePermissions->bindValue(":_remove_{$type}_{$i}", $permission); @@ -977,19 +1002,20 @@ public function updateDocument(string $collection, Document $document): Document $values = []; foreach ($additions as $type => $permissions) { foreach ($permissions as $i => $_) { - $values[] = "( :_uid, '{$type}', :_add_{$type}_{$i} )"; + $values[] = "( :_uid, '{$type}', :_add_{$type}_{$i}, :_tenant)"; } } $sql = " - INSERT INTO {$this->getSQLTable($name . '_perms')} - (_document, _type, _permission) VALUES" . \implode(', ', $values); + INSERT INTO {$this->getSQLTable($name . '_perms')} (_document, _type, _permission, _tenant) + VALUES" . \implode(', ', $values); $sql = $this->trigger(Database::EVENT_PERMISSIONS_CREATE, $sql); $stmtAddPermissions = $this->getPDO()->prepare($sql); - $stmtAddPermissions->bindValue(":_uid", $document->getId()); + $stmtAddPermissions->bindValue(':_tenant', $this->tenant); + foreach ($additions as $type => $permissions) { foreach ($permissions as $i => $permission) { $stmtAddPermissions->bindValue(":_add_{$type}_{$i}", $permission); @@ -1011,23 +1037,31 @@ public function updateDocument(string $collection, Document $document): Document $sql = " UPDATE {$this->getSQLTable($name)} - SET {$columns} _uid = :_uid WHERE _uid = :_uid + SET {$columns} _uid = :_uid + WHERE _uid = :_uid "; + if ($this->shareTables) { + $sql .= ' AND _tenant = :_tenant'; + } + $sql = $this->trigger(Database::EVENT_DOCUMENT_UPDATE, $sql); $stmt = $this->getPDO()->prepare($sql); $stmt->bindValue(':_uid', $document->getId()); + if ($this->shareTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } + $attributeIndex = 0; foreach ($attributes as $attribute => $value) { - if (is_array($value)) { // arrays & objects should be saved as strings + if (is_array($value)) { $value = json_encode($value); } $bindKey = 'key_' . $attributeIndex; - $attribute = $this->filter($attribute); $value = (is_bool($value)) ? ($value == true ? "true" : "false") : $value; $stmt->bindValue(':' . $bindKey, $value, $this->getPDOType($value)); $attributeIndex++; @@ -1097,6 +1131,7 @@ public function updateDocuments(string $collection, array $documents, int $batch foreach ($batch as $index => $document) { $attributes = $document->getAttributes(); $attributes['_uid'] = $document->getId(); + $attributes['_tenant'] = $this->tenant; $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = json_encode($document->getPermissions()); @@ -1121,12 +1156,25 @@ public function updateDocuments(string $collection, array $documents, int $batch $batchKeys[] = '(' . implode(', ', $bindKeys) . ')'; // Permissions logic - $permissionsStmt = $this->getPDO()->prepare(" + $sql = " SELECT _type, _permission FROM {$this->getSQLTable($name . '_perms')} WHERE _document = :_uid - "); + "; + + if ($this->shareTables) { + $sql .= ' AND _tenant = :_tenant'; + } + + $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql); + + $permissionsStmt = $this->getPDO()->prepare($sql); $permissionsStmt->bindValue(':_uid', $document->getId()); + + if ($this->shareTables) { + $permissionsStmt->bindValue(':_tenant', $this->tenant); + } + $permissionsStmt->execute(); $permissions = $permissionsStmt->fetchAll(); @@ -1156,8 +1204,14 @@ public function updateDocuments(string $collection, array $documents, int $batch $removeBindKeys[] = ':uid_' . $index; $removeBindValues[$bindKey] = $document->getId(); + $tenantQuery = ''; + if ($this->shareTables) { + $tenantQuery = ' AND _tenant = :_tenant'; + } + $removeQuery .= "( _document = :uid_{$index} + {$tenantQuery} AND _type = '{$type}' AND _permission IN (" . implode(', ', \array_map(function (string $i) use ($permissionsToRemove, $index, $type, &$removeBindKeys, &$removeBindValues) { $bindKey = 'remove_' . $type . '_' . $index . '_' . $i; @@ -1198,7 +1252,7 @@ public function updateDocuments(string $collection, array $documents, int $batch $bindKey = 'add_' . $type . '_' . $index . '_' . $i; $addBindValues[$bindKey] = $permission; - $addQuery .= "(:uid_{$index}, '{$type}', :{$bindKey})"; + $addQuery .= "(:uid_{$index}, '{$type}', :{$bindKey}, :_tenant)"; if ($i !== \array_key_last($permissionsToAdd) || $type !== \array_key_last($additions)) { $addQuery .= ', '; @@ -1223,7 +1277,7 @@ public function updateDocuments(string $collection, array $documents, int $batch $stmt = $this->getPDO()->prepare(" INSERT INTO {$this->getSQLTable($name)} (" . \implode(", ", $columns) . ") VALUES " . \implode(', ', $batchKeys) . " - ON CONFLICT (LOWER(_uid)) DO UPDATE SET $updateClause + ON CONFLICT (LOWER(_uid), _tenant) DO UPDATE SET $updateClause "); foreach ($bindValues as $key => $value) { @@ -1242,20 +1296,23 @@ public function updateDocuments(string $collection, array $documents, int $batch foreach ($removeBindValues as $key => $value) { $stmtRemovePermissions->bindValue($key, $value, $this->getPDOType($value)); } + if ($this->shareTables) { + $stmtRemovePermissions->bindValue(':_tenant', $this->tenant); + } $stmtRemovePermissions->execute(); } if (!empty($addQuery)) { $stmtAddPermissions = $this->getPDO()->prepare(" - INSERT INTO {$this->getSQLTable($name . '_perms')} (_document, _type, _permission) + INSERT INTO {$this->getSQLTable($name . '_perms')} (_document, _type, _permission, _tenant) VALUES {$addQuery} "); foreach ($addBindValues as $key => $value) { $stmtAddPermissions->bindValue($key, $value, $this->getPDOType($value)); } - + $stmtAddPermissions->bindValue(':_tenant', $this->tenant); $stmtAddPermissions->execute(); } } @@ -1298,18 +1355,25 @@ public function increaseDocumentAttribute(string $collection, string $id, string $sql = " UPDATE {$this->getSQLTable($name)} SET \"{$attribute}\" = \"{$attribute}\" + :val - WHERE - _uid = :_uid - {$sqlMax} - {$sqlMin} + WHERE _uid = :_uid "; + if ($this->shareTables) { + $sql .= ' AND _tenant = :_tenant'; + } + + $sql .= $sqlMax . $sqlMin; + $sql = $this->trigger(Database::EVENT_DOCUMENT_UPDATE, $sql); $stmt = $this->getPDO()->prepare($sql); $stmt->bindValue(':_uid', $id); $stmt->bindValue(':val', $value); + if ($this->shareTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } + $stmt->execute() || throw new DatabaseException('Failed to update attribute'); return true; } @@ -1333,28 +1397,43 @@ public function deleteDocument(string $collection, string $id): bool WHERE _uid = :_uid "; - $sql = $this->trigger(Database::EVENT_DOCUMENT_DELETE, $sql); + if ($this->shareTables) { + $sql .= ' AND _tenant = :_tenant'; + } + $sql = $this->trigger(Database::EVENT_DOCUMENT_DELETE, $sql); $stmt = $this->getPDO()->prepare($sql); - $stmt->bindValue(':_uid', $id, PDO::PARAM_STR); + if ($this->shareTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } + $sql = " DELETE FROM {$this->getSQLTable($name . '_perms')} WHERE _document = :_uid "; + + if ($this->shareTables) { + $sql .= ' AND _tenant = :_tenant'; + } + $sql = $this->trigger(Database::EVENT_PERMISSIONS_DELETE, $sql); $stmtPermissions = $this->getPDO()->prepare($sql); $stmtPermissions->bindValue(':_uid', $id); + if ($this->shareTables) { + $stmtPermissions->bindValue(':_tenant', $this->tenant); + } + try { if (!$stmt->execute()) { throw new DatabaseException('Failed to delete document'); } if (!$stmtPermissions->execute()) { - throw new DatabaseException('Failed to clean permissions'); + throw new DatabaseException('Failed to delete permissions'); } } catch (\Throwable $th) { $this->getPDO()->rollBack(); @@ -1397,6 +1476,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $orderAttributes = \array_map(fn ($orderAttribute) => match ($orderAttribute) { '$id' => '_uid', '$internalId' => '_id', + '$tenant' => '_tenant', '$createdAt' => '_createdAt', '$updatedAt' => '_updatedAt', default => $orderAttribute @@ -1469,6 +1549,9 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $where[] = $this->getSQLCondition($query); } + if ($this->shareTables) { + $where[] = "table_main._tenant = :_tenant"; + } if (Authorization::$status) { $where[] = $this->getSQLPermissionsCondition($name, $roles); @@ -1495,6 +1578,9 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, foreach ($queries as $query) { $this->bindConditionValue($stmt, $query); } + if ($this->shareTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } if (!empty($cursor) && !empty($orderAttributes) && array_key_exists(0, $orderAttributes)) { $attribute = $orderAttributes[0]; @@ -1502,6 +1588,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $attribute = match ($attribute) { '_uid' => '$id', '_id' => '$internalId', + '_tenant' => '$tenant', '_createdAt' => '$createdAt', '_updatedAt' => '$updatedAt', default => $attribute @@ -1538,6 +1625,10 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $results[$index]['$internalId'] = $document['_id']; unset($results[$index]['_id']); } + if (\array_key_exists('_tenant', $document)) { + $results[$index]['$tenant'] = $document['_tenant']; + unset($results[$index]['_tenant']); + } if (\array_key_exists('_createdAt', $document)) { $results[$index]['$createdAt'] = $document['_createdAt']; unset($results[$index]['_createdAt']); @@ -1583,6 +1674,10 @@ public function count(string $collection, array $queries = [], ?int $max = null) $where[] = $this->getSQLCondition($query); } + if ($this->shareTables) { + $where[] = "table_main._tenant = :_tenant"; + } + if (Authorization::$status) { $where[] = $this->getSQLPermissionsCondition($name, $roles); } @@ -1592,7 +1687,7 @@ public function count(string $collection, array $queries = [], ?int $max = null) SELECT COUNT(1) as sum FROM ( SELECT 1 FROM {$this->getSQLTable($name)} table_main - " . $sqlWhere . " + {$sqlWhere} {$limit} ) table_count "; @@ -1604,6 +1699,9 @@ public function count(string $collection, array $queries = [], ?int $max = null) foreach ($queries as $query) { $this->bindConditionValue($stmt, $query); } + if ($this->shareTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } if (!\is_null($max)) { $stmt->bindValue(':max', $max, PDO::PARAM_INT); @@ -1639,6 +1737,10 @@ public function sum(string $collection, string $attribute, array $queries = [], $where[] = $this->getSQLCondition($query); } + if ($this->shareTables) { + $where[] = "table_main._tenant = :_tenant"; + } + if (Authorization::$status) { $where[] = $this->getSQLPermissionsCondition($name, $roles); } @@ -1663,6 +1765,9 @@ public function sum(string $collection, string $attribute, array $queries = [], foreach ($queries as $query) { $this->bindConditionValue($stmt, $query); } + if ($this->shareTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } if (!\is_null($max)) { $stmt->bindValue(':max', $max, PDO::PARAM_INT); @@ -1737,6 +1842,7 @@ protected function getSQLCondition(Query $query): string $query->setAttribute(match ($query->getAttribute()) { '$id' => '_uid', '$internalId' => '_id', + '$tenant' => '_tenant', '$createdAt' => '_createdAt', '$updatedAt' => '_updatedAt', default => $query->getAttribute() diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index b4d90697c..87c0921b9 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -48,7 +48,7 @@ public function ping(): bool * @param string $database * @param string|null $collection * @return bool - * @throws Exception + * @throws DatabaseException */ public function exists(string $database, ?string $collection = null): bool { diff --git a/src/Database/Database.php b/src/Database/Database.php index dfa17b139..86ed4cb00 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -213,21 +213,6 @@ class Database '$permissions', ]; - /** - * Each database resource is created within its own schema. - */ - public const ISOLATION_MODE_SCHEMA = 'schema'; - - /** - * Each database resource is created within a shared schema - */ - public const ISOLATION_MODE_SHARED = 'shared'; - - /** - * Each database resource is created within shared tables in a shared schema. - */ - public const ISOLATION_MODE_TABLE = 'table'; - /** * Parent Collection * Defines the structure for both system and custom collections @@ -565,8 +550,6 @@ public function setNamespace(string $namespace): self * Get namespace of current set scope * * @return string - * - * @throws DatabaseException */ public function getNamespace(): string { @@ -662,7 +645,9 @@ public function clearTimeout(string $event = Database::EVENT_ALL): void } /** - * Should tables be shared between users, segmented by tenant. + * Set Share Tables + * + * Set whether to share tables between tenants * * @param bool $share * @return self @@ -674,6 +659,14 @@ public function setShareTables(bool $share): self return $this; } + /** + * Set Tenant + * + * Set tenant to use if tables are shared + * + * @param string $tenant + * @return self + */ public function setTenant(string $tenant): self { $this->adapter->setTenant($tenant); @@ -858,7 +851,7 @@ public function createCollection(string $id, array $attributes = [], array $inde throw new LimitException('Index limit of ' . $this->adapter->getLimitForIndexes() . ' exceeded. Cannot create collection.'); } - // check attribute limits, if given + // Check attribute limits, if given if ($attributes) { if ( $this->adapter->getLimitForAttributes() > 0 && @@ -3755,7 +3748,7 @@ private function updateDocumentRelationships(Document $collection, Document $old $document->setAttribute($key, $value->getId()); } elseif (\is_null($value)) { break; - } elseif(empty($value)) { + } elseif (empty($value)) { throw new DatabaseException('Invalid value for relationship'); } else { From 3813cddc47a8799c4bad77de2f422634c9edcb96 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 21 Nov 2023 01:54:08 +1300 Subject: [PATCH 16/45] WIP SQLite isolation modes --- src/Database/Adapter/SQLite.php | 127 +++++++++++++++++++++++-------- tests/e2e/Adapter/Base.php | 2 +- tests/e2e/Adapter/SQLiteTest.php | 12 +-- 3 files changed, 103 insertions(+), 38 deletions(-) diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index e10ae73fb..c09e86b4b 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -129,12 +129,13 @@ public function createCollection(string $name, array $attributes = [], array $in $attributeStrings[$key] = "`{$attrId}` {$attrType}, "; } - $sql = " + $sql = " CREATE TABLE IF NOT EXISTS `{$namespace}_{$id}` ( `_id` INTEGER PRIMARY KEY AUTOINCREMENT, - `_uid` CHAR(255) NOT NULL, - `_createdAt` datetime(3) DEFAULT NULL, - `_updatedAt` datetime(3) DEFAULT NULL, + `_uid` VARCHAR(36) NOT NULL, + `_tenant` VARCHAR(36) DEFAULT NULL, + `_createdAt` DATETIME(3) DEFAULT NULL, + `_updatedAt` DATETIME(3) DEFAULT NULL, `_permissions` MEDIUMTEXT DEFAULT NULL".(!empty($attributes) ? ',' : '')." " . \substr(\implode(' ', $attributeStrings), 0, -2) . " ) @@ -146,11 +147,11 @@ public function createCollection(string $name, array $attributes = [], array $in ->prepare($sql) ->execute(); - $this->createIndex($id, '_index1', Database::INDEX_UNIQUE, ['_uid'], [], []); - $this->createIndex($id, '_created_at', Database::INDEX_KEY, ['_createdAt'], [], []); - $this->createIndex($id, '_updated_at', Database::INDEX_KEY, ['_updatedAt'], [], []); + $this->createIndex($id, '_index1', Database::INDEX_UNIQUE, ['_uid', '_tenant'], [], []); + $this->createIndex($id, '_created_at', Database::INDEX_KEY, ['_createdAt', '_tenant'], [], []); + $this->createIndex($id, '_updated_at', Database::INDEX_KEY, ['_updatedAt', '_tenant'], [], []); - foreach ($indexes as $key => $index) { + foreach ($indexes as $index) { $indexId = $this->filter($index->getId()); $indexType = $index->getAttribute('type'); $indexAttributes = $index->getAttribute('attributes', []); @@ -163,6 +164,7 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = " CREATE TABLE IF NOT EXISTS `{$namespace}_{$id}_perms` ( `_id` INTEGER PRIMARY KEY AUTOINCREMENT, + `_uid` VARCHAR(36) DEFAULT NULL, `_type` VARCHAR(12) NOT NULL, `_permission` VARCHAR(255) NOT NULL, `_document` VARCHAR(255) NOT NULL @@ -199,11 +201,15 @@ public function getSizeOfCollection(string $collection): int $permissions = $namespace . '_' . $collection . '_perms'; $collectionSize = $this->getPDO()->prepare(" - SELECT SUM(\"pgsize\") FROM \"dbstat\" WHERE name=:name; + SELECT SUM(\"pgsize\") + FROM \"dbstat\" + WHERE name = :name; "); $permissionsSize = $this->getPDO()->prepare(" - SELECT SUM(\"pgsize\") FROM \"dbstat\" WHERE name=:name; + SELECT SUM(\"pgsize\") + FROM \"dbstat\" + WHERE name = :name; "); $collectionSize->bindParam(':name', $name); @@ -423,6 +429,7 @@ public function deleteIndex(string $collection, string $id): bool public function createDocument(string $collection, Document $document): Document { $attributes = $document->getAttributes(); + $attributes['_tenant'] = $this->tenant; $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = json_encode($document->getPermissions()); @@ -465,7 +472,7 @@ public function createDocument(string $collection, Document $document): Document $stmt->bindValue(':_uid', $document->getId(), PDO::PARAM_STR); - // Bind manual internal id if set + // Bind internal id if set if (!empty($document->getInternalId())) { $stmt->bindValue(':_id', $document->getInternalId(), PDO::PARAM_STR); } @@ -487,20 +494,19 @@ public function createDocument(string $collection, Document $document): Document foreach (Database::PERMISSIONS as $type) { foreach ($document->getPermissionsByType($type) as $permission) { $permission = \str_replace('"', '', $permission); - $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}')"; + $permissions[] = "('{$type}', '{$permission}', '{$document->getId()}', :_tenant)"; } } if (!empty($permissions)) { $queryPermissions = " - INSERT INTO `{$this->getNamespace()}_{$name}_perms` (_type, _permission, _document) - VALUES " . \implode( - ', ', - $permissions - ); + INSERT INTO `{$this->getNamespace()}_{$name}_perms` (_type, _permission, _document, _tenant) + VALUES " . \implode(', ', $permissions); + $queryPermissions = $this->trigger(Database::EVENT_PERMISSIONS_CREATE, $queryPermissions); $stmtPermissions = $this->getPDO()->prepare($queryPermissions); + $stmtPermissions->bindValue(':_tenant', $this->tenant); } try { @@ -509,6 +515,7 @@ public function createDocument(string $collection, Document $document): Document $statment = $this->getPDO()->prepare("SELECT last_insert_rowid() AS id"); $statment->execute(); $last = $statment->fetch(); + $document['$internalId'] = $last['id']; if (isset($stmtPermissions)) { @@ -542,6 +549,7 @@ public function createDocument(string $collection, Document $document): Document public function updateDocument(string $collection, Document $document): Document { $attributes = $document->getAttributes(); + $attributes['_tenant'] = $this->tenant; $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); $attributes['_permissions'] = json_encode($document->getPermissions()); @@ -556,6 +564,10 @@ public function updateDocument(string $collection, Document $document): Document WHERE _document = :_uid "; + if ($this->shareTables) { + $sql .= " AND _tenant = :_tenant"; + } + $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql); /** @@ -563,6 +575,11 @@ public function updateDocument(string $collection, Document $document): Document */ $permissionsStmt = $this->getPDO()->prepare($sql); $permissionsStmt->bindValue(':_uid', $document->getId()); + + if ($this->shareTables) { + $permissionsStmt->bindValue(':_tenant', $this->tenant); + } + $permissionsStmt->execute(); $permissions = $permissionsStmt->fetchAll(); $permissionsStmt->closeCursor(); @@ -624,18 +641,26 @@ public function updateDocument(string $collection, Document $document): Document } if (!empty($removeQuery)) { $removeQuery .= ')'; - $removeQuery = " + $sql = " DELETE FROM `{$this->getNamespace()}_{$name}_perms` - WHERE - _document = :_uid - {$removeQuery} + WHERE _document = :_uid "; + + if ($this->shareTables) { + $sql .= " AND _tenant = :_tenant"; + } + + $removeQuery = $sql . $removeQuery; $removeQuery = $this->trigger(Database::EVENT_PERMISSIONS_DELETE, $removeQuery); $stmtRemovePermissions = $this->getPDO()->prepare($removeQuery); $stmtRemovePermissions->bindValue(':_uid', $document->getId()); + if ($this->shareTables) { + $stmtRemovePermissions->bindValue(':_tenant', $this->tenant); + } + foreach ($removals as $type => $permissions) { foreach ($permissions as $i => $permission) { $stmtRemovePermissions->bindValue(":_remove_{$type}_{$i}", $permission); @@ -650,19 +675,21 @@ public function updateDocument(string $collection, Document $document): Document $values = []; foreach ($additions as $type => $permissions) { foreach ($permissions as $i => $_) { - $values[] = "( :_uid, '{$type}', :_add_{$type}_{$i} )"; + $values[] = "(:_uid, '{$type}', :_add_{$type}_{$i}, :_tenant)"; } } $sql = " - INSERT INTO `{$this->getNamespace()}_{$name}_perms` - (_document, _type, _permission) VALUES " . \implode(', ', $values); + INSERT INTO `{$this->getNamespace()}_{$name}_perms` (_document, _type, _permission, _tenant) + VALUES " . \implode(', ', $values); $sql = $this->trigger(Database::EVENT_PERMISSIONS_CREATE, $sql); $stmtAddPermissions = $this->getPDO()->prepare($sql); $stmtAddPermissions->bindValue(":_uid", $document->getId()); + $stmtAddPermissions->bindValue(":_tenant", $this->tenant); + foreach ($additions as $type => $permissions) { foreach ($permissions as $i => $permission) { $stmtAddPermissions->bindValue(":_add_{$type}_{$i}", $permission); @@ -683,15 +710,24 @@ public function updateDocument(string $collection, Document $document): Document $sql = " UPDATE `{$this->getNamespace()}_{$name}` - SET {$columns} _uid = :_uid WHERE _uid = :_uid + SET {$columns} _uid = :_uid + WHERE _uid = :_uid "; + if ($this->shareTables) { + $sql .= " AND _tenant = :_tenant"; + } + $sql = $this->trigger(Database::EVENT_DOCUMENT_UPDATE, $sql); $stmt = $this->getPDO()->prepare($sql); $stmt->bindValue(':_uid', $document->getId()); + if ($this->shareTables) { + $stmt->bindValue(':_tenant', $this->tenant); + } + $attributeIndex = 0; foreach ($attributes as $attribute => $value) { if (is_array($value)) { // arrays & objects should be saved as strings @@ -717,7 +753,8 @@ public function updateDocument(string $collection, Document $document): Document $this->getPDO()->rollBack(); throw match ($e->getCode()) { - '1062', '23000' => new Duplicate('Duplicated document: ' . $e->getMessage()), + '1062', + '23000' => new Duplicate('Duplicated document: ' . $e->getMessage()), default => $e, }; } @@ -765,6 +802,7 @@ public function updateDocuments(string $collection, array $documents, int $batch foreach ($batch as $index => $document) { $attributes = $document->getAttributes(); + $attributes['_tenant'] = $this->tenant; $attributes['_uid'] = $document->getId(); $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); @@ -790,12 +828,29 @@ public function updateDocuments(string $collection, array $documents, int $batch $batchKeys[] = '(' . implode(', ', $bindKeys) . ')'; // Permissions logic - $permissionsStmt = $this->getPDO()->prepare(" + $sql = " SELECT _type, _permission FROM {$this->getSQLTable($name . '_perms')} WHERE _document = :_uid - "); + "; + + if ($this->shareTables) { + $sql .= " AND _tenant = :_tenant"; + } + + $sql = $this->trigger(Database::EVENT_PERMISSIONS_READ, $sql); + + /** + * Get current permissions from the database + */ + $permissionsStmt = $this->getPDO()->prepare($sql); + $permissionsStmt->bindValue(':_uid', $document->getId()); + + if ($this->shareTables) { + $permissionsStmt->bindValue(':_tenant', $this->tenant); + } + $permissionsStmt->execute(); $permissions = $permissionsStmt->fetchAll(); @@ -825,8 +880,15 @@ public function updateDocuments(string $collection, array $documents, int $batch $removeBindKeys[] = ':uid_' . $index; $removeBindValues[$bindKey] = $document->getId(); + + $tenantQuery = ''; + if ($this->shareTables) { + $tenantQuery = ' AND _tenant = :_tenant'; + } + $removeQuery .= "( _document = :uid_{$index} + {$tenantQuery} AND _type = '{$type}' AND _permission IN (" . implode(', ', \array_map(function (string $i) use ($permissionsToRemove, $index, $type, &$removeBindKeys, &$removeBindValues) { $bindKey = 'remove_' . $type . '_' . $index . '_' . $i; @@ -867,7 +929,7 @@ public function updateDocuments(string $collection, array $documents, int $batch $bindKey = 'add_' . $type . '_' . $index . '_' . $i; $addBindValues[$bindKey] = $permission; - $addQuery .= "(:uid_{$index}, '{$type}', :{$bindKey})"; + $addQuery .= "(:uid_{$index}, '{$type}', :{$bindKey}, :_tenant)"; if ($i !== \array_key_last($permissionsToAdd) || $type !== \array_key_last($additions)) { $addQuery .= ', '; @@ -892,7 +954,7 @@ public function updateDocuments(string $collection, array $documents, int $batch $stmt = $this->getPDO()->prepare(" INSERT INTO {$this->getSQLTable($name)} (" . \implode(", ", $columns) . ") VALUES " . \implode(', ', $batchKeys) . " - ON CONFLICT(_uid) DO UPDATE SET $updateClause + ON CONFLICT(_uid, _tenant) DO UPDATE SET $updateClause "); foreach ($bindValues as $key => $value) { @@ -911,6 +973,9 @@ public function updateDocuments(string $collection, array $documents, int $batch foreach ($removeBindValues as $key => $value) { $stmtRemovePermissions->bindValue($key, $value, $this->getPDOType($value)); } + if ($this->shareTables) { + $stmtRemovePermissions->bindValue(':_tenant', $this->tenant); + } $stmtRemovePermissions->execute(); } @@ -924,7 +989,7 @@ public function updateDocuments(string $collection, array $documents, int $batch foreach ($addBindValues as $key => $value) { $stmtAddPermissions->bindValue($key, $value, $this->getPDOType($value)); } - + $stmtAddPermissions->bindValue(':_tenant', $this->tenant); $stmtAddPermissions->execute(); } } diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 3ce5e206f..8f94f5bf0 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -63,7 +63,7 @@ public function testCreateExistsDelete(): void { $schemaSupport = $this->getDatabase()->getAdapter()->getSupportForSchemas(); if (!$schemaSupport) { - $this->assertEquals(true, static::getDatabase()->setDatabase($this->testDatabase)); + $this->assertEquals(static::getDatabase(), static::getDatabase()->setDatabase($this->testDatabase)); $this->assertEquals(true, static::getDatabase()->create()); return; } diff --git a/tests/e2e/Adapter/SQLiteTest.php b/tests/e2e/Adapter/SQLiteTest.php index a3dd72b09..8e16ce143 100644 --- a/tests/e2e/Adapter/SQLiteTest.php +++ b/tests/e2e/Adapter/SQLiteTest.php @@ -43,18 +43,18 @@ public static function getDatabase(): Database return self::$database; } - $sqliteDir = __DIR__."/database.sql"; + $db = __DIR__."/database.sql"; - if (file_exists($sqliteDir)) { - unlink($sqliteDir); + if (file_exists($db)) { + unlink($db); } - $dsn = $sqliteDir; - $dsn = 'memory'; // Overwrite for fast tests + $dsn = $db; + //$dsn = 'memory'; // Overwrite for fast tests $pdo = new PDO("sqlite:" . $dsn, null, null, SQLite::getPDOAttributes()); $redis = new Redis(); - $redis->connect('redis', 6379); + $redis->connect('redis'); $redis->flushAll(); $cache = new Cache(new RedisAdapter($redis)); From 5550a7cd2f1c1750ed49567786496d12ca10a99a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 22 Nov 2023 19:22:02 +1300 Subject: [PATCH 17/45] Fix SQLite isolation modes --- src/Database/Adapter/SQLite.php | 40 +++++++++++++++++++++++++++------ tests/e2e/Adapter/Base.php | 12 +++++++--- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index c09e86b4b..ddc021c14 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -129,8 +129,8 @@ public function createCollection(string $name, array $attributes = [], array $in $attributeStrings[$key] = "`{$attrId}` {$attrType}, "; } - $sql = " - CREATE TABLE IF NOT EXISTS `{$namespace}_{$id}` ( + $sql = " + CREATE TABLE IF NOT EXISTS `{$this->getSQLTable($id)}` ( `_id` INTEGER PRIMARY KEY AUTOINCREMENT, `_uid` VARCHAR(36) NOT NULL, `_tenant` VARCHAR(36) DEFAULT NULL, @@ -162,9 +162,9 @@ public function createCollection(string $name, array $attributes = [], array $in } $sql = " - CREATE TABLE IF NOT EXISTS `{$namespace}_{$id}_perms` ( + CREATE TABLE IF NOT EXISTS `{$this->getSQLTable($id)}_perms` ( `_id` INTEGER PRIMARY KEY AUTOINCREMENT, - `_uid` VARCHAR(36) DEFAULT NULL, + `_tenant` VARCHAR(36) DEFAULT NULL, `_type` VARCHAR(12) NOT NULL, `_permission` VARCHAR(255) NOT NULL, `_document` VARCHAR(255) NOT NULL @@ -243,14 +243,14 @@ public function deleteCollection(string $id): bool $this->getPDO()->rollBack(); } - $sql = "DROP TABLE `{$this->getNamespace()}_{$id}`"; + $sql = "DROP TABLE IF EXISTS `{$this->getSQLTable($id)}`"; $sql = $this->trigger(Database::EVENT_COLLECTION_DELETE, $sql); $this->getPDO() ->prepare($sql) ->execute(); - $sql = "DROP TABLE `{$this->getNamespace()}_{$id}_perms`"; + $sql = "DROP TABLE IF EXISTS `{$this->getSQLTable($id)}_perms`"; $sql = $this->trigger(Database::EVENT_COLLECTION_DELETE, $sql); $this->getPDO() @@ -385,6 +385,21 @@ public function createIndex(string $collection, string $id, string $type, array $name = $this->filter($collection); $id = $this->filter($id); + + // Workaround for no support for CREATE INDEX IF NOT EXISTS + $stmt = $this->getPDO()->prepare(" + SELECT name + FROM sqlite_master + WHERE type='index' + AND name=:_index; + "); + $stmt->bindValue(':_index', "{$this->getNamespace()}_{$name}_{$id}"); + $stmt->execute(); + $index = $stmt->fetch(); + if (!empty($index)) { + return true; + } + $sql = $this->getSQLIndex($name, $id, $type, $attributes); $sql = $this->trigger(Database::EVENT_INDEX_CREATE, $sql); @@ -982,7 +997,7 @@ public function updateDocuments(string $collection, array $documents, int $batch if (!empty($addQuery)) { $stmtAddPermissions = $this->getPDO()->prepare(" - INSERT INTO {$this->getSQLTable($name . '_perms')} (`_document`, `_type`, `_permission`) + INSERT INTO {$this->getSQLTable($name . '_perms')} (_document, _type, _permission, _tenant) VALUES {$addQuery} "); @@ -1142,6 +1157,17 @@ protected function getSQLPermissionsCondition(string $collection, array $roles): )"; } + /** + * Get SQL table + * + * @param string $name + * @return string + */ + protected function getSQLTable(string $name): string + { + return "{$this->getNamespace()}_{$name}"; + } + /** * Get list of keywords that cannot be used * Refference: https://www.sqlite.org/lang_keywords.html diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 8f94f5bf0..5fa229de1 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -12548,14 +12548,18 @@ public function testIsolationModes(): void ->setNamespace('') ->create(); - $this->assertEquals(true, $database->exists('schema1')); + if ($database->getAdapter()->getSupportForSchemas()) { + $this->assertEquals(true, $database->exists('schema1')); + } $database ->setDatabase('schema2') ->setNamespace('') ->create(); - $this->assertEquals(true, $database->exists('schema2')); + if ($database->getAdapter()->getSupportForSchemas()) { + $this->assertEquals(true, $database->exists('schema2')); + } /** * Table @@ -12571,7 +12575,9 @@ public function testIsolationModes(): void ->setTenant($tenant1) ->create(); - $this->assertEquals(true, $database->exists('sharedTables')); + if ($database->getAdapter()->getSupportForSchemas()) { + $this->assertEquals(true, $database->exists('sharedTables')); + } $database->createCollection('people', [ new Document([ From c3f1d2eba182237cc5be623cafc96bd2abb1fe84 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 22 Nov 2023 20:56:14 +1300 Subject: [PATCH 18/45] Mongo isolation modes --- src/Database/Adapter/Mongo.php | 103 +++++++++++++++++++++--------- src/Database/Adapter/SQLite.php | 36 +++++------ tests/e2e/Adapter/Base.php | 18 +++--- tests/e2e/Adapter/MongoDBTest.php | 2 +- 4 files changed, 100 insertions(+), 59 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 01479c145..19792e0f7 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -168,21 +168,18 @@ public function createCollection(string $name, array $attributes = [], array $in throw new DatabaseException($e->getMessage(), $e->getCode(), $e); } - $indexesCreated = $this->client->createIndexes($id, [ - [ - 'key' => ['_uid' => $this->getOrder(Database::ORDER_DESC)], - 'name' => '_uid', - 'unique' => true, - 'collation' => [ // https://docs.mongodb.com/manual/core/index-case-insensitive/#create-a-case-insensitive-index - 'locale' => 'en', - 'strength' => 1, - ] - ], - [ - 'key' => ['_permissions' => $this->getOrder(Database::ORDER_DESC)], - 'name' => '_permissions', + $indexesCreated = $this->client->createIndexes($id, [[ + 'key' => ['_uid' => $this->getOrder(Database::ORDER_DESC)], + 'name' => '_uid', + 'unique' => true, + 'collation' => [ // https://docs.mongodb.com/manual/core/index-case-insensitive/#create-a-case-insensitive-index + 'locale' => 'en', + 'strength' => 1, ] - ]); + ], [ + 'key' => ['_permissions' => $this->getOrder(Database::ORDER_DESC)], + 'name' => '_permissions', + ]]); if (!$indexesCreated) { return false; @@ -631,6 +628,11 @@ public function getDocument(string $collection, string $id, array $queries = []) $name = $this->getNamespace() . '_' . $this->filter($collection); $filters = ['_uid' => $id]; + + if ($this->shareTables) { + $filters['_tenant'] = $this->getTenant(); + } + $options = []; $selections = $this->getAttributeSelections($queries); @@ -664,7 +666,10 @@ public function createDocument(string $collection, Document $document): Document { $name = $this->getNamespace() . '_' . $this->filter($collection); $internalId = $document->getInternalId(); - $document->removeAttribute('$internalId'); + + $document + ->removeAttribute('$internalId') + ->setAttribute('$tenant', $this->getTenant()); $record = $this->replaceChars('$', '_', (array)$document); $record = $this->timeToMongo($record); @@ -698,7 +703,9 @@ public function createDocuments(string $collection, array $documents, int $batch $records = []; foreach ($documents as $document) { - $document->removeAttribute('$internalId'); + $document + ->removeAttribute('$internalId') + ->setAttribute('$tenant', $this->getTenant()); $record = $this->replaceChars('$', '_', (array)$document); $record = $this->timeToMongo($record); @@ -731,9 +738,15 @@ private function insertDocument(string $name, array $document): array try { $this->client->insert($name, $document); + $filters = []; + $filters['_uid'] = $document['_uid']; + if ($this->shareTables) { + $filters['_tenant'] = $this->getTenant(); + } + $result = $this->client->find( $name, - ['_uid' => $document['_uid']], + $filters, ['limit' => 1] )->cursor->firstBatch[0]; @@ -760,8 +773,14 @@ public function updateDocument(string $collection, Document $document): Document $record = $this->replaceChars('$', '_', $record); $record = $this->timeToMongo($record); + $filters = []; + $filters['_uid'] = $document->getId(); + if ($this->shareTables) { + $filters['_tenant'] = $this->getTenant(); + } + try { - $this->client->update($name, ['_uid' => $document->getId()], $record); + $this->client->update($name, $filters, $record); } catch (MongoException $e) { throw new Duplicate($e->getMessage()); } @@ -789,7 +808,13 @@ public function updateDocuments(string $collection, array $documents, int $batch $document = $this->replaceChars('$', '_', $document); $document = $this->timeToMongo($document); - $this->client->update($name, ['_uid' => $document['_uid']], $document); + $filters = []; + $filters['_uid'] = $document['_uid']; + if ($this->shareTables) { + $filters['_tenant'] = $this->getTenant(); + } + + $this->client->update($name, $filters, $document); $documents[$index] = new Document($document); } @@ -812,19 +837,23 @@ public function updateDocuments(string $collection, array $documents, int $batch public function increaseDocumentAttribute(string $collection, string $id, string $attribute, int|float $value, int|float|null $min = null, int|float|null $max = null): bool { $attribute = $this->filter($attribute); - $where = ['_uid' => $id]; + $filters = ['_uid' => $id]; + + if ($this->shareTables) { + $filters['_tenant'] = $this->getTenant(); + } if ($max) { - $where[$attribute] = ['$lte' => $max]; + $filters[$attribute] = ['$lte' => $max]; } if ($min) { - $where[$attribute] = ['$gte' => $min]; + $filters[$attribute] = ['$gte' => $min]; } $this->client->update( $this->getNamespace() . '_' . $this->filter($collection), - $where, + $filters, ['$inc' => [$attribute => $value]], ); @@ -844,7 +873,13 @@ public function deleteDocument(string $collection, string $id): bool { $name = $this->getNamespace() . '_' . $this->filter($collection); - $result = $this->client->delete($name, ['_uid' => $id]); + $filters = []; + $filters['_uid'] = $id; + if ($this->shareTables) { + $filters['_tenant'] = $this->getTenant(); + } + + $result = $this->client->delete($name, $filters); return (!!$result); } @@ -889,6 +924,10 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $filters = $this->buildFilters($queries); + if ($this->shareTables) { + $filters['_tenant'] = $this->getTenant(); + } + // permissions if (Authorization::$status) { // skip if authorization is disabled $roles = \implode('|', Authorization::getRoles()); @@ -1257,27 +1296,29 @@ protected function replaceChars(string $from, string $to, array $array): array if ($from === '_') { if (array_key_exists('_id', $array)) { $result['$internalId'] = (string)$array['_id']; - unset($result['_id']); } - if (array_key_exists('_uid', $array)) { $result['$id'] = $array['_uid']; - unset($result['_uid']); } + if (array_key_exists('_tenant', $array)) { + $result['$tenant'] = $array['_tenant']; + unset($result['_tenant']); + } } elseif ($from === '$') { if (array_key_exists('$id', $array)) { $result['_uid'] = $array['$id']; - unset($result['$id']); } - if (array_key_exists('$internalId', $array)) { $result['_id'] = new ObjectId($array['$internalId']); - unset($result['$internalId']); } + if (array_key_exists('$tenant', $array)) { + $result['_tenant'] = $array['$tenant']; + unset($result['$tenant']); + } } return $result; @@ -1597,7 +1638,7 @@ public static function getCountOfDefaultAttributes(): int */ public static function getCountOfDefaultIndexes(): int { - return 5; + return 6; } /** diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index ddc021c14..7541dec07 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -386,19 +386,19 @@ public function createIndex(string $collection, string $id, string $type, array $id = $this->filter($id); - // Workaround for no support for CREATE INDEX IF NOT EXISTS - $stmt = $this->getPDO()->prepare(" + // Workaround for no support for CREATE INDEX IF NOT EXISTS + $stmt = $this->getPDO()->prepare(" SELECT name FROM sqlite_master WHERE type='index' AND name=:_index; "); - $stmt->bindValue(':_index', "{$this->getNamespace()}_{$name}_{$id}"); - $stmt->execute(); - $index = $stmt->fetch(); - if (!empty($index)) { - return true; - } + $stmt->bindValue(':_index', "{$this->getNamespace()}_{$name}_{$id}"); + $stmt->execute(); + $index = $stmt->fetch(); + if (!empty($index)) { + return true; + } $sql = $this->getSQLIndex($name, $id, $type, $attributes); @@ -1157,16 +1157,16 @@ protected function getSQLPermissionsCondition(string $collection, array $roles): )"; } - /** - * Get SQL table - * - * @param string $name - * @return string - */ - protected function getSQLTable(string $name): string - { - return "{$this->getNamespace()}_{$name}"; - } + /** + * Get SQL table + * + * @param string $name + * @return string + */ + protected function getSQLTable(string $name): string + { + return "{$this->getNamespace()}_{$name}"; + } /** * Get list of keywords that cannot be used diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 5fa229de1..4faf73c50 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -12548,18 +12548,18 @@ public function testIsolationModes(): void ->setNamespace('') ->create(); - if ($database->getAdapter()->getSupportForSchemas()) { - $this->assertEquals(true, $database->exists('schema1')); - } + if ($database->getAdapter()->getSupportForSchemas()) { + $this->assertEquals(true, $database->exists('schema1')); + } $database ->setDatabase('schema2') ->setNamespace('') ->create(); - if ($database->getAdapter()->getSupportForSchemas()) { - $this->assertEquals(true, $database->exists('schema2')); - } + if ($database->getAdapter()->getSupportForSchemas()) { + $this->assertEquals(true, $database->exists('schema2')); + } /** * Table @@ -12575,9 +12575,9 @@ public function testIsolationModes(): void ->setTenant($tenant1) ->create(); - if ($database->getAdapter()->getSupportForSchemas()) { - $this->assertEquals(true, $database->exists('sharedTables')); - } + if ($database->getAdapter()->getSupportForSchemas()) { + $this->assertEquals(true, $database->exists('sharedTables')); + } $database->createCollection('people', [ new Document([ diff --git a/tests/e2e/Adapter/MongoDBTest.php b/tests/e2e/Adapter/MongoDBTest.php index edba83f6f..887c26ec2 100644 --- a/tests/e2e/Adapter/MongoDBTest.php +++ b/tests/e2e/Adapter/MongoDBTest.php @@ -72,7 +72,7 @@ public function testCreateExistsDelete(): void $this->assertNotNull(static::getDatabase()->create()); $this->assertEquals(true, static::getDatabase()->delete($this->testDatabase)); $this->assertEquals(true, static::getDatabase()->create()); - $this->assertEquals(true, static::getDatabase()->setDatabase($this->testDatabase)); + $this->assertEquals(static::getDatabase(), static::getDatabase()->setDatabase($this->testDatabase)); } public function testRenameAttribute(): void From 01443a3480b81f806c6760495bc89a15cff094f0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 23 Nov 2023 14:23:15 +1300 Subject: [PATCH 19/45] Fix image tag --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index af5a18be8..8b386c771 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: tests: container_name: tests - image: database-dev + image: databases-dev build: context: . networks: From 92acf5e76cbfd0a5693074e957c4ba3a134e9fa8 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Nov 2023 16:23:26 +1300 Subject: [PATCH 20/45] Conditional indexes for better query performance --- src/Database/Adapter/MariaDB.php | 71 +++++++++++++++-------- src/Database/Adapter/Postgres.php | 93 ++++++++++++++++++++++--------- src/Database/Adapter/SQLite.php | 52 +++++++++++------ 3 files changed, 150 insertions(+), 66 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 5a4c9cdf3..e886c908d 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -115,23 +115,34 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = " CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id)} ( - `_id` INT(11) UNSIGNED NOT NULL AUTO_INCREMENT, - `_uid` VARCHAR(255) NOT NULL, - `_tenant` VARCHAR(36) DEFAULT NULL, - `_createdAt` DATETIME(3) DEFAULT NULL, - `_updatedAt` DATETIME(3) DEFAULT NULL, - `_permissions` MEDIUMTEXT DEFAULT NULL, + _id INT(11) UNSIGNED NOT NULL AUTO_INCREMENT, + _uid VARCHAR(255) NOT NULL, + _tenant VARCHAR(36) DEFAULT NULL, + _createdAt DATETIME(3) DEFAULT NULL, + _updatedAt DATETIME(3) DEFAULT NULL, + _permissions MEDIUMTEXT DEFAULT NULL, + PRIMARY KEY (_id), " . \implode(' ', $attributeStrings) . " - PRIMARY KEY (`_id`), " . \implode(' ', $indexStrings) . " - UNIQUE KEY `_uid` (`_uid`), - KEY `_tenant` (`_tenant`), - KEY `_created_at` (`_createdAt`, `_tenant`), - KEY `_updated_at` (`_updatedAt`, `_tenant`), - KEY `_uid_tenant` (`_uid`, `_tenant`) - ) "; + if ($this->shareTables) { + $sql .= " + UNIQUE KEY _uid_tenant (_tenant,_uid), + KEY _tenant (_tenant), + KEY _created_at (_tenant, _createdAt), + KEY _updated_at (_tenant, _updatedAt) + "; + } else { + $sql .= " + UNIQUE KEY _uid (_uid), + KEY _created_at (_createdAt), + KEY _updated_at (_updatedAt) + "; + } + + $sql .= ")"; + $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql); try { @@ -141,17 +152,28 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = " CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id . '_perms')} ( - `_id` int(11) UNSIGNED NOT NULL AUTO_INCREMENT, - `_tenant` VARCHAR(36) DEFAULT NULL, - `_type` VARCHAR(12) NOT NULL, - `_permission` VARCHAR(255) NOT NULL, - `_document` VARCHAR(255) NOT NULL, - PRIMARY KEY (`_id`), - UNIQUE INDEX `_index1` (`_document`, `_tenant`, `_type`,`_permission`), - INDEX `_permission` (`_permission`, `_type`, `_tenant`, `_document`) - ) + _id int(11) UNSIGNED NOT NULL AUTO_INCREMENT, + _tenant VARCHAR(36) DEFAULT NULL, + _type VARCHAR(12) NOT NULL, + _permission VARCHAR(255) NOT NULL, + _document VARCHAR(255) NOT NULL, + PRIMARY KEY (_id), "; + if ($this->shareTables) { + $sql .= " + UNIQUE INDEX _index1 (_document, _tenant, _type, _permission), + INDEX _permission (_permission, _type, _document, _tenant) + "; + } else { + $sql .= " + UNIQUE INDEX _index1 (_document, _type, _permission), + INDEX _permission (_permission, _type, _document) + "; + } + + $sql .= ")"; + $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql); $this->getPDO() @@ -1977,6 +1999,11 @@ protected function getSQLIndex(string $collection, string $id, string $type, arr $attributes = \implode(', ', $attributes); + if ($this->shareTables) { + // Add tenant as first index column for best performance + $attributes = "_tenant, {$attributes}"; + } + return "CREATE {$type} `{$id}` ON {$this->getSQLTable($collection)} ({$attributes})"; } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 9871564a1..cf1e7224e 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -100,21 +100,31 @@ public function createCollection(string $name, array $attributes = [], array $in */ $sql = " CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id)} ( - \"_id\" SERIAL NOT NULL, - \"_uid\" VARCHAR(255) NOT NULL, - \"_tenant\" VARCHAR(36) DEFAULT NULL, + _id SERIAL NOT NULL, + _uid VARCHAR(255) NOT NULL, + _tenant VARCHAR(36) DEFAULT NULL, \"_createdAt\" TIMESTAMP(3) DEFAULT NULL, \"_updatedAt\" TIMESTAMP(3) DEFAULT NULL, - \"_permissions\" TEXT DEFAULT NULL, + _permissions TEXT DEFAULT NULL, " . \implode(' ', $attributes) . " - PRIMARY KEY (\"_id\") + PRIMARY KEY (_id) ); - - CREATE UNIQUE INDEX \"{$namespace}_{$id}_uid\" ON {$this->getSQLTable($id)} (LOWER(\"_uid\"), \"_tenant\"); - CREATE INDEX \"{$namespace}_{$id}_created\" ON {$this->getSQLTable($id)} (\"_createdAt\", \"_tenant\"); - CREATE INDEX \"{$namespace}_{$id}_updated\" ON {$this->getSQLTable($id)} (\"_updatedAt\", \"_tenant\"); "; + if ($this->shareTables) { + $sql .= " + CREATE UNIQUE INDEX \"{$namespace}_{$this->tenant}_{$id}_uid\" ON {$this->getSQLTable($id)} (_tenant, LOWER(_uid)); + CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_created\" ON {$this->getSQLTable($id)} (_tenant, \"_createdAt\"); + CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_updated\" ON {$this->getSQLTable($id)} (_tenant, \"_updatedAt\"); + "; + } else { + $sql .= " + CREATE UNIQUE INDEX \"{$namespace}_{$id}_uid\" ON {$this->getSQLTable($id)} (LOWER(_uid)); + CREATE INDEX \"{$namespace}_{$id}_created\" ON {$this->getSQLTable($id)} (\"_createdAt\"); + CREATE INDEX \"{$namespace}_{$id}_updated\" ON {$this->getSQLTable($id)} (\"_updatedAt\"); + "; + } + $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql); $stmt = $this->getPDO()->prepare($sql); @@ -124,19 +134,31 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = " CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id . '_perms')} ( - \"_id\" SERIAL NOT NULL, - \"_tenant\" VARCHAR(36) DEFAULT NULL, - \"_type\" VARCHAR(12) NOT NULL, - \"_permission\" VARCHAR(255) NOT NULL, - \"_document\" VARCHAR(255) NOT NULL, - PRIMARY KEY (\"_id\") - ); - CREATE UNIQUE INDEX \"index_{$namespace}_{$id}_ukey\" - ON {$this->getSQLTable($id. '_perms')} USING btree (\"_document\",\"_tenant\",\"_type\",\"_permission\"); - CREATE INDEX \"index_{$namespace}_{$id}_permission\" - ON {$this->getSQLTable($id. '_perms')} USING btree (\"_permission\",\"_type\",\"_tenant\",\"_document\"); + _id SERIAL NOT NULL, + _tenant VARCHAR(36) DEFAULT NULL, + _type VARCHAR(12) NOT NULL, + _permission VARCHAR(255) NOT NULL, + _document VARCHAR(255) NOT NULL, + PRIMARY KEY (_id) + ); "; + if ($this->shareTables) { + $sql .= " + CREATE UNIQUE INDEX \"{$namespace}_{$this->tenant}_{$id}_ukey\" + ON {$this->getSQLTable($id. '_perms')} USING btree (_tenant,_document,_type,_permission); + CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_permission\" + ON {$this->getSQLTable($id. '_perms')} USING btree (_permission,_type,_document,_tenant); + "; + } else { + $sql .= " + CREATE UNIQUE INDEX \"{$namespace}_{$id}_ukey\" + ON {$this->getSQLTable($id. '_perms')} USING btree (_document,_type,_permission); + CREATE INDEX \"{$namespace}_{$id}_permission\" + ON {$this->getSQLTable($id. '_perms')} USING btree (_permission,_type,_document); + "; + } + $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql); $this->getPDO() @@ -612,7 +634,9 @@ public function deleteIndex(string $collection, string $id): bool $id = $this->filter($id); $schemaName = $this->getDatabase(); - $sql = "DROP INDEX IF EXISTS \"{$schemaName}\".{$id}"; + $key = "\"{$this->getNamespace()}_{$this->tenant}_{$collection}_{$id}\""; + + $sql = "DROP INDEX IF EXISTS \"{$schemaName}\".{$key}"; $sql = $this->trigger(Database::EVENT_INDEX_DELETE, $sql); return $this->getPDO() @@ -636,8 +660,8 @@ public function renameIndex(string $collection, string $old, string $new): bool $namespace = $this->getNamespace(); $old = $this->filter($old); $new = $this->filter($new); - $oldIndexName = $collection . "_" . $old; - $newIndexName = $namespace . $collection . "_" . $new; + $oldIndexName = "{$this->tenant}_{$collection}_{$old}"; + $newIndexName = "{$namespace}_{$this->tenant}_{$collection}_{$new}"; $sql = "ALTER INDEX {$this->getSQLTable($oldIndexName)} RENAME TO \"{$newIndexName}\""; $sql = $this->trigger(Database::EVENT_INDEX_RENAME, $sql); @@ -1274,11 +1298,18 @@ public function updateDocuments(string $collection, array $documents, int $batch $updateClause .= "{$column} = excluded.{$column}"; } - $stmt = $this->getPDO()->prepare(" + $sql = " INSERT INTO {$this->getSQLTable($name)} (" . \implode(", ", $columns) . ") VALUES " . \implode(', ', $batchKeys) . " - ON CONFLICT (LOWER(_uid), _tenant) DO UPDATE SET $updateClause - "); + "; + + if ($this->shareTables) { + $sql .= "ON CONFLICT (_tenant, LOWER(_uid)) DO UPDATE SET $updateClause"; + } else { + $sql .= "ON CONFLICT (LOWER(_uid)) DO UPDATE SET $updateClause"; + } + + $stmt = $this->getPDO()->prepare($sql); foreach ($bindValues as $key => $value) { $stmt->bindValue($key, $value, $this->getPDOType($value)); @@ -1955,7 +1986,15 @@ protected function getSQLIndex(string $collection, string $id, string $type, arr default => throw new DatabaseException('Unknown index type: ' . $type . '. Must be one of ' . Database::INDEX_KEY . ', ' . Database::INDEX_UNIQUE . ', ' . Database::INDEX_ARRAY . ', ' . Database::INDEX_FULLTEXT), }; - return 'CREATE ' . $type . ' "' . $this->getNamespace() . '_' . $collection . '_' . $id . '" ON ' . $this->getSQLTable($collection) . ' ( ' . implode(', ', $attributes) . ' );'; + $key = "\"{$this->getNamespace()}_{$this->tenant}_{$collection}_{$id}\""; + $attributes = \implode(', ', $attributes); + + if ($this->shareTables) { + // Add tenant as first index column for best performance + $attributes = "_tenant, {$attributes}"; + } + + return "CREATE {$type} {$key} ON {$this->getSQLTable($collection)} ({$attributes});"; } /** diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 7541dec07..0db2d68eb 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -120,7 +120,12 @@ public function createCollection(string $name, array $attributes = [], array $in foreach ($attributes as $key => $attribute) { $attrId = $this->filter($attribute->getId()); - $attrType = $this->getSQLType($attribute->getAttribute('type'), $attribute->getAttribute('size', 0), $attribute->getAttribute('signed', true)); + + $attrType = $this->getSQLType( + $attribute->getAttribute('type'), + $attribute->getAttribute('size', 0), + $attribute->getAttribute('signed', true) + ); if ($attribute->getAttribute('array')) { $attrType = 'LONGTEXT'; @@ -143,13 +148,11 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql); - $this->getPDO() - ->prepare($sql) - ->execute(); + $this->getPDO()->prepare($sql)->execute(); - $this->createIndex($id, '_index1', Database::INDEX_UNIQUE, ['_uid', '_tenant'], [], []); - $this->createIndex($id, '_created_at', Database::INDEX_KEY, ['_createdAt', '_tenant'], [], []); - $this->createIndex($id, '_updated_at', Database::INDEX_KEY, ['_updatedAt', '_tenant'], [], []); + $this->createIndex($id, '_index1', Database::INDEX_UNIQUE, ['_uid'], [], []); + $this->createIndex($id, '_created_at', Database::INDEX_KEY, [ '_createdAt'], [], []); + $this->createIndex($id, '_updated_at', Database::INDEX_KEY, [ '_updatedAt'], [], []); foreach ($indexes as $index) { $indexId = $this->filter($index->getId()); @@ -393,7 +396,7 @@ public function createIndex(string $collection, string $id, string $type, array WHERE type='index' AND name=:_index; "); - $stmt->bindValue(':_index', "{$this->getNamespace()}_{$name}_{$id}"); + $stmt->bindValue(':_index', "{$this->getNamespace()}_{$this->tenant}_{$name}_{$id}"); $stmt->execute(); $index = $stmt->fetch(); if (!empty($index)) { @@ -423,7 +426,7 @@ public function deleteIndex(string $collection, string $id): bool $name = $this->filter($collection); $id = $this->filter($id); - $sql = "DROP INDEX `{$this->getNamespace()}_{$name}_{$id}`"; + $sql = "DROP INDEX `{$this->getNamespace()}_{$this->tenant}_{$name}_{$id}`"; $sql = $this->trigger(Database::EVENT_INDEX_DELETE, $sql); return $this->getPDO() @@ -966,13 +969,21 @@ public function updateDocuments(string $collection, array $documents, int $batch $updateClause .= "{$column} = excluded.{$column}"; } - $stmt = $this->getPDO()->prepare(" + $sql = " INSERT INTO {$this->getSQLTable($name)} (" . \implode(", ", $columns) . ") VALUES " . \implode(', ', $batchKeys) . " - ON CONFLICT(_uid, _tenant) DO UPDATE SET $updateClause - "); - foreach ($bindValues as $key => $value) { + "; + + if ($this->shareTables) { + $sql .= "ON CONFLICT (_tenant, _uid) DO UPDATE SET $updateClause"; + } else { + $sql .= "ON CONFLICT (_uid) DO UPDATE SET $updateClause"; + } + + $stmt = $this->getPDO()->prepare($sql); + + foreach ($bindValues as $key => $value) { $stmt->bindValue($key, $value, $this->getPDOType($value)); } @@ -1113,7 +1124,7 @@ protected function getSQLIndex(string $collection, string $id, string $type, arr case Database::INDEX_UNIQUE: $type = 'UNIQUE INDEX'; - $postfix = ' COLLATE NOCASE'; + $postfix = 'COLLATE NOCASE'; break; @@ -1129,13 +1140,20 @@ protected function getSQLIndex(string $collection, string $id, string $type, arr }, $attributes); foreach ($attributes as $key => $attribute) { - $order = ''; $attribute = $this->filter($attribute); - $attributes[$key] = "`$attribute`$postfix $order"; + $attributes[$key] = "`{$attribute}` {$postfix}"; } - return "CREATE {$type} `{$this->getNamespace()}_{$collection}_{$id}` ON `{$this->getNamespace()}_{$collection}` ( " . implode(', ', $attributes) . ")"; + $key = "`{$this->getNamespace()}_{$this->tenant}_{$collection}_{$id}`"; + $attributes = implode(', ', $attributes); + + if ($this->shareTables) { + $key = "`{$this->getNamespace()}_{$collection}_{$id}`"; + $attributes = "_tenant {$postfix}, {$attributes}"; + } + + return "CREATE {$type} {$key} ON `{$this->getNamespace()}_{$collection}` ({$attributes})"; } /** From c363b5acae264ed32fe4249a210cdd60eee5ce4b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Nov 2023 16:24:32 +1300 Subject: [PATCH 21/45] Format --- src/Database/Adapter/Postgres.php | 48 +++++++++++++++---------------- src/Database/Adapter/SQLite.php | 42 +++++++++++++-------------- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index cf1e7224e..803281e2b 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -111,19 +111,19 @@ public function createCollection(string $name, array $attributes = [], array $in ); "; - if ($this->shareTables) { - $sql .= " + if ($this->shareTables) { + $sql .= " CREATE UNIQUE INDEX \"{$namespace}_{$this->tenant}_{$id}_uid\" ON {$this->getSQLTable($id)} (_tenant, LOWER(_uid)); CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_created\" ON {$this->getSQLTable($id)} (_tenant, \"_createdAt\"); CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_updated\" ON {$this->getSQLTable($id)} (_tenant, \"_updatedAt\"); "; - } else { - $sql .= " + } else { + $sql .= " CREATE UNIQUE INDEX \"{$namespace}_{$id}_uid\" ON {$this->getSQLTable($id)} (LOWER(_uid)); CREATE INDEX \"{$namespace}_{$id}_created\" ON {$this->getSQLTable($id)} (\"_createdAt\"); CREATE INDEX \"{$namespace}_{$id}_updated\" ON {$this->getSQLTable($id)} (\"_updatedAt\"); "; - } + } $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql); @@ -143,21 +143,21 @@ public function createCollection(string $name, array $attributes = [], array $in ); "; - if ($this->shareTables) { - $sql .= " + if ($this->shareTables) { + $sql .= " CREATE UNIQUE INDEX \"{$namespace}_{$this->tenant}_{$id}_ukey\" ON {$this->getSQLTable($id. '_perms')} USING btree (_tenant,_document,_type,_permission); CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_permission\" ON {$this->getSQLTable($id. '_perms')} USING btree (_permission,_type,_document,_tenant); "; - } else { - $sql .= " + } else { + $sql .= " CREATE UNIQUE INDEX \"{$namespace}_{$id}_ukey\" ON {$this->getSQLTable($id. '_perms')} USING btree (_document,_type,_permission); CREATE INDEX \"{$namespace}_{$id}_permission\" ON {$this->getSQLTable($id. '_perms')} USING btree (_permission,_type,_document); "; - } + } $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql); @@ -634,7 +634,7 @@ public function deleteIndex(string $collection, string $id): bool $id = $this->filter($id); $schemaName = $this->getDatabase(); - $key = "\"{$this->getNamespace()}_{$this->tenant}_{$collection}_{$id}\""; + $key = "\"{$this->getNamespace()}_{$this->tenant}_{$collection}_{$id}\""; $sql = "DROP INDEX IF EXISTS \"{$schemaName}\".{$key}"; $sql = $this->trigger(Database::EVENT_INDEX_DELETE, $sql); @@ -1298,16 +1298,16 @@ public function updateDocuments(string $collection, array $documents, int $batch $updateClause .= "{$column} = excluded.{$column}"; } - $sql = " + $sql = " INSERT INTO {$this->getSQLTable($name)} (" . \implode(", ", $columns) . ") VALUES " . \implode(', ', $batchKeys) . " "; - if ($this->shareTables) { - $sql .= "ON CONFLICT (_tenant, LOWER(_uid)) DO UPDATE SET $updateClause"; - } else { - $sql .= "ON CONFLICT (LOWER(_uid)) DO UPDATE SET $updateClause"; - } + if ($this->shareTables) { + $sql .= "ON CONFLICT (_tenant, LOWER(_uid)) DO UPDATE SET $updateClause"; + } else { + $sql .= "ON CONFLICT (LOWER(_uid)) DO UPDATE SET $updateClause"; + } $stmt = $this->getPDO()->prepare($sql); @@ -1986,15 +1986,15 @@ protected function getSQLIndex(string $collection, string $id, string $type, arr default => throw new DatabaseException('Unknown index type: ' . $type . '. Must be one of ' . Database::INDEX_KEY . ', ' . Database::INDEX_UNIQUE . ', ' . Database::INDEX_ARRAY . ', ' . Database::INDEX_FULLTEXT), }; - $key = "\"{$this->getNamespace()}_{$this->tenant}_{$collection}_{$id}\""; - $attributes = \implode(', ', $attributes); + $key = "\"{$this->getNamespace()}_{$this->tenant}_{$collection}_{$id}\""; + $attributes = \implode(', ', $attributes); - if ($this->shareTables) { - // Add tenant as first index column for best performance - $attributes = "_tenant, {$attributes}"; - } + if ($this->shareTables) { + // Add tenant as first index column for best performance + $attributes = "_tenant, {$attributes}"; + } - return "CREATE {$type} {$key} ON {$this->getSQLTable($collection)} ({$attributes});"; + return "CREATE {$type} {$key} ON {$this->getSQLTable($collection)} ({$attributes});"; } /** diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 0db2d68eb..a3b9afcf7 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -122,10 +122,10 @@ public function createCollection(string $name, array $attributes = [], array $in $attrId = $this->filter($attribute->getId()); $attrType = $this->getSQLType( - $attribute->getAttribute('type'), - $attribute->getAttribute('size', 0), - $attribute->getAttribute('signed', true) - ); + $attribute->getAttribute('type'), + $attribute->getAttribute('size', 0), + $attribute->getAttribute('signed', true) + ); if ($attribute->getAttribute('array')) { $attrType = 'LONGTEXT'; @@ -150,9 +150,9 @@ public function createCollection(string $name, array $attributes = [], array $in $this->getPDO()->prepare($sql)->execute(); - $this->createIndex($id, '_index1', Database::INDEX_UNIQUE, ['_uid'], [], []); - $this->createIndex($id, '_created_at', Database::INDEX_KEY, [ '_createdAt'], [], []); - $this->createIndex($id, '_updated_at', Database::INDEX_KEY, [ '_updatedAt'], [], []); + $this->createIndex($id, '_index1', Database::INDEX_UNIQUE, ['_uid'], [], []); + $this->createIndex($id, '_created_at', Database::INDEX_KEY, [ '_createdAt'], [], []); + $this->createIndex($id, '_updated_at', Database::INDEX_KEY, [ '_updatedAt'], [], []); foreach ($indexes as $index) { $indexId = $this->filter($index->getId()); @@ -969,21 +969,21 @@ public function updateDocuments(string $collection, array $documents, int $batch $updateClause .= "{$column} = excluded.{$column}"; } - $sql = " + $sql = " INSERT INTO {$this->getSQLTable($name)} (" . \implode(", ", $columns) . ") VALUES " . \implode(', ', $batchKeys) . " "; - if ($this->shareTables) { - $sql .= "ON CONFLICT (_tenant, _uid) DO UPDATE SET $updateClause"; - } else { - $sql .= "ON CONFLICT (_uid) DO UPDATE SET $updateClause"; - } + if ($this->shareTables) { + $sql .= "ON CONFLICT (_tenant, _uid) DO UPDATE SET $updateClause"; + } else { + $sql .= "ON CONFLICT (_uid) DO UPDATE SET $updateClause"; + } - $stmt = $this->getPDO()->prepare($sql); + $stmt = $this->getPDO()->prepare($sql); - foreach ($bindValues as $key => $value) { + foreach ($bindValues as $key => $value) { $stmt->bindValue($key, $value, $this->getPDOType($value)); } @@ -1145,13 +1145,13 @@ protected function getSQLIndex(string $collection, string $id, string $type, arr $attributes[$key] = "`{$attribute}` {$postfix}"; } - $key = "`{$this->getNamespace()}_{$this->tenant}_{$collection}_{$id}`"; - $attributes = implode(', ', $attributes); + $key = "`{$this->getNamespace()}_{$this->tenant}_{$collection}_{$id}`"; + $attributes = implode(', ', $attributes); - if ($this->shareTables) { - $key = "`{$this->getNamespace()}_{$collection}_{$id}`"; - $attributes = "_tenant {$postfix}, {$attributes}"; - } + if ($this->shareTables) { + $key = "`{$this->getNamespace()}_{$collection}_{$id}`"; + $attributes = "_tenant {$postfix}, {$attributes}"; + } return "CREATE {$type} {$key} ON `{$this->getNamespace()}_{$collection}` ({$attributes})"; } From 3183694da553bc73123d0d3e13e581a00845ac65 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Nov 2023 16:53:25 +1300 Subject: [PATCH 22/45] Use integer for tenant column --- src/Database/Adapter.php | 10 +++++----- src/Database/Adapter/MariaDB.php | 4 ++-- src/Database/Adapter/Postgres.php | 4 ++-- src/Database/Adapter/SQLite.php | 4 ++-- src/Database/Database.php | 4 ++-- tests/e2e/Adapter/Base.php | 11 +++++------ 6 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index d6c0ce588..fa8b3f9ac 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -13,7 +13,7 @@ abstract class Adapter protected bool $shareTables = false; - protected ?string $tenant = null; + protected ?int $tenant = null; /** * @var array @@ -162,11 +162,11 @@ public function getShareTables(): bool * * Set tenant to use if tables are shared * - * @param string $tenant + * @param int $tenant * * @return bool */ - public function setTenant(string $tenant): bool + public function setTenant(int $tenant): bool { $this->tenant = $tenant; @@ -178,9 +178,9 @@ public function setTenant(string $tenant): bool * * Get tenant to use for shared tables * - * @return string + * @return ?int */ - public function getTenant(): ?string + public function getTenant(): ?int { return $this->tenant; } diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index e886c908d..24ae2eeb8 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -117,7 +117,7 @@ public function createCollection(string $name, array $attributes = [], array $in CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id)} ( _id INT(11) UNSIGNED NOT NULL AUTO_INCREMENT, _uid VARCHAR(255) NOT NULL, - _tenant VARCHAR(36) DEFAULT NULL, + _tenant INT(11) UNSIGNED DEFAULT NULL, _createdAt DATETIME(3) DEFAULT NULL, _updatedAt DATETIME(3) DEFAULT NULL, _permissions MEDIUMTEXT DEFAULT NULL, @@ -153,7 +153,7 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = " CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id . '_perms')} ( _id int(11) UNSIGNED NOT NULL AUTO_INCREMENT, - _tenant VARCHAR(36) DEFAULT NULL, + _tenant INT(11) UNSIGNED DEFAULT NULL, _type VARCHAR(12) NOT NULL, _permission VARCHAR(255) NOT NULL, _document VARCHAR(255) NOT NULL, diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 803281e2b..9dca56cfa 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -102,7 +102,7 @@ public function createCollection(string $name, array $attributes = [], array $in CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id)} ( _id SERIAL NOT NULL, _uid VARCHAR(255) NOT NULL, - _tenant VARCHAR(36) DEFAULT NULL, + _tenant INTEGER DEFAULT NULL, \"_createdAt\" TIMESTAMP(3) DEFAULT NULL, \"_updatedAt\" TIMESTAMP(3) DEFAULT NULL, _permissions TEXT DEFAULT NULL, @@ -135,7 +135,7 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = " CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id . '_perms')} ( _id SERIAL NOT NULL, - _tenant VARCHAR(36) DEFAULT NULL, + _tenant INTEGER DEFAULT NULL, _type VARCHAR(12) NOT NULL, _permission VARCHAR(255) NOT NULL, _document VARCHAR(255) NOT NULL, diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index a3b9afcf7..22901cd2a 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -138,7 +138,7 @@ public function createCollection(string $name, array $attributes = [], array $in CREATE TABLE IF NOT EXISTS `{$this->getSQLTable($id)}` ( `_id` INTEGER PRIMARY KEY AUTOINCREMENT, `_uid` VARCHAR(36) NOT NULL, - `_tenant` VARCHAR(36) DEFAULT NULL, + `_tenant` INTEGER DEFAULT NULL, `_createdAt` DATETIME(3) DEFAULT NULL, `_updatedAt` DATETIME(3) DEFAULT NULL, `_permissions` MEDIUMTEXT DEFAULT NULL".(!empty($attributes) ? ',' : '')." @@ -167,7 +167,7 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = " CREATE TABLE IF NOT EXISTS `{$this->getSQLTable($id)}_perms` ( `_id` INTEGER PRIMARY KEY AUTOINCREMENT, - `_tenant` VARCHAR(36) DEFAULT NULL, + `_tenant` INTEGER DEFAULT NULL, `_type` VARCHAR(12) NOT NULL, `_permission` VARCHAR(255) NOT NULL, `_document` VARCHAR(255) NOT NULL diff --git a/src/Database/Database.php b/src/Database/Database.php index 3f25afc5a..1272dfa25 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -664,10 +664,10 @@ public function setShareTables(bool $share): self * * Set tenant to use if tables are shared * - * @param string $tenant + * @param int $tenant * @return self */ - public function setTenant(string $tenant): self + public function setTenant(int $tenant): self { $this->adapter->setTenant($tenant); diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 0784fe31a..04a2a9f28 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -12599,8 +12599,8 @@ public function testIsolationModes(): void * Table */ - $tenant1 = ID::unique(); - $tenant2 = ID::unique(); + $tenant1 = 1; + $tenant2 = 2; $database ->setDatabase('sharedTables') @@ -12629,11 +12629,11 @@ public function testIsolationModes(): void '$permissions' => [ Permission::read(Role::any()), ], - 'name' => $tenant1, + 'name' => 'Spiderman', ])); $doc = $database->getDocument('people', $docId); - $this->assertEquals($tenant1, $doc['name']); + $this->assertEquals('Spiderman', $doc['name']); $docs = $database->find('people'); $this->assertEquals(1, \count($docs)); @@ -12659,8 +12659,7 @@ public function testIsolationModes(): void $database->setTenant($tenant1); $doc = $database->getDocument('people', $docId); - $this->assertEquals($tenant1, $doc['name']); - + $this->assertEquals('Spiderman', $doc['name']); $docs = $database->find('people'); $this->assertEquals(1, \count($docs)); From baea354ee6324c6b058836bd4c97769a2792f68f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Nov 2023 17:35:15 +1300 Subject: [PATCH 23/45] deleteCached* -> purgeCached* --- README.md | 4 ++-- src/Database/Database.php | 36 ++++++++++++++++++------------------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 9c6b5bf55..1adf58143 100644 --- a/README.md +++ b/README.md @@ -402,7 +402,7 @@ $database->deleteCollection( ); // Delete cached documents of a collection -$database->deleteCachedCollection( +$database->purgeCachedCollection( collection: 'users' ); ``` @@ -836,7 +836,7 @@ $database->deleteDocument( // Delete a cached document Note: Cached Documents or Collections are automatically deleted when a document or collection is updated or deleted -$database->deleteCachedDocument( +$database->purgeCachedDocument( collection: 'movies', id: $document->getId() ); diff --git a/src/Database/Database.php b/src/Database/Database.php index 1272dfa25..28f58fe35 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -1535,10 +1535,10 @@ public function updateAttribute(string $collection, string $id, string $type = n throw new DatabaseException('Failed to update attribute'); } - $this->deleteCachedCollection($collection); + $this->purgeCachedCollection($collection); } - $this->deleteCachedDocument(self::METADATA, $collection); + $this->purgeCachedDocument(self::METADATA, $collection); }); } @@ -2014,7 +2014,7 @@ public function updateRelationship( $junctionAttribute->setAttribute('key', $newTwoWayKey); }); - $this->deleteCachedCollection($junction); + $this->purgeCachedCollection($junction); } if ($altering) { @@ -2034,8 +2034,8 @@ public function updateRelationship( } } - $this->deleteCachedCollection($collection->getId()); - $this->deleteCachedCollection($relatedCollection->getId()); + $this->purgeCachedCollection($collection->getId()); + $this->purgeCachedCollection($relatedCollection->getId()); $renameIndex = function (string $collection, string $key, string $newKey) { $this->updateIndexMeta( @@ -2197,8 +2197,8 @@ public function deleteRelationship(string $collection, string $id): bool throw new DatabaseException('Failed to delete relationship'); } - $this->deleteCachedCollection($collection->getId()); - $this->deleteCachedCollection($relatedCollection->getId()); + $this->purgeCachedCollection($collection->getId()); + $this->purgeCachedCollection($relatedCollection->getId()); $this->trigger(self::EVENT_ATTRIBUTE_DELETE, $relationship); @@ -3216,7 +3216,7 @@ private function relateDocumentsById( } break; case Database::RELATION_MANY_TO_MANY: - $this->deleteCachedDocument($relatedCollection->getId(), $relationId); + $this->purgeCachedDocument($relatedCollection->getId(), $relationId); $junction = $this->getJunctionCollection($collection, $relatedCollection, $side); @@ -3397,7 +3397,7 @@ public function updateDocument(string $collection, string $id, Document $documen $this->purgeRelatedDocuments($collection, $id); - $this->deleteCachedDocument($collection->getId(), $id); + $this->purgeCachedDocument($collection->getId(), $id); $this->trigger(self::EVENT_DOCUMENT_UPDATE, $document); @@ -3466,7 +3466,7 @@ public function updateDocuments(string $collection, array $documents, int $batch foreach ($documents as $key => $document) { $documents[$key] = $this->decode($collection, $document); - $this->deleteCachedDocument($collection->getId(), $document->getId()); + $this->purgeCachedDocument($collection->getId(), $document->getId()); } $this->trigger(self::EVENT_DOCUMENTS_UPDATE, $documents); @@ -3724,7 +3724,7 @@ private function updateDocumentRelationships(Document $collection, Document $old // For many-one we need to update the related key to null if no relation exists $document->setAttribute($key, null); } - $this->deleteCachedDocument($relatedCollection->getId(), $value); + $this->purgeCachedDocument($relatedCollection->getId(), $value); } elseif ($value instanceof Document) { $related = $this->getDocument($relatedCollection->getId(), $value->getId()); @@ -3742,7 +3742,7 @@ private function updateDocumentRelationships(Document $collection, Document $old $related->getId(), $value ); - $this->deleteCachedDocument($relatedCollection->getId(), $related->getId()); + $this->purgeCachedDocument($relatedCollection->getId(), $related->getId()); } $document->setAttribute($key, $value->getId()); @@ -3921,7 +3921,7 @@ public function increaseDocumentAttribute(string $collection, string $id, string $max = $max ? $max - $value : null; $result = $this->adapter->increaseDocumentAttribute($collection->getId(), $id, $attribute, $value, null, $max); - $this->deleteCachedDocument($collection->getId(), $id); + $this->purgeCachedDocument($collection->getId(), $id); $this->trigger(self::EVENT_DOCUMENT_INCREASE, $document); @@ -3992,7 +3992,7 @@ public function decreaseDocumentAttribute(string $collection, string $id, string $min = $min ? $min + $value : null; $result = $this->adapter->increaseDocumentAttribute($collection->getId(), $id, $attribute, $value * -1, $min); - $this->deleteCachedDocument($collection->getId(), $id); + $this->purgeCachedDocument($collection->getId(), $id); $this->trigger(self::EVENT_DOCUMENT_DECREASE, $document); return $result; @@ -4052,7 +4052,7 @@ public function deleteDocument(string $collection, string $id): bool $deleted = $this->adapter->deleteDocument($collection->getId(), $id); $this->purgeRelatedDocuments($collection, $id); - $this->deleteCachedDocument($collection->getId(), $id); + $this->purgeCachedDocument($collection->getId(), $id); $this->trigger(self::EVENT_DOCUMENT_DELETE, $document); @@ -4436,7 +4436,7 @@ private function deleteCascade(Document $collection, Document $relatedCollection * @return bool * @throws DatabaseException */ - public function deleteCachedCollection(string $collection): bool + public function purgeCachedCollection(string $collection): bool { return $this->cache->purge('cache-' . $this->getNamespace() . ':' . $this->adapter->getTenant() . ':' . $collection . ':*'); } @@ -4450,7 +4450,7 @@ public function deleteCachedCollection(string $collection): bool * @return bool * @throws DatabaseException */ - public function deleteCachedDocument(string $collection, string $id): bool + public function purgeCachedDocument(string $collection, string $id): bool { return $this->cache->purge('cache-' . $this->getNamespace() . ':' . $this->adapter->getTenant() . ':' . $collection . ':' . $id . ':*'); } @@ -5165,7 +5165,7 @@ private function purgeRelatedDocuments(Document $collection, string $id): void if (!empty($cache)) { foreach ($cache as $v) { list($collectionId, $documentId) = explode(':', $v); - $this->deleteCachedDocument($collectionId, $documentId); + $this->purgeCachedDocument($collectionId, $documentId); } $this->cache->purge($key); } From 0d65504735a2c47c46ef0a3eccadf9cc258b6690 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Nov 2023 17:36:48 +1300 Subject: [PATCH 24/45] Tabs -> spaces --- src/Database/Adapter/SQL.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 87c0921b9..994726c0d 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -110,7 +110,7 @@ public function getDocument(string $collection, string $id, array $queries = []) $selections = $this->getAttributeSelections($queries); $sql = " - SELECT {$this->getAttributeProjection($selections)} + SELECT {$this->getAttributeProjection($selections)} FROM {$this->getSQLTable($name)} WHERE _uid = :_uid "; From 32b10a3771af2029a0ebffe6ad9ec91cd2351c31 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Nov 2023 18:09:51 +1300 Subject: [PATCH 25/45] Automate default attributes/index counts --- src/Database/Adapter/MariaDB.php | 3 +- src/Database/Adapter/Mongo.php | 11 +++- src/Database/Adapter/SQL.php | 4 +- src/Database/Database.php | 82 +++++++++++++------------ src/Database/Document.php | 7 ++- src/Database/Validator/Query/Select.php | 8 ++- tests/e2e/Adapter/MySQLTest.php | 9 --- tests/e2e/Adapter/SQLiteTest.php | 9 --- 8 files changed, 68 insertions(+), 65 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 24ae2eeb8..fc90a1acd 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -128,8 +128,7 @@ public function createCollection(string $name, array $attributes = [], array $in if ($this->shareTables) { $sql .= " - UNIQUE KEY _uid_tenant (_tenant,_uid), - KEY _tenant (_tenant), + UNIQUE KEY _uid_tenant (_tenant, _uid), KEY _created_at (_tenant, _createdAt), KEY _updated_at (_tenant, _updatedAt) "; diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 19792e0f7..a3a29b369 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -1459,9 +1459,14 @@ protected function getAttributeProjection(array $selections, string $prefix = '' { $projection = []; + $internalKeys = \array_map( + fn ($attr) => $attr['$id'], + Database::INTERNAL_ATTRIBUTES + ); + foreach ($selections as $selection) { // Skip internal attributes since all are selected by default - if (\in_array($selection, Database::INTERNAL_ATTRIBUTES)) { + if (\in_array($selection, $internalKeys)) { continue; } @@ -1628,7 +1633,7 @@ public function getCountOfIndexes(Document $collection): int */ public static function getCountOfDefaultAttributes(): int { - return 6; + return \count(Database::INTERNAL_ATTRIBUTES); } /** @@ -1638,7 +1643,7 @@ public static function getCountOfDefaultAttributes(): int */ public static function getCountOfDefaultIndexes(): int { - return 6; + return \count(Database::INTERNAL_INDEXES); } /** diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 994726c0d..0150c76cd 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -282,7 +282,7 @@ public function getCountOfIndexes(Document $collection): int */ public static function getCountOfDefaultAttributes(): int { - return 5; + return \count(Database::INTERNAL_ATTRIBUTES); } /** @@ -292,7 +292,7 @@ public static function getCountOfDefaultAttributes(): int */ public static function getCountOfDefaultIndexes(): int { - return 6; + return \count(Database::INTERNAL_INDEXES); } /** diff --git a/src/Database/Database.php b/src/Database/Database.php index 28f58fe35..c6b4db7e5 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -145,7 +145,7 @@ class Database * * @var array> */ - protected const ATTRIBUTES = [ + public const INTERNAL_ATTRIBUTES = [ [ '$id' => '$id', 'type' => self::VAR_STRING, @@ -155,6 +155,15 @@ class Database 'array' => false, 'filters' => [], ], + [ + '$id' => '$internalId', + 'type' => self::VAR_STRING, + 'size' => Database::LENGTH_KEY, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], [ '$id' => '$collection', 'type' => self::VAR_STRING, @@ -196,21 +205,26 @@ class Database 'array' => false, 'filters' => ['datetime'] ], + [ + '$id' => '$permissions', + 'type' => Database::VAR_STRING, + 'size' => 1000000, + 'signed' => true, + 'required' => false, + 'default' => [], + 'array' => false, + 'filters' => ['json'] + ], ]; - /** - * List of Internal Attributes - * - * @var array - */ - public const INTERNAL_ATTRIBUTES = [ - '$id', - '$internalId', - '$collection', - '$tenant', - '$createdAt', - '$updatedAt', - '$permissions', + public const INTERNAL_INDEXES = [ + '_id', + '_uid', + '_createdAt', + '_updatedAt', + '_permissions_id', + '_permissions_forwards', + '_permissions_backwards', ]; /** @@ -219,7 +233,7 @@ class Database * * @var array */ - protected array $collection = [ + public const COLLECTION = [ '$id' => self::METADATA, '$collection' => self::METADATA, 'name' => 'collections', @@ -705,18 +719,8 @@ public function create(?string $database = null): bool * @var array $attributes */ $attributes = array_map(function ($attribute) { - return new Document([ - '$id' => ID::custom($attribute[0]), - 'type' => $attribute[1], - 'size' => $attribute[2], - 'required' => $attribute[3], - ]); - }, [ // Array of [$id, $type, $size, $required] - ['name', self::VAR_STRING, 512, true], - ['attributes', self::VAR_STRING, 1000000, false], - ['indexes', self::VAR_STRING, 1000000, false], - ['documentSecurity', self::VAR_BOOLEAN, 0, false], - ]); + return new Document($attribute); + }, self::COLLECTION['attributes']); $this->silent(fn () => $this->createCollection(self::METADATA, $attributes)); @@ -843,7 +847,7 @@ public function createCollection(string $id, array $attributes = [], array $inde $this->adapter->createCollection($id, $attributes, $indexes); if ($id === self::METADATA) { - return new Document($this->collection); + return new Document(self::COLLECTION); } // Check index limits, if given @@ -2414,6 +2418,7 @@ public function deleteIndex(string $collection, string $id): bool * * @return Document * @throws DatabaseException + * @throws Exception */ public function getDocument(string $collection, string $id, array $queries = []): Document { @@ -2422,7 +2427,7 @@ public function getDocument(string $collection, string $id, array $queries = []) } if ($collection === self::METADATA && $id === self::METADATA) { - return new Document($this->collection); + return new Document(self::COLLECTION); } if (empty($collection)) { @@ -2587,8 +2592,8 @@ public function getDocument(string $collection, string $id, array $queries = []) if ($query->getMethod() === Query::TYPE_SELECT) { $values = $query->getValues(); foreach (Database::INTERNAL_ATTRIBUTES as $internalAttribute) { - if (!in_array($internalAttribute, $values)) { - $document->removeAttribute($internalAttribute); + if (!in_array($internalAttribute['$id'], $values)) { + $document->removeAttribute($internalAttribute['$id']); } } } @@ -4599,8 +4604,8 @@ public function find(string $collection, array $queries = []): array $values = $query->getValues(); foreach ($results as $result) { foreach (Database::INTERNAL_ATTRIBUTES as $internalAttribute) { - if (!\in_array($internalAttribute, $values)) { - $result->removeAttribute($internalAttribute); + if (!\in_array($internalAttribute['$id'], $values)) { + $result->removeAttribute($internalAttribute['$id']); } } } @@ -4735,7 +4740,7 @@ public static function addFilter(string $name, callable $encode, callable $decod public static function getInternalAttributes(): array { $attributes = []; - foreach (Database::ATTRIBUTES as $internal) { + foreach (Database::INTERNAL_ATTRIBUTES as $internal) { $attributes[] = new Document($internal); } return $attributes; @@ -4753,7 +4758,7 @@ public static function getInternalAttributes(): array public function encode(Document $collection, Document $document): Document { $attributes = $collection->getAttribute('attributes', []); - $attributes = array_merge($attributes, $this->getInternalAttributes()); + $attributes = \array_merge($attributes, $this->getInternalAttributes()); foreach ($attributes as $attribute) { $key = $attribute['$id'] ?? ''; $array = $attribute['array'] ?? false; @@ -5019,10 +5024,11 @@ private function validateSelections(Document $collection, array $queries): array } } - $keys = []; - // Allow querying internal attributes - $keys = array_merge($keys, self::INTERNAL_ATTRIBUTES); + $keys = \array_map( + fn ($attribute) => $attribute['$id'], + self::INTERNAL_ATTRIBUTES + ); foreach ($collection->getAttribute('attributes', []) as $attribute) { if ($attribute['type'] !== self::VAR_RELATIONSHIP) { diff --git a/src/Database/Document.php b/src/Database/Document.php index ba7846bf6..7afddc25f 100644 --- a/src/Database/Document.php +++ b/src/Database/Document.php @@ -166,8 +166,13 @@ public function getAttributes(): array { $attributes = []; + $internalKeys = \array_map( + fn ($attr) => $attr['$id'], + Database::INTERNAL_ATTRIBUTES + ); + foreach ($this as $attribute => $value) { - if (\in_array($attribute, Database::INTERNAL_ATTRIBUTES)) { + if (\in_array($attribute, $internalKeys)) { continue; } diff --git a/src/Database/Validator/Query/Select.php b/src/Database/Validator/Query/Select.php index 833284c5e..e00c3916e 100644 --- a/src/Database/Validator/Query/Select.php +++ b/src/Database/Validator/Query/Select.php @@ -2,6 +2,7 @@ namespace Utopia\Database\Validator\Query; +use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Query; @@ -56,6 +57,11 @@ public function isValid($value): bool return false; } + $internalKeys = \array_map( + fn ($attr) => $attr['$id'], + Database::INTERNAL_ATTRIBUTES + ); + foreach ($value->getValues() as $attribute) { if (\str_contains($attribute, '.')) { //special symbols with `dots` @@ -69,7 +75,7 @@ public function isValid($value): bool } // Skip internal attributes - if (\in_array($attribute, self::INTERNAL_ATTRIBUTES)) { + if (\in_array($attribute, $internalKeys)) { continue; } diff --git a/tests/e2e/Adapter/MySQLTest.php b/tests/e2e/Adapter/MySQLTest.php index 8704b1567..d204e8a40 100644 --- a/tests/e2e/Adapter/MySQLTest.php +++ b/tests/e2e/Adapter/MySQLTest.php @@ -25,15 +25,6 @@ public static function getAdapterName(): string return "mysql"; } - /** - * - * @return int - */ - public static function getUsedIndexes(): int - { - return MySQL::getCountOfDefaultIndexes(); - } - /** * @return Database */ diff --git a/tests/e2e/Adapter/SQLiteTest.php b/tests/e2e/Adapter/SQLiteTest.php index 8e16ce143..16c36f6fd 100644 --- a/tests/e2e/Adapter/SQLiteTest.php +++ b/tests/e2e/Adapter/SQLiteTest.php @@ -25,15 +25,6 @@ public static function getAdapterName(): string return "sqlite"; } - /** - * - * @return int - */ - public static function getUsedIndexes(): int - { - return SQLite::getCountOfDefaultIndexes(); - } - /** * @return Database */ From d057eaf0cb7a24541818803d4868075fd250ba81 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Nov 2023 18:13:19 +1300 Subject: [PATCH 26/45] Add back size comments --- src/Database/Adapter/SQL.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 0150c76cd..ade4141e1 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -328,10 +328,17 @@ public function getAttributeWidth(Document $collection): int switch ($attribute['type']) { case Database::VAR_STRING: $total += match (true) { + // 8 bytes length + 4 bytes for LONGTEXT $attribute['size'] > 16777215 => 12, + // 8 bytes length + 3 bytes for MEDIUMTEXT $attribute['size'] > 65535 => 11, + // 8 bytes length + 2 bytes for TEXT $attribute['size'] > $this->getMaxVarcharLength() => 10, + // $size = $size * 4; // utf8mb4 up to 4 bytes per char + // 8 bytes length + 2 bytes for VARCHAR(>255) $attribute['size'] > 255 => ($attribute['size'] * 4) + 2, + // $size = $size * 4; // utf8mb4 up to 4 bytes per char + // 8 bytes length + 1 bytes for VARCHAR(<=255) default => ($attribute['size'] * 4) + 1, }; break; From c365f85a71d2526fca7cdbe8787742df2b6e5448 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Nov 2023 18:15:39 +1300 Subject: [PATCH 27/45] Format --- composer.lock | 30 +++++++++++++++--------------- src/Database/Adapter/SQL.php | 14 +++++++------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/composer.lock b/composer.lock index 8f3d208b3..eae5cf37d 100644 --- a/composer.lock +++ b/composer.lock @@ -513,16 +513,16 @@ }, { "name": "laravel/pint", - "version": "v1.13.5", + "version": "v1.13.6", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "df105cf8ce7a8f0b8a9425ff45cd281a5448e423" + "reference": "3e3d2ab01c7d8b484c18e6100ecf53639c744fa7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/df105cf8ce7a8f0b8a9425ff45cd281a5448e423", - "reference": "df105cf8ce7a8f0b8a9425ff45cd281a5448e423", + "url": "https://api.github.com/repos/laravel/pint/zipball/3e3d2ab01c7d8b484c18e6100ecf53639c744fa7", + "reference": "3e3d2ab01c7d8b484c18e6100ecf53639c744fa7", "shasum": "" }, "require": { @@ -533,13 +533,13 @@ "php": "^8.1.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.34.1", - "illuminate/view": "^10.26.2", - "laravel-zero/framework": "^10.1.2", + "friendsofphp/php-cs-fixer": "^3.38.0", + "illuminate/view": "^10.30.1", + "laravel-zero/framework": "^10.3.0", "mockery/mockery": "^1.6.6", "nunomaduro/larastan": "^2.6.4", "nunomaduro/termwind": "^1.15.1", - "pestphp/pest": "^2.20.0" + "pestphp/pest": "^2.24.2" }, "bin": [ "builds/pint" @@ -575,7 +575,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2023-10-26T09:26:10+00:00" + "time": "2023-11-07T17:59:57+00:00" }, { "name": "myclabs/deep-copy", @@ -839,16 +839,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.10.40", + "version": "1.10.44", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "93c84b5bf7669920d823631e39904d69b9c7dc5d" + "reference": "bf84367c53a23f759513985c54ffe0d0c249825b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/93c84b5bf7669920d823631e39904d69b9c7dc5d", - "reference": "93c84b5bf7669920d823631e39904d69b9c7dc5d", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/bf84367c53a23f759513985c54ffe0d0c249825b", + "reference": "bf84367c53a23f759513985c54ffe0d0c249825b", "shasum": "" }, "require": { @@ -897,7 +897,7 @@ "type": "tidelift" } ], - "time": "2023-10-30T14:48:31+00:00" + "time": "2023-11-21T16:30:46+00:00" }, { "name": "phpunit/php-code-coverage", @@ -2608,5 +2608,5 @@ "php": ">=8.0" }, "platform-dev": [], - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.6.0" } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index ade4141e1..4f3904f91 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -328,17 +328,17 @@ public function getAttributeWidth(Document $collection): int switch ($attribute['type']) { case Database::VAR_STRING: $total += match (true) { - // 8 bytes length + 4 bytes for LONGTEXT + // 8 bytes length + 4 bytes for LONGTEXT $attribute['size'] > 16777215 => 12, - // 8 bytes length + 3 bytes for MEDIUMTEXT + // 8 bytes length + 3 bytes for MEDIUMTEXT $attribute['size'] > 65535 => 11, - // 8 bytes length + 2 bytes for TEXT + // 8 bytes length + 2 bytes for TEXT $attribute['size'] > $this->getMaxVarcharLength() => 10, - // $size = $size * 4; // utf8mb4 up to 4 bytes per char - // 8 bytes length + 2 bytes for VARCHAR(>255) + // $size = $size * 4; // utf8mb4 up to 4 bytes per char + // 8 bytes length + 2 bytes for VARCHAR(>255) $attribute['size'] > 255 => ($attribute['size'] * 4) + 2, - // $size = $size * 4; // utf8mb4 up to 4 bytes per char - // 8 bytes length + 1 bytes for VARCHAR(<=255) + // $size = $size * 4; // utf8mb4 up to 4 bytes per char + // 8 bytes length + 1 bytes for VARCHAR(<=255) default => ($attribute['size'] * 4) + 1, }; break; From 52fc889d9e419cab60c6906d54ce69aa7aed3455 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Nov 2023 19:20:11 +1300 Subject: [PATCH 28/45] Fix attribute checks --- src/Database/Database.php | 24 +++++++++--------------- src/Database/Validator/Index.php | 7 ++++--- tests/e2e/Adapter/Base.php | 2 +- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index c6b4db7e5..fd3542473 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -4733,19 +4733,6 @@ public static function addFilter(string $name, callable $encode, callable $decod ]; } - /** - * @return array - * @throws DatabaseException - */ - public static function getInternalAttributes(): array - { - $attributes = []; - foreach (Database::INTERNAL_ATTRIBUTES as $internal) { - $attributes[] = new Document($internal); - } - return $attributes; - } - /** * Encode Document * @@ -4758,7 +4745,14 @@ public static function getInternalAttributes(): array public function encode(Document $collection, Document $document): Document { $attributes = $collection->getAttribute('attributes', []); - $attributes = \array_merge($attributes, $this->getInternalAttributes()); + + $internalAttributes = \array_filter(Database::INTERNAL_ATTRIBUTES, function ($attribute) { + // We don't want to encode permissions into a JSON string + return $attribute['$id'] !== '$permissions'; + }); + + $attributes = \array_merge($attributes, $internalAttributes); + foreach ($attributes as $attribute) { $key = $attribute['$id'] ?? ''; $array = $attribute['array'] ?? false; @@ -4835,7 +4829,7 @@ public function decode(Document $collection, Document $document, array $selectio } } - $attributes = array_merge($attributes, $this->getInternalAttributes()); + $attributes = array_merge($attributes, Database::INTERNAL_ATTRIBUTES); foreach ($attributes as $attribute) { $key = $attribute['$id'] ?? ''; diff --git a/src/Database/Validator/Index.php b/src/Database/Validator/Index.php index a0b6b80f5..6cdf9d89c 100644 --- a/src/Database/Validator/Index.php +++ b/src/Database/Validator/Index.php @@ -20,6 +20,7 @@ class Index extends Validator /** * @param array $attributes * @param int $maxLength + * @throws DatabaseException */ public function __construct(array $attributes, int $maxLength) { @@ -29,9 +30,9 @@ public function __construct(array $attributes, int $maxLength) $key = \strtolower($attribute->getAttribute('key', $attribute->getAttribute('$id'))); $this->attributes[$key] = $attribute; } - foreach (Database::getInternalAttributes() as $attribute) { - $key = \strtolower($attribute->getAttribute('$id')); - $this->attributes[$key] = $attribute; + foreach (Database::INTERNAL_ATTRIBUTES as $attribute) { + $key = \strtolower($attribute['$id']); + $this->attributes[$key] = new Document($attribute); } } diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 04a2a9f28..3cf9ecea7 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -3946,7 +3946,7 @@ public function testGetAttributeLimit(): void public function testGetIndexLimit(): void { - $this->assertEquals(58, $this->getDatabase()->getLimitForIndexes()); + $this->assertEquals(57, $this->getDatabase()->getLimitForIndexes()); } public function testGetId(): void From 381811806c6b4a2eaf4ba8f2dc933dde8181d2a5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Nov 2023 20:30:09 +1300 Subject: [PATCH 29/45] Fix int type checks account for STRINGIFY_FETCHES --- src/Database/Database.php | 23 ++++++++++++----------- src/Database/Validator/Structure.php | 4 ++-- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index fd3542473..5a0dfd2ee 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -175,8 +175,8 @@ class Database ], [ '$id' => '$tenant', - 'type' => self::VAR_STRING, - 'size' => Database::LENGTH_KEY, + 'type' => self::VAR_INTEGER, + 'size' => 0, 'required' => false, 'default' => null, 'signed' => true, @@ -907,7 +907,8 @@ public function updateCollection(string $id, array $permissions, bool $documentS throw new DatabaseException('Collection not found'); } - if ($this->adapter->getShareTables() && $collection->getAttribute('$tenant') !== $this->adapter->getTenant()) { + if ($this->adapter->getShareTables() + && $collection->getAttribute('$tenant') != $this->adapter->getTenant()) { throw new DatabaseException('Collection not found'); } @@ -940,7 +941,7 @@ public function getCollection(string $id): Document if ($id !== self::METADATA && $this->adapter->getShareTables() - && $collection->getAttribute('$tenant') !== $this->adapter->getTenant()) { + && $collection->getAttribute('$tenant') != $this->adapter->getTenant()) { return new Document(); } @@ -999,7 +1000,7 @@ public function getSizeOfCollection(string $collection): int throw new DatabaseException('Collection not found'); } - if ($this->adapter->getShareTables() && $collection->getAttribute('$tenant') !== $this->adapter->getTenant()) { + if ($this->adapter->getShareTables() && $collection->getAttribute('$tenant') != $this->adapter->getTenant()) { throw new DatabaseException('Collection not found'); } @@ -1025,7 +1026,7 @@ public function deleteCollection(string $id): bool throw new DatabaseException('Collection not found'); } - if ($this->adapter->getShareTables() && $collection->getAttribute('$tenant') !== $this->adapter->getTenant()) { + if ($this->adapter->getShareTables() && $collection->getAttribute('$tenant') != $this->adapter->getTenant()) { throw new DatabaseException('Collection not found'); } @@ -1083,7 +1084,7 @@ public function createAttribute(string $collection, string $id, string $type, in throw new DatabaseException('Collection not found'); } - if ($this->adapter->getShareTables() && $collection->getAttribute('$tenant') !== $this->adapter->getTenant()) { + if ($this->adapter->getShareTables() && $collection->getAttribute('$tenant') != $this->adapter->getTenant()) { throw new DatabaseException('Collection not found'); } @@ -4746,10 +4747,10 @@ public function encode(Document $collection, Document $document): Document { $attributes = $collection->getAttribute('attributes', []); - $internalAttributes = \array_filter(Database::INTERNAL_ATTRIBUTES, function ($attribute) { - // We don't want to encode permissions into a JSON string - return $attribute['$id'] !== '$permissions'; - }); + $internalAttributes = \array_filter(Database::INTERNAL_ATTRIBUTES, function ($attribute) { + // We don't want to encode permissions into a JSON string + return $attribute['$id'] !== '$permissions'; + }); $attributes = \array_merge($attributes, $internalAttributes); diff --git a/src/Database/Validator/Structure.php b/src/Database/Validator/Structure.php index 3d7d2ade7..9ee55a866 100644 --- a/src/Database/Validator/Structure.php +++ b/src/Database/Validator/Structure.php @@ -54,8 +54,8 @@ class Structure extends Validator ], [ '$id' => '$tenant', - 'type' => Database::VAR_STRING, - 'size' => 36, + 'type' => Database::VAR_INTEGER, + 'size' => 0, 'required' => false, 'default' => null, 'signed' => true, From 576461baac18eb0bdcfe08a542ef7776a8caee8e Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 24 Nov 2023 20:55:36 +1300 Subject: [PATCH 30/45] Treat _tenant as a string to allow structure validation account for sintrigy_fetches and mongodb not stringifying --- src/Database/Adapter/Mongo.php | 18 +++++++++--------- src/Database/Database.php | 4 ++-- src/Database/Validator/Structure.php | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index a3a29b369..2e7ba36b4 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -630,7 +630,7 @@ public function getDocument(string $collection, string $id, array $queries = []) $filters = ['_uid' => $id]; if ($this->shareTables) { - $filters['_tenant'] = $this->getTenant(); + $filters['_tenant'] = (string)$this->getTenant(); } $options = []; @@ -669,7 +669,7 @@ public function createDocument(string $collection, Document $document): Document $document ->removeAttribute('$internalId') - ->setAttribute('$tenant', $this->getTenant()); + ->setAttribute('$tenant', (string)$this->getTenant()); $record = $this->replaceChars('$', '_', (array)$document); $record = $this->timeToMongo($record); @@ -705,7 +705,7 @@ public function createDocuments(string $collection, array $documents, int $batch foreach ($documents as $document) { $document ->removeAttribute('$internalId') - ->setAttribute('$tenant', $this->getTenant()); + ->setAttribute('$tenant', (string)$this->getTenant()); $record = $this->replaceChars('$', '_', (array)$document); $record = $this->timeToMongo($record); @@ -741,7 +741,7 @@ private function insertDocument(string $name, array $document): array $filters = []; $filters['_uid'] = $document['_uid']; if ($this->shareTables) { - $filters['_tenant'] = $this->getTenant(); + $filters['_tenant'] = (string)$this->getTenant(); } $result = $this->client->find( @@ -776,7 +776,7 @@ public function updateDocument(string $collection, Document $document): Document $filters = []; $filters['_uid'] = $document->getId(); if ($this->shareTables) { - $filters['_tenant'] = $this->getTenant(); + $filters['_tenant'] = (string)$this->getTenant(); } try { @@ -811,7 +811,7 @@ public function updateDocuments(string $collection, array $documents, int $batch $filters = []; $filters['_uid'] = $document['_uid']; if ($this->shareTables) { - $filters['_tenant'] = $this->getTenant(); + $filters['_tenant'] = (string)$this->getTenant(); } $this->client->update($name, $filters, $document); @@ -840,7 +840,7 @@ public function increaseDocumentAttribute(string $collection, string $id, string $filters = ['_uid' => $id]; if ($this->shareTables) { - $filters['_tenant'] = $this->getTenant(); + $filters['_tenant'] = (string)$this->getTenant(); } if ($max) { @@ -876,7 +876,7 @@ public function deleteDocument(string $collection, string $id): bool $filters = []; $filters['_uid'] = $id; if ($this->shareTables) { - $filters['_tenant'] = $this->getTenant(); + $filters['_tenant'] = (string)$this->getTenant(); } $result = $this->client->delete($name, $filters); @@ -925,7 +925,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $filters = $this->buildFilters($queries); if ($this->shareTables) { - $filters['_tenant'] = $this->getTenant(); + $filters['_tenant'] = (string)$this->getTenant(); } // permissions diff --git a/src/Database/Database.php b/src/Database/Database.php index 5a0dfd2ee..a017b4960 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -175,8 +175,8 @@ class Database ], [ '$id' => '$tenant', - 'type' => self::VAR_INTEGER, - 'size' => 0, + 'type' => self::VAR_STRING, + 'size' => 36, 'required' => false, 'default' => null, 'signed' => true, diff --git a/src/Database/Validator/Structure.php b/src/Database/Validator/Structure.php index 9ee55a866..3d7d2ade7 100644 --- a/src/Database/Validator/Structure.php +++ b/src/Database/Validator/Structure.php @@ -54,8 +54,8 @@ class Structure extends Validator ], [ '$id' => '$tenant', - 'type' => Database::VAR_INTEGER, - 'size' => 0, + 'type' => Database::VAR_STRING, + 'size' => 36, 'required' => false, 'default' => null, 'signed' => true, From 04e233399db7f1808261c1bf061b7e7954baef89 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 27 Nov 2023 14:43:04 +1300 Subject: [PATCH 31/45] Optimize unique index order for tenant uid --- src/Database/Adapter/MariaDB.php | 5 ++--- src/Database/Adapter/Postgres.php | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index fc90a1acd..2415ae6cf 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -128,7 +128,7 @@ public function createCollection(string $name, array $attributes = [], array $in if ($this->shareTables) { $sql .= " - UNIQUE KEY _uid_tenant (_tenant, _uid), + UNIQUE KEY _uid_tenant (_uid, _tenant), KEY _created_at (_tenant, _createdAt), KEY _updated_at (_tenant, _updatedAt) "; @@ -857,8 +857,7 @@ public function createDocuments(string $collection, array $documents, int $batch } } - $stmt = $this->getPDO()->prepare( - " + $stmt = $this->getPDO()->prepare(" INSERT INTO {$this->getSQLTable($name)} {$columns} VALUES " . \implode(', ', $batchKeys) ); diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 9dca56cfa..e377d599e 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -113,7 +113,7 @@ public function createCollection(string $name, array $attributes = [], array $in if ($this->shareTables) { $sql .= " - CREATE UNIQUE INDEX \"{$namespace}_{$this->tenant}_{$id}_uid\" ON {$this->getSQLTable($id)} (_tenant, LOWER(_uid)); + CREATE UNIQUE INDEX \"{$namespace}_{$this->tenant}_{$id}_uid\" ON {$this->getSQLTable($id)} (LOWER(_uid), _tenant); CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_created\" ON {$this->getSQLTable($id)} (_tenant, \"_createdAt\"); CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_updated\" ON {$this->getSQLTable($id)} (_tenant, \"_updatedAt\"); "; From 3c9da71d05de9e3c956c8d4eedfa7e40e2701415 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 27 Nov 2023 14:48:00 +1300 Subject: [PATCH 32/45] Assert fetched tenant matches inserted --- src/Database/Adapter/MariaDB.php | 3 ++- tests/e2e/Adapter/Base.php | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 2415ae6cf..f1b97a424 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -857,7 +857,8 @@ public function createDocuments(string $collection, array $documents, int $batch } } - $stmt = $this->getPDO()->prepare(" + $stmt = $this->getPDO()->prepare( + " INSERT INTO {$this->getSQLTable($name)} {$columns} VALUES " . \implode(', ', $batchKeys) ); diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 3cf9ecea7..870d4c586 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -12634,6 +12634,7 @@ public function testIsolationModes(): void $doc = $database->getDocument('people', $docId); $this->assertEquals('Spiderman', $doc['name']); + $this->assertEquals($tenant1, $doc->getAttribute('$tenant')); $docs = $database->find('people'); $this->assertEquals(1, \count($docs)); From f838c9dcc77bf86cb9fd87ae551ccf9dc8680adc Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 27 Nov 2023 14:49:21 +1300 Subject: [PATCH 33/45] Format --- tests/e2e/Adapter/Base.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 870d4c586..e7d498106 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -12634,7 +12634,7 @@ public function testIsolationModes(): void $doc = $database->getDocument('people', $docId); $this->assertEquals('Spiderman', $doc['name']); - $this->assertEquals($tenant1, $doc->getAttribute('$tenant')); + $this->assertEquals($tenant1, $doc->getAttribute('$tenant')); $docs = $database->find('people'); $this->assertEquals(1, \count($docs)); From 8756141bef49993643acea5120314b9390bfe30a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 27 Nov 2023 20:58:37 +1300 Subject: [PATCH 34/45] Allow null tenant --- src/Database/Adapter.php | 6 +++--- src/Database/Database.php | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index fa8b3f9ac..764f41df6 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -162,11 +162,11 @@ public function getShareTables(): bool * * Set tenant to use if tables are shared * - * @param int $tenant + * @param ?int $tenant * * @return bool */ - public function setTenant(int $tenant): bool + public function setTenant(?int $tenant): bool { $this->tenant = $tenant; @@ -753,7 +753,7 @@ protected function getAttributeSelections(array $queries): array */ public function filter(string $value): string { - $value = preg_replace("/[^A-Za-z0-9\_\-]/", '', $value); + $value = \preg_replace("/[^A-Za-z0-9\_\-]/", '', $value); if (\is_null($value)) { throw new DatabaseException('Failed to filter key'); diff --git a/src/Database/Database.php b/src/Database/Database.php index 73eb15574..8eee2c7f3 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -704,10 +704,10 @@ public function setShareTables(bool $share): self * * Set tenant to use if tables are shared * - * @param int $tenant + * @param ?int $tenant * @return self */ - public function setTenant(int $tenant): self + public function setTenant(?int $tenant): self { $this->adapter->setTenant($tenant); From 4b0f8978b26ec63a52060ced1f0d0f0a7b7a302d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Nov 2023 00:25:26 +1300 Subject: [PATCH 35/45] Add exception check --- tests/e2e/Adapter/Base.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index e10e7f571..0611afde2 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -12715,6 +12715,16 @@ public function testIsolationModes(): void $docs = $database->find('people'); $this->assertEquals(1, \count($docs)); + // Remove tenant but leave shared tables enabled + $database->setTenant(null); + + try { + $database->getDocument('people', $docId); + $this->fail('Failed to throw exception'); + } catch (Exception $e) { + $this->assertEquals('Missing tenant. Tenant must be set when table sharing is enabled.', $e->getMessage()); + } + // Reset state $database->setShareTables(false); $database->setNamespace(static::$namespace); From 1dd550b013e164075cc449674c25e69992e8cba3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Nov 2023 00:34:07 +1300 Subject: [PATCH 36/45] Optimize permission indexes --- src/Database/Adapter/MariaDB.php | 4 ++-- src/Database/Adapter/Postgres.php | 4 ++-- src/Database/Adapter/SQLite.php | 2 +- tests/e2e/Adapter/Base.php | 18 +++++++++--------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index f1b97a424..4f6df3bdd 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -162,12 +162,12 @@ public function createCollection(string $name, array $attributes = [], array $in if ($this->shareTables) { $sql .= " UNIQUE INDEX _index1 (_document, _tenant, _type, _permission), - INDEX _permission (_permission, _type, _document, _tenant) + INDEX _permission (_permission, _type, _tenant) "; } else { $sql .= " UNIQUE INDEX _index1 (_document, _type, _permission), - INDEX _permission (_permission, _type, _document) + INDEX _permission (_permission, _type) "; } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index e377d599e..952227f41 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -148,14 +148,14 @@ public function createCollection(string $name, array $attributes = [], array $in CREATE UNIQUE INDEX \"{$namespace}_{$this->tenant}_{$id}_ukey\" ON {$this->getSQLTable($id. '_perms')} USING btree (_tenant,_document,_type,_permission); CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_permission\" - ON {$this->getSQLTable($id. '_perms')} USING btree (_permission,_type,_document,_tenant); + ON {$this->getSQLTable($id. '_perms')} USING btree (_permission,_type,_tenant); "; } else { $sql .= " CREATE UNIQUE INDEX \"{$namespace}_{$id}_ukey\" ON {$this->getSQLTable($id. '_perms')} USING btree (_document,_type,_permission); CREATE INDEX \"{$namespace}_{$id}_permission\" - ON {$this->getSQLTable($id. '_perms')} USING btree (_permission,_type,_document); + ON {$this->getSQLTable($id. '_perms')} USING btree (_permission,_type); "; } diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 22901cd2a..4ab96ad5d 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -181,7 +181,7 @@ public function createCollection(string $name, array $attributes = [], array $in ->execute(); $this->createIndex("{$id}_perms", '_index_1', Database::INDEX_UNIQUE, ['_document', '_type', '_permission'], [], []); - $this->createIndex("{$id}_perms", '_index_2', Database::INDEX_KEY, ['_permission'], [], []); + $this->createIndex("{$id}_perms", '_index_2', Database::INDEX_KEY, ['_permission', '_type'], [], []); $this->getPDO()->commit(); diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 0611afde2..641515afc 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -12715,15 +12715,15 @@ public function testIsolationModes(): void $docs = $database->find('people'); $this->assertEquals(1, \count($docs)); - // Remove tenant but leave shared tables enabled - $database->setTenant(null); - - try { - $database->getDocument('people', $docId); - $this->fail('Failed to throw exception'); - } catch (Exception $e) { - $this->assertEquals('Missing tenant. Tenant must be set when table sharing is enabled.', $e->getMessage()); - } + // Remove tenant but leave shared tables enabled + $database->setTenant(null); + + try { + $database->getDocument('people', $docId); + $this->fail('Failed to throw exception'); + } catch (Exception $e) { + $this->assertEquals('Missing tenant. Tenant must be set when table sharing is enabled.', $e->getMessage()); + } // Reset state $database->setShareTables(false); From ba6d104d2d0ac2de115bb426b01cdcbdc5143ce6 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Nov 2023 00:35:51 +1300 Subject: [PATCH 37/45] Optimize permission index order --- src/Database/Adapter/MariaDB.php | 2 +- src/Database/Adapter/Postgres.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 4f6df3bdd..bbfad0c82 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -162,7 +162,7 @@ public function createCollection(string $name, array $attributes = [], array $in if ($this->shareTables) { $sql .= " UNIQUE INDEX _index1 (_document, _tenant, _type, _permission), - INDEX _permission (_permission, _type, _tenant) + INDEX _permission (_tenant, _permission, _type) "; } else { $sql .= " diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 952227f41..0f28cc785 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -148,7 +148,7 @@ public function createCollection(string $name, array $attributes = [], array $in CREATE UNIQUE INDEX \"{$namespace}_{$this->tenant}_{$id}_ukey\" ON {$this->getSQLTable($id. '_perms')} USING btree (_tenant,_document,_type,_permission); CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_permission\" - ON {$this->getSQLTable($id. '_perms')} USING btree (_permission,_type,_tenant); + ON {$this->getSQLTable($id. '_perms')} USING btree (_tenant,_permission,_type); "; } else { $sql .= " From 989eaf64ffc8f225b868153f94e34b4466136d75 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Nov 2023 01:26:06 +1300 Subject: [PATCH 38/45] Remove index --- src/Database/Database.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 8eee2c7f3..ca7920ac1 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -223,8 +223,7 @@ class Database '_createdAt', '_updatedAt', '_permissions_id', - '_permissions_forwards', - '_permissions_backwards', + '_permissions', ]; /** From c1341e2daa90aaa93511a98838cd48243aa2eae5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 28 Nov 2023 01:29:12 +1300 Subject: [PATCH 39/45] Fix index count test --- tests/e2e/Adapter/Base.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 641515afc..cfaf5f4c3 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -3946,7 +3946,7 @@ public function testGetAttributeLimit(): void public function testGetIndexLimit(): void { - $this->assertEquals(57, $this->getDatabase()->getLimitForIndexes()); + $this->assertEquals(58, $this->getDatabase()->getLimitForIndexes()); } public function testGetId(): void From fbdacca075acfd348c5b797371aa95c6d627d246 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 30 Nov 2023 16:53:02 +1300 Subject: [PATCH 40/45] Don't cover tenant on fulltext indexes --- src/Database/Adapter/MariaDB.php | 6 +++--- src/Database/Adapter/Postgres.php | 6 +++--- tests/e2e/Adapter/Base.php | 22 ++++++++++++++++++++-- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index bbfad0c82..b7a940728 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1988,7 +1988,7 @@ protected function getSQLType(string $type, int $size, bool $signed = true): str */ protected function getSQLIndex(string $collection, string $id, string $type, array $attributes): string { - $type = match ($type) { + $sqlType = match ($type) { Database::INDEX_KEY, Database::INDEX_ARRAY => 'INDEX', Database::INDEX_UNIQUE => 'UNIQUE INDEX', @@ -1998,12 +1998,12 @@ protected function getSQLIndex(string $collection, string $id, string $type, arr $attributes = \implode(', ', $attributes); - if ($this->shareTables) { + if ($this->shareTables && $type !== Database::INDEX_FULLTEXT) { // Add tenant as first index column for best performance $attributes = "_tenant, {$attributes}"; } - return "CREATE {$type} `{$id}` ON {$this->getSQLTable($collection)} ({$attributes})"; + return "CREATE {$sqlType} `{$id}` ON {$this->getSQLTable($collection)} ({$attributes})"; } /** diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 0f28cc785..56c59cb83 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1978,7 +1978,7 @@ protected function getSQLType(string $type, int $size, bool $signed = true): str */ protected function getSQLIndex(string $collection, string $id, string $type, array $attributes): string { - $type = match ($type) { + $sqlType = match ($type) { Database::INDEX_KEY, Database::INDEX_ARRAY, Database::INDEX_FULLTEXT => 'INDEX', @@ -1989,12 +1989,12 @@ protected function getSQLIndex(string $collection, string $id, string $type, arr $key = "\"{$this->getNamespace()}_{$this->tenant}_{$collection}_{$id}\""; $attributes = \implode(', ', $attributes); - if ($this->shareTables) { + if ($this->shareTables && $type !== Database::INDEX_FULLTEXT) { // Add tenant as first index column for best performance $attributes = "_tenant, {$attributes}"; } - return "CREATE {$type} {$key} ON {$this->getSQLTable($collection)} ({$attributes});"; + return "CREATE {$sqlType} {$key} ON {$this->getSQLTable($collection)} ({$attributes});"; } /** diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index cfaf5f4c3..632b9c1d4 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -12670,8 +12670,25 @@ public function testIsolationModes(): void 'type' => Database::VAR_STRING, 'size' => 128, 'required' => true, - ]) - ]); + ]), + new Document([ + '$id' => 'lifeStory', + 'type' => Database::VAR_STRING, + 'size' => 65536, + 'required' => true, + ]) + ], [ + new Document([ + '$id' => 'idx_name', + 'type' => Database::INDEX_KEY, + 'attributes' => ['name'] + ]), + new Document([ + '$id' => 'idx_lifeStory', + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['lifeStory'] + ]), + ]); $docId = ID::unique(); @@ -12681,6 +12698,7 @@ public function testIsolationModes(): void Permission::read(Role::any()), ], 'name' => 'Spiderman', + 'lifeStory' => 'Spider-Man is a superhero appearing in American comic books published by Marvel Comics.' ])); $doc = $database->getDocument('people', $docId); From 03619294ae4cbaf13fd5aa5ce825834145a17eb4 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 30 Nov 2023 18:29:49 +1300 Subject: [PATCH 41/45] Format --- tests/e2e/Adapter/Base.php | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 632b9c1d4..16872a6b3 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -12671,24 +12671,24 @@ public function testIsolationModes(): void 'size' => 128, 'required' => true, ]), - new Document([ - '$id' => 'lifeStory', - 'type' => Database::VAR_STRING, - 'size' => 65536, - 'required' => true, - ]) + new Document([ + '$id' => 'lifeStory', + 'type' => Database::VAR_STRING, + 'size' => 65536, + 'required' => true, + ]) ], [ - new Document([ - '$id' => 'idx_name', - 'type' => Database::INDEX_KEY, - 'attributes' => ['name'] - ]), - new Document([ - '$id' => 'idx_lifeStory', - 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['lifeStory'] - ]), - ]); + new Document([ + '$id' => 'idx_name', + 'type' => Database::INDEX_KEY, + 'attributes' => ['name'] + ]), + new Document([ + '$id' => 'idx_lifeStory', + 'type' => Database::INDEX_FULLTEXT, + 'attributes' => ['lifeStory'] + ]), + ]); $docId = ID::unique(); @@ -12698,7 +12698,7 @@ public function testIsolationModes(): void Permission::read(Role::any()), ], 'name' => 'Spiderman', - 'lifeStory' => 'Spider-Man is a superhero appearing in American comic books published by Marvel Comics.' + 'lifeStory' => 'Spider-Man is a superhero appearing in American comic books published by Marvel Comics.' ])); $doc = $database->getDocument('people', $docId); From 1711f3120974300541e1c8a79f014b651b5b0460 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 30 Nov 2023 22:38:37 +1300 Subject: [PATCH 42/45] Fix sqlite test --- tests/e2e/Adapter/Base.php | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 16872a6b3..925dbe0b7 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -12682,14 +12682,18 @@ public function testIsolationModes(): void '$id' => 'idx_name', 'type' => Database::INDEX_KEY, 'attributes' => ['name'] - ]), - new Document([ - '$id' => 'idx_lifeStory', - 'type' => Database::INDEX_FULLTEXT, - 'attributes' => ['lifeStory'] - ]), + ]) ]); + if ($database->getAdapter()->getSupportForFulltextIndex()) { + $database->createIndex( + collection: 'people', + id: 'idx_lifeStory', + type: Database::INDEX_FULLTEXT, + attributes: ['lifeStory'] + ); + } + $docId = ID::unique(); $database->createDocument('people', new Document([ From 7fd50707242d0cf6bb3e6b5afe5dc1c92557cedf Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 30 Nov 2023 22:40:58 +1300 Subject: [PATCH 43/45] Format --- tests/e2e/Adapter/Base.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 925dbe0b7..95cdd9cae 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -12685,14 +12685,14 @@ public function testIsolationModes(): void ]) ]); - if ($database->getAdapter()->getSupportForFulltextIndex()) { - $database->createIndex( - collection: 'people', - id: 'idx_lifeStory', - type: Database::INDEX_FULLTEXT, - attributes: ['lifeStory'] - ); - } + if ($database->getAdapter()->getSupportForFulltextIndex()) { + $database->createIndex( + collection: 'people', + id: 'idx_lifeStory', + type: Database::INDEX_FULLTEXT, + attributes: ['lifeStory'] + ); + } $docId = ID::unique(); From 426476b9030b1e7095edc32d79f2ef9a1ce212ca Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 8 Dec 2023 14:25:33 +0100 Subject: [PATCH 44/45] Add tenant + internal ID index for pagination query speed --- src/Database/Adapter/MariaDB.php | 1 + src/Database/Adapter/Postgres.php | 1 + src/Database/Adapter/SQLite.php | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index b7a940728..78a3d5750 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -131,6 +131,7 @@ public function createCollection(string $name, array $attributes = [], array $in UNIQUE KEY _uid_tenant (_uid, _tenant), KEY _created_at (_tenant, _createdAt), KEY _updated_at (_tenant, _updatedAt) + KEY _tenant_id (_tenant, _id) "; } else { $sql .= " diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 56c59cb83..b92a780fb 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -116,6 +116,7 @@ public function createCollection(string $name, array $attributes = [], array $in CREATE UNIQUE INDEX \"{$namespace}_{$this->tenant}_{$id}_uid\" ON {$this->getSQLTable($id)} (LOWER(_uid), _tenant); CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_created\" ON {$this->getSQLTable($id)} (_tenant, \"_createdAt\"); CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_updated\" ON {$this->getSQLTable($id)} (_tenant, \"_updatedAt\"); + CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_tenant_id\" ON {$this->getSQLTable($id)} (_tenant, _id); "; } else { $sql .= " diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 4ab96ad5d..c4ea3a9f0 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -154,6 +154,10 @@ public function createCollection(string $name, array $attributes = [], array $in $this->createIndex($id, '_created_at', Database::INDEX_KEY, [ '_createdAt'], [], []); $this->createIndex($id, '_updated_at', Database::INDEX_KEY, [ '_updatedAt'], [], []); + if ($this->shareTables) { + $this->createIndex($id, '_tenant_id', Database::INDEX_KEY, [ '_id'], [], []); + } + foreach ($indexes as $index) { $indexId = $this->filter($index->getId()); $indexType = $index->getAttribute('type'); From 539892230f4d39b1c9bd5e7831cbfc92fecf221b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 8 Dec 2023 15:47:23 +0100 Subject: [PATCH 45/45] Fix syntax --- src/Database/Adapter/MariaDB.php | 2 +- src/Database/Adapter/SQLite.php | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 78a3d5750..35e6b2103 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -130,7 +130,7 @@ public function createCollection(string $name, array $attributes = [], array $in $sql .= " UNIQUE KEY _uid_tenant (_uid, _tenant), KEY _created_at (_tenant, _createdAt), - KEY _updated_at (_tenant, _updatedAt) + KEY _updated_at (_tenant, _updatedAt), KEY _tenant_id (_tenant, _id) "; } else { diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index c4ea3a9f0..5e9cffd61 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -154,9 +154,9 @@ public function createCollection(string $name, array $attributes = [], array $in $this->createIndex($id, '_created_at', Database::INDEX_KEY, [ '_createdAt'], [], []); $this->createIndex($id, '_updated_at', Database::INDEX_KEY, [ '_updatedAt'], [], []); - if ($this->shareTables) { - $this->createIndex($id, '_tenant_id', Database::INDEX_KEY, [ '_id'], [], []); - } + if ($this->shareTables) { + $this->createIndex($id, '_tenant_id', Database::INDEX_KEY, [ '_id'], [], []); + } foreach ($indexes as $index) { $indexId = $this->filter($index->getId());