Skip structure validation for now in createDocuments

src/Database/Adapter/MariaDB.php

@@ -496,12 +496,23 @@ public function createDocument(string $collection, Document $document): Document
             $bindIndex++;
         }
 
+        // Insert manual id if set
+        if (!empty($document->getInternalId())) {
+            $bindKey = '_id';
+            $columns .= " _id" . ' = :' . $bindKey . ',';
+        }
+
         $stmt = $this->getPDO()
             ->prepare("INSERT INTO {$this->getSQLTable($name)}
                 SET {$columns} _uid = :_uid");
 
         $stmt->bindValue(':_uid', $document->getId(), PDO::PARAM_STR);
 
+        // Bind manual internal id if set
+        if (!empty($document->getInternalId())) {
+            $stmt->bindValue(':_id', $document->getInternalId(), PDO::PARAM_STR);
+        }
+
         $attributeIndex = 0;
         foreach ($attributes as $attribute => $value) {
             if (is_array($value)) { // arrays & objects should be saved as strings
@@ -531,11 +542,7 @@ public function createDocument(string $collection, Document $document): Document
         try {
             $stmt->execute();
 
-            $statment = $this->getPDO()->prepare("select last_insert_id() as id");
-            $statment->execute();
-            $last = $statment->fetch();
-            $document['$internalId'] = $last['id'];
-
+            $document['$internalId'] = $this->getDocument($collection, $document->getId())->getInternalId();
             if (isset($stmtPermissions)) {
                 $stmtPermissions->execute();
             }
@@ -585,6 +592,9 @@ public function createDocuments(string $collection, array $documents, int $batch
                     $attributes['_createdAt'] = $document->getCreatedAt();
                     $attributes['_updatedAt'] = $document->getUpdatedAt();
                     $attributes['_permissions'] = \json_encode($document->getPermissions());
+                    if(!empty($document->getInternalId())) {
+                        $attributes['_id'] = $document->getInternalId();
+                    }
 
                     $columns = [];
                     foreach (\array_keys($attributes) as $key => $attribute) {

src/Database/Adapter/Mongo/MongoDBAdapter.php

@@ -386,6 +386,11 @@ public function createDocument(string $collection, Document $document): Document
 
         $record =  $this->replaceChars('$', '_', $document->getArrayCopy());
 
+        // Insert manual id if set
+        if (!empty($internalId)) {
+            $record['_id'] = $internalId;
+        }
+
         $this->client->insert($name, $record);
 
         return $document;
@@ -397,6 +402,9 @@ public function createDocuments(string $collection, array $documents, int $batch
 
         $records = [];
         foreach ($documents as $document) {
+            if(!empty($document->getInternalId())) {
+                $document->setAttribute('_id', $document->getInternalId());
+            }
             $document->removeAttribute('$internalId');
 
             $records[] = $this->replaceChars('$', '_', (array)$document);

src/Database/Adapter/SQLite.php

@@ -319,12 +319,23 @@ public function createDocument(string $collection, Document $document): Document
             $bindIndex++;
         }
 
+         // Insert manual id if set
+         if (!empty($document->getInternalId())) {
+            $values[] = '_id';
+            $columns[] = "_id";
+        }
+
         $stmt = $this->getPDO()
             ->prepare("INSERT INTO `{$this->getNamespace()}_{$name}`
                 (".implode(', ', $columns).") VALUES (:".implode(', :', $values).");");
 
         $stmt->bindValue(':_uid', $document->getId(), PDO::PARAM_STR);
 
+        // Bind manual internal id if set
+        if (!empty($document->getInternalId())) {
+            $stmt->bindValue(':_id', $document->getInternalId(), PDO::PARAM_STR);
+        }
+
         $attributeIndex = 0;
         foreach ($attributes as $attribute => $value) {
             if (is_array($value)) { // arrays & objects should be saved as strings

src/Database/Database.php

@@ -108,6 +108,8 @@ class Database
      */
     protected Cache $cache;
 
+    protected string $name;
+
     /**
      * @var array
      */
@@ -206,6 +208,16 @@ class Database
                 'array' => false,
                 'filters' => ['json'],
             ],
+            [
+                '$id' => 'documentSecurity',
+                'key' => 'documentSecurity',
+                'type' => self::VAR_BOOLEAN,
+                'size' => 0,
+                'required' => false,
+                'signed' => true,
+                'array' => false,
+                'filters' => []
+            ]
         ],
         'indexes' => [],
     ];
@@ -236,11 +248,12 @@ class Database
      * @param Adapter $adapter
      * @param Cache $cache
      */
-    public function __construct(Adapter $adapter, Cache $cache, array $filters = [])
+    public function __construct(Adapter $adapter, Cache $cache, array $filters = [], String $name = 'default')
     {
         $this->adapter = $adapter;
         $this->cache = $cache;
         $this->instanceFilters = $filters;
+        $this->name = $name;
 
         self::addFilter(
             'json',
@@ -1289,7 +1302,11 @@ public function deleteIndex(string $collection, string $id): bool
     public function getDocument(string $collection, string $id): Document
     {
         if ($collection === self::METADATA && $id === self::METADATA) {
-            return new Document($this->collection);
+            $coll = $this->collection;
+            if(!str_contains($this->name, 'v14x') || $this->getNamespace() === '_console') {
+                unset($coll['attributes'][3]);
+            }
+            return new Document($coll);
         }
 
         if (empty($collection)) {
@@ -1303,7 +1320,7 @@ public function getDocument(string $collection, string $id): Document
         $validator = new Authorization(self::PERMISSION_READ);
 
         // TODO@kodumbeats Check if returned cache id matches request
-        if ($cache = $this->cache->load('cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $id, self::TTL)) {
+        if ($cache = $this->cache->load($this->name . '-cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $id, self::TTL)) {
             $document = new Document($cache);
 
             if ($collection->getId() !== self::METADATA
@@ -1330,7 +1347,7 @@ public function getDocument(string $collection, string $id): Document
         $document = $this->casting($collection, $document);
         $document = $this->decode($collection, $document);
 
-        $this->cache->save('cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $id, $document->getArrayCopy()); // save to cache after fetching from db
+        $this->cache->save($this->name . '-cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $id, $document->getArrayCopy()); // save to cache after fetching from db
 
         $this->trigger(self::EVENT_DOCUMENT_READ, $document);
         return $document;
@@ -1380,7 +1397,7 @@ public function createDocument(string $collection, Document $document): Document
      * Create Documents in a batch
      *
      * @param string $collection
-     * @param Document $document
+     * @param array<string, Document> $documents
      *
      * @return Document
      *
@@ -1399,18 +1416,22 @@ public function createDocuments(string $collection, array $documents, int $batch
         $time = DateTime::now();
 
         foreach ($documents as $key => $document) {
+            $documentId = $document->getId();
+            $createdAt = $document->getCreatedAt();
+            $updatedAt = $document->getUpdatedAt();
             $document
-                ->setAttribute('$id', empty($document->getId()) ? ID::unique() : $document->getId())
+                ->setAttribute('$id', empty($documentId) ? ID::unique() : $documentId)
                 ->setAttribute('$collection', $collection->getId())
-                ->setAttribute('$createdAt', $time)
-                ->setAttribute('$updatedAt', $time);
+                ->setAttribute('$createdAt', empty($createdAt) ? $time : $createdAt)
+                ->setAttribute('$updatedAt', empty($updatedAt) ? $time : $updatedAt);
 
             $document = $this->encode($collection, $document);
 
-            $validator = new Structure($collection);
-            if (!$validator->isValid($document)) {
-                throw new StructureException($validator->getDescription());
-            }
+            // TODO find a way to skip with special parameter for migrations
+            // $validator = new Structure($collection);
+            // if (!$validator->isValid($document)) {
+            //     throw new StructureException($validator->getDescription());
+            // }
 
             $documents[$key] = $document;
         }
@@ -1466,7 +1487,7 @@ public function updateDocument(string $collection, string $id, Document $documen
         $document = $this->adapter->updateDocument($collection->getId(), $document);
         $document = $this->decode($collection, $document);
 
-        $this->cache->purge('cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $id);
+        $this->cache->purge($this->name . '-cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $id);
 
         $this->trigger(self::EVENT_DOCUMENT_UPDATE, $document);
         return $document;
@@ -1516,7 +1537,7 @@ public function updateDocuments(string $collection, array $documents, int $batch
          foreach ($documents as $key => $document) {
              $documents[$key] = $this->decode($collection, $document);
 
-             $this->cache->purge('cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $document->getId() . ':*');
+             $this->cache->purge($this->name . '-cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $document->getId() . ':*');
          }
 
          $this->trigger(self::EVENT_DOCUMENTS_UPDATE, $documents);
@@ -1546,7 +1567,7 @@ public function deleteDocument(string $collection, string $id): bool
             throw new AuthorizationException($validator->getDescription());
         }
 
-        $this->cache->purge('cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $id);
+        $this->cache->purge($this->name . '-cache-' . $this->getNamespace() . ':' . $collection->getId() . ':' . $id);
 
         $deleted = $this->adapter->deleteDocument($collection->getId(), $id);
         $this->trigger(self::EVENT_DOCUMENT_DELETE, $document);
@@ -1562,7 +1583,7 @@ public function deleteDocument(string $collection, string $id): bool
      */
     public function deleteCachedCollection(string $collection): bool
     {
-        return $this->cache->purge('cache-' . $this->getNamespace() . ':' . $collection . ':*');
+        return $this->cache->purge($this->name . '-cache-' . $this->getNamespace() . ':' . $collection . ':*');
     }
 
     /**
@@ -1575,7 +1596,7 @@ public function deleteCachedCollection(string $collection): bool
      */
     public function deleteCachedDocument(string $collection, string $id): bool
     {
-        return $this->cache->purge('cache-' . $this->getNamespace() . ':' . $collection . ':' . $id);
+        return $this->cache->purge($this->name . '-cache-' . $this->getNamespace() . ':' . $collection . ':' . $id);
     }
 
     /**
@@ -1773,6 +1794,11 @@ public function encode(Document $collection, Document $document): Document
             }
 
             $document->setAttribute($key, $value);
+            $documentId = $document->getId();
+            $keepDocumentSecurity = str_starts_with($documentId, 'database_') && !str_contains($documentId, '_collection_');
+            if ($collection->getId() === self::METADATA && !$keepDocumentSecurity && !str_contains($this->name, 'v14x') && $this->getNamespace() != '_console') {
+                $document->removeAttribute('documentSecurity');
+            }
         }
 
         return $document;
@@ -1807,6 +1833,11 @@ public function decode(Document $collection, Document $document): Document
             }
 
             $document->setAttribute($key, ($array) ? $value : $value[0]);
+            $documentId = $document->getId();
+            $keepDocumentSecurity = str_starts_with($documentId, 'database_') && !str_contains($documentId, '_collection_');
+            if ($collection->getId() === self::METADATA && !$keepDocumentSecurity && !str_contains($this->name, 'v14x') && $this->getNamespace() != '_console') {
+                $document->removeAttribute('documentSecurity');
+            }
         }
 
         return $document;
@@ -1866,6 +1897,11 @@ public function casting(Document $collection, Document $document): Document
             }
 
             $document->setAttribute($key, ($array) ? $value : $value[0]);
+            $documentId = $document->getId();
+            $keepDocumentSecurity = str_starts_with($documentId, 'database_') && !str_contains($documentId, '_collection_');
+            if ($collection->getId() === self::METADATA && !$keepDocumentSecurity && !str_contains($this->name, 'v14x') && $this->getNamespace() != '_console') {
+                $document->removeAttribute('documentSecurity');
+            }
         }
 
         return $document;

src/Database/Document.php

@@ -226,7 +226,11 @@ public function setAttribute(string $key, $value, string $type = self::SET_TYPE_
      */
     public function removeAttribute(string $key): self
     {
-        if (isset($this[$key])) {
+        // isset fails when explicitely set to null
+        // prevents from removing if the key is set
+        // as null value explicitly
+        // if (isset($this[$key])) {
+        if(array_key_exists($key, $this->getArrayCopy())) {
             unset($this[$key]);
         }
 
@@ -397,4 +401,4 @@ public function getArrayCopy(array $allow = [], array $disallow = []): array
 
         return $output;
     }
-}
+}

src/Database/Validator/Structure.php

@@ -34,7 +34,7 @@ class Structure extends Validator
         [
             '$id' => '$internalId',
             'type' => Database::VAR_STRING,
-            'size' => 64,
+            'size' => 0,
             'required' => false,
             'signed' => true,
             'array' => false,

tests/Database/Base.php

@@ -605,6 +605,50 @@ public function testCreateDocument()
         $this->assertEquals([], $document->getAttribute('empty'));
         $this->assertEquals('Works', $document->getAttribute('with-dash'));
 
+        // Test create document with manual internal id
+        $manualIdDocument = static::getDatabase()->createDocument('documents', new Document([
+            '$internalId' => '56000',
+            '$permissions' => [
+                Permission::read(Role::any()),
+                Permission::read(Role::user(ID::custom('1'))),
+                Permission::read(Role::user(ID::custom('2'))),
+                Permission::create(Role::any()),
+                Permission::create(Role::user(ID::custom('1x'))),
+                Permission::create(Role::user(ID::custom('2x'))),
+                Permission::update(Role::any()),
+                Permission::update(Role::user(ID::custom('1x'))),
+                Permission::update(Role::user(ID::custom('2x'))),
+                Permission::delete(Role::any()),
+                Permission::delete(Role::user(ID::custom('1x'))),
+                Permission::delete(Role::user(ID::custom('2x'))),
+            ],
+            'string' => 'text📝',
+            'integer' => 5,
+            'bigint' => 8589934592, // 2^33
+            'float' => 5.55,
+            'boolean' => true,
+            'colors' => ['pink', 'green', 'blue'],
+            'empty' => [],
+            'with-dash' => 'Works',
+        ]));
+
+        $this->assertEquals('56000', $manualIdDocument->getInternalId());
+        $this->assertNotEmpty(true, $manualIdDocument->getId());
+        $this->assertIsString($manualIdDocument->getAttribute('string'));
+        $this->assertEquals('text📝', $manualIdDocument->getAttribute('string')); // Also makes sure an emoji is working
+        $this->assertIsInt($manualIdDocument->getAttribute('integer'));
+        $this->assertEquals(5, $manualIdDocument->getAttribute('integer'));
+        $this->assertIsInt($manualIdDocument->getAttribute('bigint'));
+        $this->assertEquals(8589934592, $manualIdDocument->getAttribute('bigint'));
+        $this->assertIsFloat($manualIdDocument->getAttribute('float'));
+        $this->assertEquals(5.55, $manualIdDocument->getAttribute('float'));
+        $this->assertIsBool($manualIdDocument->getAttribute('boolean'));
+        $this->assertEquals(true, $manualIdDocument->getAttribute('boolean'));
+        $this->assertIsArray($manualIdDocument->getAttribute('colors'));
+        $this->assertEquals(['pink', 'green', 'blue'], $manualIdDocument->getAttribute('colors'));
+        $this->assertEquals([], $manualIdDocument->getAttribute('empty'));
+        $this->assertEquals('Works', $manualIdDocument->getAttribute('with-dash'));
+
         return $document;
     }
 
@@ -2636,6 +2680,7 @@ public function testExceptionDuplicate(Document $document)
     public function testExceptionCaseInsensitiveDuplicate(Document $document)
     {
         $document->setAttribute('$id', 'caseSensitive');
+        $document->setAttribute('$internalId', '200');
         static::getDatabase()->createDocument($document->getCollection(), $document);
 
         $document->setAttribute('$id', 'CaseSensitive');