From c6525691994a8ff17c3ec177fabbab13327f52b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sun, 15 Oct 2023 17:54:08 +0200 Subject: [PATCH 01/42] Add proxy texts --- composer.json | 11 +- composer.lock | 70 ++- docker-compose.yml | 20 + phpunit.xml | 2 +- src/Database/Adapter.php | 4 +- src/Database/Adapter/Proxy.php | 555 ++++++++++++++++++++ src/Database/Adapter/ProxyMariaDB.php | 471 +++++++++++++++++ src/Database/Exception.php | 8 + src/Database/Exception/Conflict.php | 4 +- src/Database/Exception/Restricted.php | 4 +- src/Database/Exception/Timeout.php | 4 +- src/Database/Query.php | 18 +- tests/Database/Adapter/ProxyMariaDBTest.php | 111 ++++ 13 files changed, 1264 insertions(+), 18 deletions(-) create mode 100644 src/Database/Adapter/Proxy.php create mode 100644 src/Database/Adapter/ProxyMariaDB.php create mode 100644 tests/Database/Adapter/ProxyMariaDBTest.php diff --git a/composer.json b/composer.json index ae846a3ca..efe2564fb 100755 --- a/composer.json +++ b/composer.json @@ -35,7 +35,8 @@ "php": ">=8.0", "utopia-php/framework": "0.*.*", "utopia-php/cache": "0.8.*", - "utopia-php/mongo": "0.3.*" + "utopia-php/mongo": "0.3.*", + "utopia-php/fetch": "dev-main as 0.1.99" }, "require-dev": { "fakerphp/faker": "^1.14", @@ -52,5 +53,11 @@ "ext-redis": "Needed to support Redis Cache Adapter", "ext-pdo": "Needed to support MariaDB, MySQL or SQLite Database Adapter", "mongodb/mongodb": "Needed to support MongoDB Database Adapter" - } + }, + "repositories": [ + { + "type": "git", + "url": "https://github.com/utopia-php/fetch.git" + } + ] } diff --git a/composer.lock b/composer.lock index 3ab0753d4..cf4375df1 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6811aec6a1675620a039ab1a7c240a17", + "content-hash": "380105c8b3dd19b33bfc29e7207b077b", "packages": [ { "name": "jean85/pretty-package-versions", @@ -266,6 +266,49 @@ }, "time": "2022-10-16T16:48:09+00:00" }, + { + "name": "utopia-php/fetch", + "version": "dev-main", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/fetch.git", + "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5" + }, + "require": { + "php": ">=8.0" + }, + "require-dev": { + "laravel/pint": "^1.5.0", + "phpstan/phpstan": "^1.10", + "phpunit/phpunit": "^9.5" + }, + "default-branch": true, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Fetch\\": "src/" + } + }, + "scripts": { + "lint": [ + "./vendor/bin/pint --test --config pint.json" + ], + "format": [ + "./vendor/bin/pint --config pint.json" + ], + "check": [ + "./vendor/bin/phpstan analyse -c phpstan.neon" + ], + "test": [ + "./vendor/bin/phpunit --configuration phpunit.xml --debug" + ] + }, + "license": [ + "MIT" + ], + "description": "A simple library that provides an interface for making HTTP Requests.", + "time": "2023-10-10T11:58:32+00:00" + }, { "name": "utopia-php/framework", "version": "0.31.0", @@ -839,16 +882,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.10.35", + "version": "1.10.38", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "e730e5facb75ffe09dfb229795e8c01a459f26c3" + "reference": "5302bb402c57f00fb3c2c015bac86e0827e4b691" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/e730e5facb75ffe09dfb229795e8c01a459f26c3", - "reference": "e730e5facb75ffe09dfb229795e8c01a459f26c3", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/5302bb402c57f00fb3c2c015bac86e0827e4b691", + "reference": "5302bb402c57f00fb3c2c015bac86e0827e4b691", "shasum": "" }, "require": { @@ -897,7 +940,7 @@ "type": "tidelift" } ], - "time": "2023-09-19T15:27:56+00:00" + "time": "2023-10-06T14:19:14+00:00" }, { "name": "phpunit/php-code-coverage", @@ -2597,9 +2640,18 @@ "time": "2022-10-09T10:19:07+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/fetch", + "version": "dev-main", + "alias": "0.1.99", + "alias_normalized": "0.1.99.0" + } + ], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "utopia-php/fetch": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -2608,5 +2660,5 @@ "php": ">=8.0" }, "platform-dev": [], - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.3.0" } diff --git a/docker-compose.yml b/docker-compose.yml index 296cd6095..5c93d041e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,5 +82,25 @@ services: networks: - database + proxy: + image: utopia-php/database-proxy:0.1.0 + container_name: utopia-proxy + networks: + - database + environment: + - UTOPIA_DATABASE_PROXY_SECRET=test-secret + - UTOPIA_DATABASE_PROXY_SECRET_CONNECTIONS=default=mariadb://root:password@proxy-mariadb:3306/appwrite + + proxy-mariadb: + image: mariadb:10.7 + container_name: utopia-proxy-mariadb + networks: + - database + environment: + - MYSQL_ROOT_PASSWORD=password + - MARIADB_DATABASE=appwrite + ports: + - "8770:3306" + networks: database: diff --git a/phpunit.xml b/phpunit.xml index 31b947dd6..3833748e0 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false"> + stopOnFailure="true"> ./tests/ diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index decfd7db4..6849cd4b2 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -191,11 +191,11 @@ abstract public function createCollection(string $name, array $attributes = [], /** * Delete Collection * - * @param string $name + * @param string $id * * @return bool */ - abstract public function deleteCollection(string $name): bool; + abstract public function deleteCollection(string $id): bool; /** * Create Attribute diff --git a/src/Database/Adapter/Proxy.php b/src/Database/Adapter/Proxy.php new file mode 100644 index 000000000..19c9fab41 --- /dev/null +++ b/src/Database/Adapter/Proxy.php @@ -0,0 +1,555 @@ +endpoint = $endpoint; + $this->secret = $secret; + $this->database = $database; + } + + private function query(string $action, mixed $body = []): mixed + { + $response = Client::fetch( + url: $this->endpoint . '/queries/' . $action, + method: 'POST', + headers: [ + 'x-utopia-secret' => $this->secret, + 'x-utopia-database' => $this->database, + 'x-utopia-namespace' => $this->getNamespace(), + 'x-utopia-default-database' => $this->defaultDatabase, + 'x-utopia-timeout' => self::$timeout, + 'content-type' => 'application/json' + ], + body: $body + ); + + if ($response->getStatusCode() >= 400) { + if(empty($response->getBody())) { + throw new Exception('Internal ' . $response->getBody() . ' HTTP error in database proxy.'); + } + + $error = \json_decode($response->getBody(), true); + + /** + * @var \Utopia\Database\Exception $exception + */ + try { + $exception = new $error['type']($error['message'], $error['code']); + $exception->setFile($error['file']); + $exception->setLine($error['line']); + // TODO: If possible in PHP, set trace too for better error reporting + // $exception->setTrace($error['trace']); + } catch(Throwable $err) { + // Cannot find exception type + throw new Exception($error['message'], $error['code']); + } + + throw $exception; + } + + $body = \json_decode($response->getBody(), true); + return $body['output'] ?? ''; + } + + /** + * Ping Database + * + * @return bool + */ + public function ping(): bool + { + return $this->query('ping'); + } + + /** + * Create Database + * + * @param string $name + * + * @return bool + */ + public function create(string $name): bool + { + return $this->query('create', ['name' => $name]); + } + + /** + * Check if database exists + * Optionally check if collection exists in database + * + * @param string $database database name + * @param string $collection (optional) collection name + * + * @return bool + */ + public function exists(string $database, ?string $collection): bool + { + return $this->query('exists', ['database' => $database, 'collection' => $collection]); + } + + /** + * Delete Database + * + * @param string $name + * + * @return bool + */ + public function delete(string $name): bool + { + return $this->query('delete', ['name' => $name]); + } + + /** + * Create Collection + * + * @param string $name + * @param array $attributes (optional) + * @param array $indexes (optional) + * @return bool + */ + public function createCollection(string $name, array $attributes = [], array $indexes = []): bool + { + return $this->query('createCollection', ['name' => $name, 'attributes' => $attributes, 'indexes' => $indexes]); + } + + /** + * Delete Collection + * + * @param string $id + * + * @return bool + */ + public function deleteCollection(string $id): bool + { + return $this->query('deleteCollection', ['id' => $id]); + } + + /** + * Create Attribute + * + * @param string $collection + * @param string $id + * @param string $type + * @param int $size + * @param bool $signed + * @param bool $array + * @return bool + */ + public function createAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false): bool + { + return $this->query('createAttribute', [ + 'collection' => $collection, + 'id' => $id, + 'type' => $type, + 'size' => $size, + 'signed' => $signed, + 'array' => $array, + ]); + } + + /** + * Update Attribute + * + * @param string $collection + * @param string $id + * @param string $type + * @param int $size + * @param bool $signed + * @param bool $array + * + * @return bool + */ + public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false): bool + { + return $this->query('updateAttribute', [ + 'collection' => $collection, + 'id' => $id, + 'type' => $type, + 'size' => $size, + 'signed' => $signed, + 'array' => $array, + ]); + } + + /** + * Delete Attribute + * + * @param string $collection + * @param string $id + * + * @return bool + */ + public function deleteAttribute(string $collection, string $id): bool + { + return $this->query('deleteAttribute', ['collection' => $collection, 'id' => $id]); + } + + /** + * Rename Attribute + * + * @param string $collection + * @param string $old + * @param string $new + * @return bool + */ + public function renameAttribute(string $collection, string $old, string $new): bool + { + return $this->query('renameAttribute', ['collection' => $collection, 'old' => $old, 'new' => $new]); + } + + /** + * @param string $collection + * @param string $relatedCollection + * @param string $type + * @param bool $twoWay + * @param string $id + * @param string $twoWayKey + * @return bool + */ + public function createRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay = false, string $id = '', string $twoWayKey = ''): bool + { + return $this->query('createRelationship', [ + 'collection' => $collection, + 'relatedCollection' => $relatedCollection, + 'type' => $type, + 'twoWay' => $twoWay, + 'id' => $id, + 'twoWayKey' => $twoWayKey + ]); + } + + /** + * Update Relationship + * + * @param string $collection + * @param string $relatedCollection + * @param string $type + * @param bool $twoWay + * @param string $key + * @param string $twoWayKey + * @param string|null $newKey + * @param string|null $newTwoWayKey + * @return bool + */ + public function updateRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay, string $key, string $twoWayKey, ?string $newKey = null, ?string $newTwoWayKey = null): bool + { + return $this->query('updateRelationship', [ + 'collection' => $collection, + 'relatedCollection' => $relatedCollection, + 'type' => $type, + 'twoWay' => $twoWay, + 'key' => $key, + 'twoWayKey' => $twoWayKey, + 'newKey' => $newKey, + 'newTwoWayKey' => $newTwoWayKey + ]); + } + + /** + * Delete Relationship + * + * @param string $collection + * @param string $relatedCollection + * @param string $type + * @param bool $twoWay + * @param string $key + * @param string $twoWayKey + * @param string $side + * @return bool + */ + public function deleteRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay, string $key, string $twoWayKey, string $side): bool + { + return $this->query('deleteRelationship', [ + 'collection' => $collection, + 'relatedCollection' => $relatedCollection, + 'type' => $type, + 'twoWay' => $twoWay, + 'key' => $key, + 'twoWayKey' => $twoWayKey, + 'side' => $side, + ]); + } + + /** + * Rename Index + * + * @param string $collection + * @param string $old + * @param string $new + * @return bool + */ + public function renameIndex(string $collection, string $old, string $new): bool + { + return $this->query('renameIndex', ['collection' => $collection, 'old' => $old, 'new' => $new]); + } + + /** + * Create Index + * + * @param string $collection + * @param string $id + * @param string $type + * @param array $attributes + * @param array $lengths + * @param array $orders + * + * @return bool + */ + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders): bool + { + return $this->query('createIndex', ['collection' => $collection, 'id' => $id, 'type' => $type, 'attributes' => $attributes, 'lengths' => $lengths, 'orders' => $orders]); + } + + /** + * Delete Index + * + * @param string $collection + * @param string $id + * + * @return bool + */ + public function deleteIndex(string $collection, string $id): bool + { + return $this->query('deleteIndex', ['collection' => $collection, 'id' => $id]); + } + + /** + * Get Document + * + * @param string $collection + * @param string $id + * @param array $queries + * @return Document + */ + public function getDocument(string $collection, string $id, array $queries = []): Document + { + return new Document($this->query('getDocument', ['collection' => $collection, 'id' => $id, 'queries' => $queries])); + } + + /** + * Create Document + * + * @param string $collection + * @param Document $document + * + * @return Document + */ + public function createDocument(string $collection, Document $document): Document + { + return new Document($this->query('createDocument', ['collection' => $collection, 'document' => $document])); + } + + /** + * Update Document + * + * @param string $collection + * @param Document $document + * + * @return Document + */ + public function updateDocument(string $collection, Document $document): Document + { + return new Document($this->query('updateDocument', ['collection' => $collection, 'document' => $document])); + } + + /** + * Delete Document + * + * @param string $collection + * @param string $id + * + * @return bool + */ + public function deleteDocument(string $collection, string $id): bool + { + return $this->query('deleteDocument', ['collection' => $collection, 'id' => $id]); + } + + /** + * Find Documents + * + * Find data sets using chosen queries + * + * @param string $collection + * @param array $queries + * @param int|null $limit + * @param int|null $offset + * @param array $orderAttributes + * @param array $orderTypes + * @param array $cursor + * @param string $cursorDirection + * @param int|null $timeout + * + * @return array + */ + public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, ?int $timeout = null): array + { + $results = $this->query('find', [ + 'collection' => $collection, + 'queries' => $queries, + 'limit' => $limit, + 'offset' => $offset, + 'orderAttributes' => $orderAttributes, + 'orderTypes' => $orderTypes, + 'cursor' => $cursor, + 'cursorDirection' => $cursorDirection, + 'timeout' => $timeout + ]); + + foreach ($results as $index => $document) { + $results[$index] = new Document($results[$index]); + } + + return $results; + } + + /** + * Sum an attribute + * + * @param string $collection + * @param string $attribute + * @param array $queries + * @param int|null $max + * + * @return int|float + */ + public function sum(string $collection, string $attribute, array $queries = [], ?int $max = null, ?int $timeout = null): float|int + { + return $this->query('sum', [ + 'collection' => $collection, + 'attribute' => $attribute, + 'queries' => $queries, + 'max' => $max, + 'timeout' => $timeout + ]); + } + + /** + * Count Documents + * + * @param string $collection + * @param array $queries + * @param int|null $max + * + * @return int + */ + public function count(string $collection, array $queries = [], ?int $max = null, ?int $timeout = null): int + { + return $this->query('count', [ + 'collection' => $collection, + 'queries' => $queries, + 'max' => $max, + 'timeout' => $timeout + ]); + } + + /** + * Get Collection Size + * + * @param string $collection + * @return int + * @throws DatabaseException + */ + public function getSizeOfCollection(string $collection): int + { + return $this->query('getSizeOfCollection', ['collection' => $collection]); + } + + /** + * Get current attribute count from collection document + * + * @param Document $collection + * @return int + */ + public function getCountOfAttributes(Document $collection): int + { + return $this->query('getCountOfAttributes', ['collection' => $collection]); + } + + /** + * Get current index count from collection document + * + * @param Document $collection + * @return int + */ + public function getCountOfIndexes(Document $collection): int + { + return $this->query('getCountOfIndexes', ['collection' => $collection]); + } + + /** + * Estimate maximum number of bytes required to store a document in $collection. + * Byte requirement varies based on column type and size. + * Needed to satisfy MariaDB/MySQL row width limit. + * Return 0 when no restrictions apply to row width + * + * @param Document $collection + * @return int + */ + public function getAttributeWidth(Document $collection): int + { + return $this->query('getAttributeWidth', ['collection' => $collection]); + } + + /** + * Get an attribute projection given a list of selected attributes + * + * @param array $selections + * @param string $prefix + * @return mixed + */ + protected function getAttributeProjection(array $selections, string $prefix = ''): mixed + { + // Not nessessary for this adapter + return []; + } + + /** + * Increase and Decrease Attribute Value + * + * @param string $collection + * @param string $id + * @param string $attribute + * @param int|float $value + * @param int|float|null $min + * @param int|float|null $max + * @return bool + * @throws Exception + */ + public function increaseDocumentAttribute(string $collection, string $id, string $attribute, int|float $value, int|float|null $min = null, int|float|null $max = null): bool + { + return $this->query('increaseDocumentAttribute', [ + 'collection' => $collection, + 'id' => $id, + 'attribute' => $attribute, + 'value' => $value, + 'min' => $min, + 'max' => $max + ]); + } +} diff --git a/src/Database/Adapter/ProxyMariaDB.php b/src/Database/Adapter/ProxyMariaDB.php new file mode 100644 index 000000000..5203a9292 --- /dev/null +++ b/src/Database/Adapter/ProxyMariaDB.php @@ -0,0 +1,471 @@ + + */ + public function list(): array + { + return []; + } + + /** + * Is fulltext Wildcard index supported? + * + * @return bool + */ + public function getSupportForFulltextWildcardIndex(): bool + { + return true; + } + + /** + * Are timeouts supported? + * + * @return bool + */ + public function getSupportForTimeouts(): bool + { + return true; + } + + /** + * Does the adapter handle casting? + * + * @return bool + */ + public function getSupportForCasting(): bool + { + return false; + } + + /** + * Does the adapter handle Query Array Contains? + * + * @return bool + */ + public function getSupportForQueryContains(): bool + { + return false; + } + + public function getSupportForRelationships(): bool + { + return true; + } + + /** + * Get list of keywords that cannot be used + * Refference: https://mariadb.com/kb/en/reserved-words/ + * + * @return array + */ + public function getKeywords(): array + { + return [ + 'ACCESSIBLE', + 'ADD', + 'ALL', + 'ALTER', + 'ANALYZE', + 'AND', + 'AS', + 'ASC', + 'ASENSITIVE', + 'BEFORE', + 'BETWEEN', + 'BIGINT', + 'BINARY', + 'BLOB', + 'BOTH', + 'BY', + 'CALL', + 'CASCADE', + 'CASE', + 'CHANGE', + 'CHAR', + 'CHARACTER', + 'CHECK', + 'COLLATE', + 'COLUMN', + 'CONDITION', + 'CONSTRAINT', + 'CONTINUE', + 'CONVERT', + 'CREATE', + 'CROSS', + 'CURRENT_DATE', + 'CURRENT_ROLE', + 'CURRENT_TIME', + 'CURRENT_TIMESTAMP', + 'CURRENT_USER', + 'CURSOR', + 'DATABASE', + 'DATABASES', + 'DAY_HOUR', + 'DAY_MICROSECOND', + 'DAY_MINUTE', + 'DAY_SECOND', + 'DEC', + 'DECIMAL', + 'DECLARE', + 'DEFAULT', + 'DELAYED', + 'DELETE', + 'DELETE_DOMAIN_ID', + 'DESC', + 'DESCRIBE', + 'DETERMINISTIC', + 'DISTINCT', + 'DISTINCTROW', + 'DIV', + 'DO_DOMAIN_IDS', + 'DOUBLE', + 'DROP', + 'DUAL', + 'EACH', + 'ELSE', + 'ELSEIF', + 'ENCLOSED', + 'ESCAPED', + 'EXCEPT', + 'EXISTS', + 'EXIT', + 'EXPLAIN', + 'FALSE', + 'FETCH', + 'FLOAT', + 'FLOAT4', + 'FLOAT8', + 'FOR', + 'FORCE', + 'FOREIGN', + 'FROM', + 'FULLTEXT', + 'GENERAL', + 'GRANT', + 'GROUP', + 'HAVING', + 'HIGH_PRIORITY', + 'HOUR_MICROSECOND', + 'HOUR_MINUTE', + 'HOUR_SECOND', + 'IF', + 'IGNORE', + 'IGNORE_DOMAIN_IDS', + 'IGNORE_SERVER_IDS', + 'IN', + 'INDEX', + 'INFILE', + 'INNER', + 'INOUT', + 'INSENSITIVE', + 'INSERT', + 'INT', + 'INT1', + 'INT2', + 'INT3', + 'INT4', + 'INT8', + 'INTEGER', + 'INTERSECT', + 'INTERVAL', + 'INTO', + 'IS', + 'ITERATE', + 'JOIN', + 'KEY', + 'KEYS', + 'KILL', + 'LEADING', + 'LEAVE', + 'LEFT', + 'LIKE', + 'LIMIT', + 'LINEAR', + 'LINES', + 'LOAD', + 'LOCALTIME', + 'LOCALTIMESTAMP', + 'LOCK', + 'LONG', + 'LONGBLOB', + 'LONGTEXT', + 'LOOP', + 'LOW_PRIORITY', + 'MASTER_HEARTBEAT_PERIOD', + 'MASTER_SSL_VERIFY_SERVER_CERT', + 'MATCH', + 'MAXVALUE', + 'MEDIUMBLOB', + 'MEDIUMINT', + 'MEDIUMTEXT', + 'MIDDLEINT', + 'MINUTE_MICROSECOND', + 'MINUTE_SECOND', + 'MOD', + 'MODIFIES', + 'NATURAL', + 'NOT', + 'NO_WRITE_TO_BINLOG', + 'NULL', + 'NUMERIC', + 'OFFSET', + 'ON', + 'OPTIMIZE', + 'OPTION', + 'OPTIONALLY', + 'OR', + 'ORDER', + 'OUT', + 'OUTER', + 'OUTFILE', + 'OVER', + 'PAGE_CHECKSUM', + 'PARSE_VCOL_EXPR', + 'PARTITION', + 'POSITION', + 'PRECISION', + 'PRIMARY', + 'PROCEDURE', + 'PURGE', + 'RANGE', + 'READ', + 'READS', + 'READ_WRITE', + 'REAL', + 'RECURSIVE', + 'REF_SYSTEM_ID', + 'REFERENCES', + 'REGEXP', + 'RELEASE', + 'RENAME', + 'REPEAT', + 'REPLACE', + 'REQUIRE', + 'RESIGNAL', + 'RESTRICT', + 'RETURN', + 'RETURNING', + 'REVOKE', + 'RIGHT', + 'RLIKE', + 'ROWS', + 'SCHEMA', + 'SCHEMAS', + 'SECOND_MICROSECOND', + 'SELECT', + 'SENSITIVE', + 'SEPARATOR', + 'SET', + 'SHOW', + 'SIGNAL', + 'SLOW', + 'SMALLINT', + 'SPATIAL', + 'SPECIFIC', + 'SQL', + 'SQLEXCEPTION', + 'SQLSTATE', + 'SQLWARNING', + 'SQL_BIG_RESULT', + 'SQL_CALC_FOUND_ROWS', + 'SQL_SMALL_RESULT', + 'SSL', + 'STARTING', + 'STATS_AUTO_RECALC', + 'STATS_PERSISTENT', + 'STATS_SAMPLE_PAGES', + 'STRAIGHT_JOIN', + 'TABLE', + 'TERMINATED', + 'THEN', + 'TINYBLOB', + 'TINYINT', + 'TINYTEXT', + 'TO', + 'TRAILING', + 'TRIGGER', + 'TRUE', + 'UNDO', + 'UNION', + 'UNIQUE', + 'UNLOCK', + 'UNSIGNED', + 'UPDATE', + 'USAGE', + 'USE', + 'USING', + 'UTC_DATE', + 'UTC_TIME', + 'UTC_TIMESTAMP', + 'VALUES', + 'VARBINARY', + 'VARCHAR', + 'VARCHARACTER', + 'VARYING', + 'WHEN', + 'WHERE', + 'WHILE', + 'WINDOW', + 'WITH', + 'WRITE', + 'XOR', + 'YEAR_MONTH', + 'ZEROFILL', + 'ACTION', + 'BIT', + 'DATE', + 'ENUM', + 'NO', + 'TEXT', + 'TIME', + 'TIMESTAMP', + 'BODY', + 'ELSIF', + 'GOTO', + 'HISTORY', + 'MINUS', + 'OTHERS', + 'PACKAGE', + 'PERIOD', + 'RAISE', + 'ROWNUM', + 'ROWTYPE', + 'SYSDATE', + 'SYSTEM', + 'SYSTEM_TIME', + 'VERSIONING', + 'WITHOUT' + ]; + } +} diff --git a/src/Database/Exception.php b/src/Database/Exception.php index 663abce47..b2b3954ad 100644 --- a/src/Database/Exception.php +++ b/src/Database/Exception.php @@ -4,4 +4,12 @@ class Exception extends \Exception { + public function setFile(string $file): void + { + $this->file = $file; + } + public function setLine(int $line): void + { + $this->line = $line; + } } diff --git a/src/Database/Exception/Conflict.php b/src/Database/Exception/Conflict.php index c02dd138a..8803bf902 100644 --- a/src/Database/Exception/Conflict.php +++ b/src/Database/Exception/Conflict.php @@ -2,6 +2,8 @@ namespace Utopia\Database\Exception; -class Conflict extends \Exception +use Utopia\Database\Exception; + +class Conflict extends Exception { } diff --git a/src/Database/Exception/Restricted.php b/src/Database/Exception/Restricted.php index 0905161df..1ef9fefd7 100644 --- a/src/Database/Exception/Restricted.php +++ b/src/Database/Exception/Restricted.php @@ -2,6 +2,8 @@ namespace Utopia\Database\Exception; -class Restricted extends \Exception +use Utopia\Database\Exception; + +class Restricted extends Exception { } diff --git a/src/Database/Exception/Timeout.php b/src/Database/Exception/Timeout.php index 4768e835f..613e74e55 100644 --- a/src/Database/Exception/Timeout.php +++ b/src/Database/Exception/Timeout.php @@ -2,6 +2,8 @@ namespace Utopia\Database\Exception; -class Timeout extends \Exception +use Utopia\Database\Exception; + +class Timeout extends Exception { } diff --git a/src/Database/Query.php b/src/Database/Query.php index f2ed42416..3547e5d8e 100644 --- a/src/Database/Query.php +++ b/src/Database/Query.php @@ -2,9 +2,10 @@ namespace Utopia\Database; +use JsonSerializable; use Utopia\Database\Exception\Query as QueryException; -class Query +class Query implements JsonSerializable { // Filter methods public const TYPE_EQUAL = 'equal'; @@ -828,4 +829,19 @@ public static function parseQueries(array $queries): array return $parsed; } + + /** + * PHP native method override + * + * @return array + * @throws Exception + */ + public function jsonSerialize(): array + { + return [ + 'method' => $this->method, + 'attribute' => $this->attribute, + 'values' => $this->values, + ]; + } } diff --git a/tests/Database/Adapter/ProxyMariaDBTest.php b/tests/Database/Adapter/ProxyMariaDBTest.php new file mode 100644 index 000000000..ba350805d --- /dev/null +++ b/tests/Database/Adapter/ProxyMariaDBTest.php @@ -0,0 +1,111 @@ +setDefaultDatabase('utopiaTests'); + $database->setNamespace('myapp_' . uniqid()); + + if ($database->exists('utopiaTests')) { + $database->delete('utopiaTests'); + } + + $database->create(); + + return self::$database = $database; + } + + public function testCreateExistsDelete(): void + { + $schemaSupport = $this->getDatabase()->getAdapter()->getSupportForSchemas(); + if (!$schemaSupport) { + $this->assertEquals(true, static::getDatabase()->setDefaultDatabase($this->testDatabase)); + $this->assertEquals(true, static::getDatabase()->create()); + return; + } + + if (!static::getDatabase()->exists($this->testDatabase)) { + $this->assertEquals(true, static::getDatabase()->create()); + } + $this->assertEquals(true, static::getDatabase()->exists($this->testDatabase)); + $this->assertEquals(true, static::getDatabase()->delete($this->testDatabase)); + $this->assertEquals(false, static::getDatabase()->exists($this->testDatabase)); + $this->assertEquals(true, static::getDatabase()->setDefaultDatabase($this->testDatabase)); + $this->assertEquals(true, static::getDatabase()->create()); + } + + /** + * @depends testCreateExistsDelete + */ + public function testCreateListExistsDeleteCollection(): void + { + $this->assertInstanceOf('Utopia\Database\Document', static::getDatabase()->createCollection('actors')); + + static::getDatabase()->createCollection('actors2'); + static::getDatabase()->createCollection('actors3'); + static::getDatabase()->createCollection('actors4'); + + sleep(3); + + \var_dump(static::getDatabase()->listCollections()); + \var_dump(static::getDatabase()->listCollections()); + \var_dump(static::getDatabase()->listCollections()); + \var_dump(static::getDatabase()->listCollections()); + $this->assertCount(2, static::getDatabase()->listCollections()); + $this->assertEquals(true, static::getDatabase()->exists($this->testDatabase, 'actors')); + + // Collection names should not be unique + $this->assertInstanceOf('Utopia\Database\Document', static::getDatabase()->createCollection('actors2')); + $this->assertCount(3, static::getDatabase()->listCollections()); + $this->assertEquals(true, static::getDatabase()->exists($this->testDatabase, 'actors2')); + $collection = static::getDatabase()->getCollection('actors2'); + $collection->setAttribute('name', 'actors'); // change name to one that exists + $this->assertInstanceOf('Utopia\Database\Document', static::getDatabase()->updateDocument($collection->getCollection(), $collection->getId(), $collection)); + $this->assertEquals(true, static::getDatabase()->deleteCollection('actors2')); // Delete collection when finished + $this->assertCount(2, static::getDatabase()->listCollections()); + + $this->assertEquals(false, static::getDatabase()->getCollection('actors')->isEmpty()); + $this->assertEquals(true, static::getDatabase()->deleteCollection('actors')); + $this->assertEquals(true, static::getDatabase()->getCollection('actors')->isEmpty()); + $this->assertEquals(false, static::getDatabase()->exists($this->testDatabase, 'actors')); + } +} From 1f1d191ff1f3f5ccfad0b44c7d00539e1123207c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 16 Oct 2023 11:13:47 +0200 Subject: [PATCH 02/42] Proxy auth state --- src/Database/Adapter/Proxy.php | 5 ++ tests/Database/Adapter/ProxyMariaDBTest.php | 64 +-------------------- 2 files changed, 6 insertions(+), 63 deletions(-) diff --git a/src/Database/Adapter/Proxy.php b/src/Database/Adapter/Proxy.php index 19c9fab41..ba665db99 100644 --- a/src/Database/Adapter/Proxy.php +++ b/src/Database/Adapter/Proxy.php @@ -10,6 +10,7 @@ use Utopia\Fetch\Client; use Utopia\Database\Query; use Utopia\Database\Exception as DatabaseException; +use Utopia\Database\Validator\Authorization; abstract class Proxy extends Adapter { @@ -34,6 +35,7 @@ public function __construct(string $endpoint, string $secret, string $database) private function query(string $action, mixed $body = []): mixed { + $roles = \implode(',', Authorization::getRoles()); $response = Client::fetch( url: $this->endpoint . '/queries/' . $action, method: 'POST', @@ -42,6 +44,9 @@ private function query(string $action, mixed $body = []): mixed 'x-utopia-database' => $this->database, 'x-utopia-namespace' => $this->getNamespace(), 'x-utopia-default-database' => $this->defaultDatabase, + 'x-utopia-auth-roles' => $roles, + 'x-utopia-auth-status' => Authorization::$status, + 'x-utopia-auth-status-default' => Authorization::$statusDefault, 'x-utopia-timeout' => self::$timeout, 'content-type' => 'application/json' ], diff --git a/tests/Database/Adapter/ProxyMariaDBTest.php b/tests/Database/Adapter/ProxyMariaDBTest.php index ba350805d..4e4d3e5ed 100644 --- a/tests/Database/Adapter/ProxyMariaDBTest.php +++ b/tests/Database/Adapter/ProxyMariaDBTest.php @@ -7,17 +7,10 @@ use Utopia\Database\Database; use Utopia\Cache\Cache; use Utopia\Database\Adapter\ProxyMariaDB; -use Utopia\Database\Document; -use Utopia\Database\Exception\Duplicate; -use Utopia\Database\Helpers\ID; -use Utopia\Database\Helpers\Permission; -use Utopia\Database\Helpers\Role; use Utopia\Tests\Base; -class ProxyMariaDBTest extends TestCase +class ProxyMariaDBTest extends Base { - protected string $testDatabase = 'utopiaTests'; - public static ?Database $database = null; // TODO@kodumbeats hacky way to identify adapters for tests @@ -53,59 +46,4 @@ public static function getDatabase(): Database return self::$database = $database; } - - public function testCreateExistsDelete(): void - { - $schemaSupport = $this->getDatabase()->getAdapter()->getSupportForSchemas(); - if (!$schemaSupport) { - $this->assertEquals(true, static::getDatabase()->setDefaultDatabase($this->testDatabase)); - $this->assertEquals(true, static::getDatabase()->create()); - return; - } - - if (!static::getDatabase()->exists($this->testDatabase)) { - $this->assertEquals(true, static::getDatabase()->create()); - } - $this->assertEquals(true, static::getDatabase()->exists($this->testDatabase)); - $this->assertEquals(true, static::getDatabase()->delete($this->testDatabase)); - $this->assertEquals(false, static::getDatabase()->exists($this->testDatabase)); - $this->assertEquals(true, static::getDatabase()->setDefaultDatabase($this->testDatabase)); - $this->assertEquals(true, static::getDatabase()->create()); - } - - /** - * @depends testCreateExistsDelete - */ - public function testCreateListExistsDeleteCollection(): void - { - $this->assertInstanceOf('Utopia\Database\Document', static::getDatabase()->createCollection('actors')); - - static::getDatabase()->createCollection('actors2'); - static::getDatabase()->createCollection('actors3'); - static::getDatabase()->createCollection('actors4'); - - sleep(3); - - \var_dump(static::getDatabase()->listCollections()); - \var_dump(static::getDatabase()->listCollections()); - \var_dump(static::getDatabase()->listCollections()); - \var_dump(static::getDatabase()->listCollections()); - $this->assertCount(2, static::getDatabase()->listCollections()); - $this->assertEquals(true, static::getDatabase()->exists($this->testDatabase, 'actors')); - - // Collection names should not be unique - $this->assertInstanceOf('Utopia\Database\Document', static::getDatabase()->createCollection('actors2')); - $this->assertCount(3, static::getDatabase()->listCollections()); - $this->assertEquals(true, static::getDatabase()->exists($this->testDatabase, 'actors2')); - $collection = static::getDatabase()->getCollection('actors2'); - $collection->setAttribute('name', 'actors'); // change name to one that exists - $this->assertInstanceOf('Utopia\Database\Document', static::getDatabase()->updateDocument($collection->getCollection(), $collection->getId(), $collection)); - $this->assertEquals(true, static::getDatabase()->deleteCollection('actors2')); // Delete collection when finished - $this->assertCount(2, static::getDatabase()->listCollections()); - - $this->assertEquals(false, static::getDatabase()->getCollection('actors')->isEmpty()); - $this->assertEquals(true, static::getDatabase()->deleteCollection('actors')); - $this->assertEquals(true, static::getDatabase()->getCollection('actors')->isEmpty()); - $this->assertEquals(false, static::getDatabase()->exists($this->testDatabase, 'actors')); - } } From 1959611661db599b04c5534c57a86595952b82de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 16 Oct 2023 15:17:12 +0200 Subject: [PATCH 03/42] Finish db proxy adapter --- phpunit.xml | 2 +- src/Database/Adapter/Proxy.php | 4 ++-- tests/Database/Base.php | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/phpunit.xml b/phpunit.xml index 3833748e0..31b947dd6 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="true"> + stopOnFailure="false"> ./tests/ diff --git a/src/Database/Adapter/Proxy.php b/src/Database/Adapter/Proxy.php index ba665db99..986f5e0f9 100644 --- a/src/Database/Adapter/Proxy.php +++ b/src/Database/Adapter/Proxy.php @@ -45,8 +45,8 @@ private function query(string $action, mixed $body = []): mixed 'x-utopia-namespace' => $this->getNamespace(), 'x-utopia-default-database' => $this->defaultDatabase, 'x-utopia-auth-roles' => $roles, - 'x-utopia-auth-status' => Authorization::$status, - 'x-utopia-auth-status-default' => Authorization::$statusDefault, + 'x-utopia-auth-status' => Authorization::$status ? 'true' : 'false', + 'x-utopia-auth-status-default' => Authorization::$statusDefault ? 'true' : 'false', 'x-utopia-timeout' => self::$timeout, 'content-type' => 'application/json' ], diff --git a/tests/Database/Base.php b/tests/Database/Base.php index 34609d113..98e78afec 100644 --- a/tests/Database/Base.php +++ b/tests/Database/Base.php @@ -5088,7 +5088,7 @@ public function testOneToOneTwoWayRelationship(): void ], ]); - static::getDatabase()->createDocument('country', $doc); + static::getDatabase()->createDocument('country', new Document($doc->getArrayCopy())); $country1 = static::getDatabase()->getDocument('country', 'country1'); $this->assertEquals('London', $country1->getAttribute('city')->getAttribute('name')); @@ -5116,7 +5116,7 @@ public function testOneToOneTwoWayRelationship(): void $this->assertTrue(static::getDatabase()->deleteDocument('country', 'country1')); - static::getDatabase()->createDocument('country', $doc); + static::getDatabase()->createDocument('country', new Document($doc->getArrayCopy())); $country1 = static::getDatabase()->getDocument('country', 'country1'); $this->assertEquals('London', $country1->getAttribute('city')->getAttribute('name')); From 12fc78d7d5850891b69b12f0425f0e5a1f2ff82a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 16 Oct 2023 15:20:14 +0200 Subject: [PATCH 04/42] linter fix --- src/Database/Adapter/Proxy.php | 13 +++++++------ tests/Database/Adapter/ProxyMariaDBTest.php | 1 - 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Database/Adapter/Proxy.php b/src/Database/Adapter/Proxy.php index 986f5e0f9..192c00ef4 100644 --- a/src/Database/Adapter/Proxy.php +++ b/src/Database/Adapter/Proxy.php @@ -47,24 +47,25 @@ private function query(string $action, mixed $body = []): mixed 'x-utopia-auth-roles' => $roles, 'x-utopia-auth-status' => Authorization::$status ? 'true' : 'false', 'x-utopia-auth-status-default' => Authorization::$statusDefault ? 'true' : 'false', - 'x-utopia-timeout' => self::$timeout, + 'x-utopia-timeout' => self::$timeout ? \strval(self::$timeout) : '', 'content-type' => 'application/json' ], body: $body ); if ($response->getStatusCode() >= 400) { - if(empty($response->getBody())) { - throw new Exception('Internal ' . $response->getBody() . ' HTTP error in database proxy.'); + if (empty($response->getBody())) { + throw new Exception('Internal ' . $response->getStatusCode() . ' HTTP error in database proxy'); } $error = \json_decode($response->getBody(), true); - /** - * @var \Utopia\Database\Exception $exception - */ try { $exception = new $error['type']($error['message'], $error['code']); + /** + * @var \Utopia\Database\Exception $exception + */ + $exception->setFile($error['file']); $exception->setLine($error['line']); // TODO: If possible in PHP, set trace too for better error reporting diff --git a/tests/Database/Adapter/ProxyMariaDBTest.php b/tests/Database/Adapter/ProxyMariaDBTest.php index 4e4d3e5ed..e26938087 100644 --- a/tests/Database/Adapter/ProxyMariaDBTest.php +++ b/tests/Database/Adapter/ProxyMariaDBTest.php @@ -2,7 +2,6 @@ namespace Utopia\Tests\Adapter; -use PHPUnit\Framework\TestCase; use Utopia\Cache\Adapter\None; use Utopia\Database\Database; use Utopia\Cache\Cache; From 644ca9a71dd5fbd2559517bf3a6bff8418c9c466 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 17 Oct 2023 11:44:14 +0200 Subject: [PATCH 05/42] Address TODOs --- .github/workflows/tests.yml | 8 +++++++ composer.json | 10 ++------- composer.lock | 43 +++++++++++++------------------------ docker-compose.yml | 3 ++- 4 files changed, 27 insertions(+), 37 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 970fc22de..efebd21f0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,6 +28,14 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + - name: Build Database Proxy + run: | + git clone https://github.com/utopia-php/database-proxy.git + cd database-proxy + git checkout dev + docker build -t utopia-php/database-proxy-test . + cd .. + - name: Start Databases run: | docker compose up -d diff --git a/composer.json b/composer.json index efe2564fb..734fcc9ea 100755 --- a/composer.json +++ b/composer.json @@ -36,7 +36,7 @@ "utopia-php/framework": "0.*.*", "utopia-php/cache": "0.8.*", "utopia-php/mongo": "0.3.*", - "utopia-php/fetch": "dev-main as 0.1.99" + "utopia-php/fetch": "0.1.*" }, "require-dev": { "fakerphp/faker": "^1.14", @@ -53,11 +53,5 @@ "ext-redis": "Needed to support Redis Cache Adapter", "ext-pdo": "Needed to support MariaDB, MySQL or SQLite Database Adapter", "mongodb/mongodb": "Needed to support MongoDB Database Adapter" - }, - "repositories": [ - { - "type": "git", - "url": "https://github.com/utopia-php/fetch.git" - } - ] + } } diff --git a/composer.lock b/composer.lock index cf4375df1..6de483eec 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "380105c8b3dd19b33bfc29e7207b077b", + "content-hash": "166c47b45e7e2dab4adf13ab91abd446", "packages": [ { "name": "jean85/pretty-package-versions", @@ -268,12 +268,18 @@ }, { "name": "utopia-php/fetch", - "version": "dev-main", + "version": "0.1.0", "source": { "type": "git", "url": "https://github.com/utopia-php/fetch.git", "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5" }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/2fa214b9262acd1a3583515a364da4f35929d5c5", + "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5", + "shasum": "" + }, "require": { "php": ">=8.0" }, @@ -282,31 +288,21 @@ "phpstan/phpstan": "^1.10", "phpunit/phpunit": "^9.5" }, - "default-branch": true, "type": "library", "autoload": { "psr-4": { "Utopia\\Fetch\\": "src/" } }, - "scripts": { - "lint": [ - "./vendor/bin/pint --test --config pint.json" - ], - "format": [ - "./vendor/bin/pint --config pint.json" - ], - "check": [ - "./vendor/bin/phpstan analyse -c phpstan.neon" - ], - "test": [ - "./vendor/bin/phpunit --configuration phpunit.xml --debug" - ] - }, + "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], "description": "A simple library that provides an interface for making HTTP Requests.", + "support": { + "issues": "https://github.com/utopia-php/fetch/issues", + "source": "https://github.com/utopia-php/fetch/tree/0.1.0" + }, "time": "2023-10-10T11:58:32+00:00" }, { @@ -2640,18 +2636,9 @@ "time": "2022-10-09T10:19:07+00:00" } ], - "aliases": [ - { - "package": "utopia-php/fetch", - "version": "dev-main", - "alias": "0.1.99", - "alias_normalized": "0.1.99.0" - } - ], + "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "utopia-php/fetch": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/docker-compose.yml b/docker-compose.yml index 5c93d041e..3ae491650 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,7 +83,8 @@ services: - database proxy: - image: utopia-php/database-proxy:0.1.0 + # TODO: Release and use official image + image: utopia-php/database-proxy-test # Build from https://github.com/utopia-php/database-proxy/ container_name: utopia-proxy networks: - database From 377a2d060f66efeb75c7f5880c356429a1762585 Mon Sep 17 00:00:00 2001 From: fogelito Date: Fri, 8 Dec 2023 22:58:30 +0100 Subject: [PATCH 06/42] Chnage API --- docker-compose.yml | 2 +- phpunit.xml | 2 +- src/Database/Adapter/Proxy.php | 119 ++++++++++++++------------ src/Database/Adapter/ProxyMariaDB.php | 26 ++++++ 4 files changed, 93 insertions(+), 56 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3ae491650..f153adb43 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,7 +84,7 @@ services: proxy: # TODO: Release and use official image - image: utopia-php/database-proxy-test # Build from https://github.com/utopia-php/database-proxy/ + image: database-proxy # Build from https://github.com/utopia-php/database-proxy/ container_name: utopia-proxy networks: - database diff --git a/phpunit.xml b/phpunit.xml index 31b947dd6..3833748e0 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false"> + stopOnFailure="true"> ./tests/ diff --git a/src/Database/Adapter/Proxy.php b/src/Database/Adapter/Proxy.php index 192c00ef4..489156de2 100644 --- a/src/Database/Adapter/Proxy.php +++ b/src/Database/Adapter/Proxy.php @@ -11,6 +11,7 @@ use Utopia\Database\Query; use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Validator\Authorization; +use Utopia\Fetch\FetchException; abstract class Proxy extends Adapter { @@ -33,12 +34,16 @@ public function __construct(string $endpoint, string $secret, string $database) $this->database = $database; } - private function query(string $action, mixed $body = []): mixed + /** + * @throws FetchException + * @throws DatabaseException + * @throws Exception + */ + private function query(string $method, string $path, mixed $body = []): mixed { $roles = \implode(',', Authorization::getRoles()); $response = Client::fetch( - url: $this->endpoint . '/queries/' . $action, - method: 'POST', + url: $this->endpoint . $path, headers: [ 'x-utopia-secret' => $this->secret, 'x-utopia-database' => $this->database, @@ -50,9 +55,13 @@ private function query(string $action, mixed $body = []): mixed 'x-utopia-timeout' => self::$timeout ? \strval(self::$timeout) : '', 'content-type' => 'application/json' ], + method: $method, body: $body ); + + var_dump($response); + if ($response->getStatusCode() >= 400) { if (empty($response->getBody())) { throw new Exception('Internal ' . $response->getStatusCode() . ' HTTP error in database proxy'); @@ -86,10 +95,11 @@ private function query(string $action, mixed $body = []): mixed * Ping Database * * @return bool + * @throws DatabaseException|FetchException */ public function ping(): bool { - return $this->query('ping'); + return $this->query('GET', '/ping', []); } /** @@ -98,10 +108,11 @@ public function ping(): bool * @param string $name * * @return bool + * @throws DatabaseException|FetchException */ public function create(string $name): bool { - return $this->query('create', ['name' => $name]); + return $this->query('POST', '/databases', ['database' => $name]); } /** @@ -115,7 +126,15 @@ public function create(string $name): bool */ public function exists(string $database, ?string $collection): bool { - return $this->query('exists', ['database' => $database, 'collection' => $collection]); + $path = '/databases/' . $database; + $body = []; + + if(!empty($collectione)){ + $path .= '/collections/' . $collection; + $body = ['database' => $database]; + } + + return $this->query('GET', $path, $body); } /** @@ -127,7 +146,7 @@ public function exists(string $database, ?string $collection): bool */ public function delete(string $name): bool { - return $this->query('delete', ['name' => $name]); + return $this->query('DELETE', '/databases/' . $name, []); } /** @@ -140,7 +159,11 @@ public function delete(string $name): bool */ public function createCollection(string $name, array $attributes = [], array $indexes = []): bool { - return $this->query('createCollection', ['name' => $name, 'attributes' => $attributes, 'indexes' => $indexes]); + return $this->query('POST', '/collections', [ + 'collection' => $name, + 'attributes' => $attributes, + 'indexes' => $indexes + ]); } /** @@ -152,7 +175,7 @@ public function createCollection(string $name, array $attributes = [], array $in */ public function deleteCollection(string $id): bool { - return $this->query('deleteCollection', ['id' => $id]); + return $this->query('DELETE', '/collections/' . $id, []); } /** @@ -168,9 +191,8 @@ public function deleteCollection(string $id): bool */ public function createAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false): bool { - return $this->query('createAttribute', [ - 'collection' => $collection, - 'id' => $id, + return $this->query('POST', '/collections/' . $collection . '/attributes', [ + 'attribute' => $id, 'type' => $type, 'size' => $size, 'signed' => $signed, @@ -192,9 +214,7 @@ public function createAttribute(string $collection, string $id, string $type, in */ public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false): bool { - return $this->query('updateAttribute', [ - 'collection' => $collection, - 'id' => $id, + return $this->query('PUT', '/collections/'.$collection.'/attributes/' . $id, [ 'type' => $type, 'size' => $size, 'signed' => $signed, @@ -212,7 +232,7 @@ public function updateAttribute(string $collection, string $id, string $type, in */ public function deleteAttribute(string $collection, string $id): bool { - return $this->query('deleteAttribute', ['collection' => $collection, 'id' => $id]); + return $this->query('DELETE', '/collections/' . $collection . '/attributes/' . $id, []); } /** @@ -225,7 +245,7 @@ public function deleteAttribute(string $collection, string $id): bool */ public function renameAttribute(string $collection, string $old, string $new): bool { - return $this->query('renameAttribute', ['collection' => $collection, 'old' => $old, 'new' => $new]); + return $this->query('PATCH', '/collections/' . $collection . '/attributes/' . $old . '/name', ['new' => $new]); } /** @@ -239,8 +259,7 @@ public function renameAttribute(string $collection, string $old, string $new): b */ public function createRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay = false, string $id = '', string $twoWayKey = ''): bool { - return $this->query('createRelationship', [ - 'collection' => $collection, + return $this->query('POST', '/collections/' . $collection . '/relationships', [ 'relatedCollection' => $relatedCollection, 'type' => $type, 'twoWay' => $twoWay, @@ -264,9 +283,7 @@ public function createRelationship(string $collection, string $relatedCollection */ public function updateRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay, string $key, string $twoWayKey, ?string $newKey = null, ?string $newTwoWayKey = null): bool { - return $this->query('updateRelationship', [ - 'collection' => $collection, - 'relatedCollection' => $relatedCollection, + return $this->query('PUT', '/collections/'. $collection .'/relationships/' . $relatedCollection, [ 'type' => $type, 'twoWay' => $twoWay, 'key' => $key, @@ -290,9 +307,7 @@ public function updateRelationship(string $collection, string $relatedCollection */ public function deleteRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay, string $key, string $twoWayKey, string $side): bool { - return $this->query('deleteRelationship', [ - 'collection' => $collection, - 'relatedCollection' => $relatedCollection, + return $this->query('DELETE', '/collections/'. $collection .'/relationships/' . $relatedCollection, [ 'type' => $type, 'twoWay' => $twoWay, 'key' => $key, @@ -311,7 +326,9 @@ public function deleteRelationship(string $collection, string $relatedCollection */ public function renameIndex(string $collection, string $old, string $new): bool { - return $this->query('renameIndex', ['collection' => $collection, 'old' => $old, 'new' => $new]); + return $this->query('PATCH', '/collections/'.$collection.'/indexes/'.$old.'/name',[ + 'new' => $new + ]); } /** @@ -328,7 +345,13 @@ public function renameIndex(string $collection, string $old, string $new): bool */ public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders): bool { - return $this->query('createIndex', ['collection' => $collection, 'id' => $id, 'type' => $type, 'attributes' => $attributes, 'lengths' => $lengths, 'orders' => $orders]); + return $this->query('POST', '/collections/' . $collection . '/indexes', [ + 'index' => $id, + 'type' => $type, + 'attributes' => $attributes, + 'lengths' => $lengths, + 'orders' => $orders + ]); } /** @@ -341,7 +364,7 @@ public function createIndex(string $collection, string $id, string $type, array */ public function deleteIndex(string $collection, string $id): bool { - return $this->query('deleteIndex', ['collection' => $collection, 'id' => $id]); + return $this->query('DELETE', '/collections/' . $collection . '/indexes/' . $id . '/name', []); } /** @@ -354,7 +377,7 @@ public function deleteIndex(string $collection, string $id): bool */ public function getDocument(string $collection, string $id, array $queries = []): Document { - return new Document($this->query('getDocument', ['collection' => $collection, 'id' => $id, 'queries' => $queries])); + return new Document($this->query('GET', '/collections/'.$collection.'/documents/' . $id, ['queries' => $queries])); } /** @@ -364,10 +387,11 @@ public function getDocument(string $collection, string $id, array $queries = []) * @param Document $document * * @return Document + * @throws DatabaseException|FetchException */ public function createDocument(string $collection, Document $document): Document { - return new Document($this->query('createDocument', ['collection' => $collection, 'document' => $document])); + return new Document($this->query('POST', '/collections/' . $collection . '/documents', ['document' => $document])); } /** @@ -377,10 +401,11 @@ public function createDocument(string $collection, Document $document): Document * @param Document $document * * @return Document + * @throws DatabaseException|FetchException */ public function updateDocument(string $collection, Document $document): Document { - return new Document($this->query('updateDocument', ['collection' => $collection, 'document' => $document])); + return new Document($this->query('PUT', '/collections/'. $collection .'/documents', ['document' => $document])); } /** @@ -393,7 +418,7 @@ public function updateDocument(string $collection, Document $document): Document */ public function deleteDocument(string $collection, string $id): bool { - return $this->query('deleteDocument', ['collection' => $collection, 'id' => $id]); + return $this->query('DELETE','/collections/'. $collection .'/documents/' . $id, []); } /** @@ -415,8 +440,7 @@ public function deleteDocument(string $collection, string $id): bool */ public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, ?int $timeout = null): array { - $results = $this->query('find', [ - 'collection' => $collection, + $results = $this->query('GET', '/collections/' . $collection . '/documents', [ 'queries' => $queries, 'limit' => $limit, 'offset' => $offset, @@ -446,8 +470,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, */ public function sum(string $collection, string $attribute, array $queries = [], ?int $max = null, ?int $timeout = null): float|int { - return $this->query('sum', [ - 'collection' => $collection, + return $this->query('GET', '/collections/'. $collection .'/documents-sum', [ 'attribute' => $attribute, 'queries' => $queries, 'max' => $max, @@ -466,8 +489,7 @@ public function sum(string $collection, string $attribute, array $queries = [], */ public function count(string $collection, array $queries = [], ?int $max = null, ?int $timeout = null): int { - return $this->query('count', [ - 'collection' => $collection, + return $this->query('GET', '/collections/'. $collection .'/documents-count', [ 'queries' => $queries, 'max' => $max, 'timeout' => $timeout @@ -479,22 +501,11 @@ public function count(string $collection, array $queries = [], ?int $max = null, * * @param string $collection * @return int - * @throws DatabaseException + * @throws DatabaseException|FetchException */ public function getSizeOfCollection(string $collection): int { - return $this->query('getSizeOfCollection', ['collection' => $collection]); - } - - /** - * Get current attribute count from collection document - * - * @param Document $collection - * @return int - */ - public function getCountOfAttributes(Document $collection): int - { - return $this->query('getCountOfAttributes', ['collection' => $collection]); + return $this->query('GET', '/collections/' . $collection . '/size', []); } /** @@ -505,6 +516,7 @@ public function getCountOfAttributes(Document $collection): int */ public function getCountOfIndexes(Document $collection): int { + //todo: removed... return $this->query('getCountOfIndexes', ['collection' => $collection]); } @@ -519,6 +531,7 @@ public function getCountOfIndexes(Document $collection): int */ public function getAttributeWidth(Document $collection): int { + // todo: removed... return $this->query('getAttributeWidth', ['collection' => $collection]); } @@ -549,9 +562,7 @@ protected function getAttributeProjection(array $selections, string $prefix = '' */ public function increaseDocumentAttribute(string $collection, string $id, string $attribute, int|float $value, int|float|null $min = null, int|float|null $max = null): bool { - return $this->query('increaseDocumentAttribute', [ - 'collection' => $collection, - 'id' => $id, + return $this->query('PATCH', '/collections/'. $collection .'/documents/'. $id .'/increase', [ 'attribute' => $attribute, 'value' => $value, 'min' => $min, diff --git a/src/Database/Adapter/ProxyMariaDB.php b/src/Database/Adapter/ProxyMariaDB.php index 5203a9292..929b19f78 100644 --- a/src/Database/Adapter/ProxyMariaDB.php +++ b/src/Database/Adapter/ProxyMariaDB.php @@ -183,6 +183,32 @@ public function getSupportForRelationships(): bool return true; } + /** + * Get current attribute count from collection document + * + * @param Document $collection + * @return int + */ + public function getCountOfAttributes(Document $collection): int + { + $attributes = \count($collection->getAttribute('attributes') ?? []); + + // +1 ==> virtual columns count as total, so add as buffer + return $attributes + static::getCountOfDefaultAttributes() + 1; + } + + /** + * Get current index count from collection document + * + * @param Document $collection + * @return int + */ + public function getCountOfIndexes(Document $collection): int + { + $indexes = \count($collection->getAttribute('indexes') ?? []); + return $indexes + static::getCountOfDefaultIndexes(); + } + /** * Get list of keywords that cannot be used * Refference: https://mariadb.com/kb/en/reserved-words/ From dd8a9a2915de531822123891597290f5372b28fa Mon Sep 17 00:00:00 2001 From: fogelito Date: Sat, 9 Dec 2023 19:18:15 +0100 Subject: [PATCH 07/42] Queries --- composer.lock | 52 +++++++-------- docker-compose.yml | 2 +- src/Database/Adapter/Proxy.php | 55 +++++----------- src/Database/Adapter/ProxyMariaDB.php | 94 +++++++++++++++++++++++++++ tests/Database/Base.php | 71 ++++++++++---------- 5 files changed, 177 insertions(+), 97 deletions(-) diff --git a/composer.lock b/composer.lock index 6de483eec..3da2aed95 100644 --- a/composer.lock +++ b/composer.lock @@ -307,16 +307,16 @@ }, { "name": "utopia-php/framework", - "version": "0.31.0", + "version": "0.31.1", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "207f77378965fca9a9bc3783ea379d3549f86bc0" + "reference": "e50d2d16f4bc31319043f3f6d3dbea36c6fd6b68" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/207f77378965fca9a9bc3783ea379d3549f86bc0", - "reference": "207f77378965fca9a9bc3783ea379d3549f86bc0", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/e50d2d16f4bc31319043f3f6d3dbea36c6fd6b68", + "reference": "e50d2d16f4bc31319043f3f6d3dbea36c6fd6b68", "shasum": "" }, "require": { @@ -346,9 +346,9 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.31.0" + "source": "https://github.com/utopia-php/framework/tree/0.31.1" }, - "time": "2023-08-30T16:10:04+00:00" + "time": "2023-12-08T18:47:29+00:00" }, { "name": "utopia-php/mongo", @@ -878,16 +878,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.10.38", + "version": "1.10.48", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "5302bb402c57f00fb3c2c015bac86e0827e4b691" + "reference": "087ed4b5f4a7a6e8f3bbdfbfe98ce5c181380bc6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/5302bb402c57f00fb3c2c015bac86e0827e4b691", - "reference": "5302bb402c57f00fb3c2c015bac86e0827e4b691", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/087ed4b5f4a7a6e8f3bbdfbfe98ce5c181380bc6", + "reference": "087ed4b5f4a7a6e8f3bbdfbfe98ce5c181380bc6", "shasum": "" }, "require": { @@ -936,7 +936,7 @@ "type": "tidelift" } ], - "time": "2023-10-06T14:19:14+00:00" + "time": "2023-12-08T14:34:28+00:00" }, { "name": "phpunit/php-code-coverage", @@ -1259,16 +1259,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.6.13", + "version": "9.6.15", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "f3d767f7f9e191eab4189abe41ab37797e30b1be" + "reference": "05017b80304e0eb3f31d90194a563fd53a6021f1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/f3d767f7f9e191eab4189abe41ab37797e30b1be", - "reference": "f3d767f7f9e191eab4189abe41ab37797e30b1be", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/05017b80304e0eb3f31d90194a563fd53a6021f1", + "reference": "05017b80304e0eb3f31d90194a563fd53a6021f1", "shasum": "" }, "require": { @@ -1342,7 +1342,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.13" + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.15" }, "funding": [ { @@ -1358,7 +1358,7 @@ "type": "tidelift" } ], - "time": "2023-09-19T05:39:22+00:00" + "time": "2023-12-01T16:55:19+00:00" }, { "name": "psr/container", @@ -2467,7 +2467,7 @@ }, { "name": "symfony/deprecation-contracts", - "version": "v3.3.0", + "version": "v3.4.0", "source": { "type": "git", "url": "https://github.com/symfony/deprecation-contracts.git", @@ -2514,7 +2514,7 @@ "description": "A generic function and convention to trigger deprecation notices", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/deprecation-contracts/tree/v3.3.0" + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.4.0" }, "funding": [ { @@ -2534,16 +2534,16 @@ }, { "name": "theseer/tokenizer", - "version": "1.2.1", + "version": "1.2.2", "source": { "type": "git", "url": "https://github.com/theseer/tokenizer.git", - "reference": "34a41e998c2183e22995f158c581e7b5e755ab9e" + "reference": "b2ad5003ca10d4ee50a12da31de12a5774ba6b96" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/theseer/tokenizer/zipball/34a41e998c2183e22995f158c581e7b5e755ab9e", - "reference": "34a41e998c2183e22995f158c581e7b5e755ab9e", + "url": "https://api.github.com/repos/theseer/tokenizer/zipball/b2ad5003ca10d4ee50a12da31de12a5774ba6b96", + "reference": "b2ad5003ca10d4ee50a12da31de12a5774ba6b96", "shasum": "" }, "require": { @@ -2572,7 +2572,7 @@ "description": "A small library for converting tokenized PHP source code into XML and potentially other formats", "support": { "issues": "https://github.com/theseer/tokenizer/issues", - "source": "https://github.com/theseer/tokenizer/tree/1.2.1" + "source": "https://github.com/theseer/tokenizer/tree/1.2.2" }, "funding": [ { @@ -2580,7 +2580,7 @@ "type": "github" } ], - "time": "2021-07-28T10:34:58+00:00" + "time": "2023-11-20T00:12:19+00:00" }, { "name": "utopia-php/cli", @@ -2647,5 +2647,5 @@ "php": ">=8.0" }, "platform-dev": [], - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.6.0" } diff --git a/docker-compose.yml b/docker-compose.yml index f153adb43..d2dd7f8c8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -90,7 +90,7 @@ services: - database environment: - UTOPIA_DATABASE_PROXY_SECRET=test-secret - - UTOPIA_DATABASE_PROXY_SECRET_CONNECTIONS=default=mariadb://root:password@proxy-mariadb:3306/appwrite + - UTOPIA_DATABASE_PROXY_SECRET_CONNECTION=mariadb://root:password@proxy-mariadb:3306/appwrite proxy-mariadb: image: mariadb:10.7 diff --git a/src/Database/Adapter/Proxy.php b/src/Database/Adapter/Proxy.php index 489156de2..5efc18cfc 100644 --- a/src/Database/Adapter/Proxy.php +++ b/src/Database/Adapter/Proxy.php @@ -59,9 +59,6 @@ private function query(string $method, string $path, mixed $body = []): mixed body: $body ); - - var_dump($response); - if ($response->getStatusCode() >= 400) { if (empty($response->getBody())) { throw new Exception('Internal ' . $response->getStatusCode() . ' HTTP error in database proxy'); @@ -127,14 +124,13 @@ public function create(string $name): bool public function exists(string $database, ?string $collection): bool { $path = '/databases/' . $database; - $body = []; - if(!empty($collectione)){ - $path .= '/collections/' . $collection; - $body = ['database' => $database]; + if(!empty($collection)) + { + $path = '/collections/' . $collection . '?database=' . $database; } - return $this->query('GET', $path, $body); + return $this->query('GET', $path, []); } /** @@ -364,7 +360,7 @@ public function createIndex(string $collection, string $id, string $type, array */ public function deleteIndex(string $collection, string $id): bool { - return $this->query('DELETE', '/collections/' . $collection . '/indexes/' . $id . '/name', []); + return $this->query('DELETE', '/collections/'. $collection .'/indexes/' . $id, []); } /** @@ -374,10 +370,20 @@ public function deleteIndex(string $collection, string $id): bool * @param string $id * @param array $queries * @return Document + * @throws DatabaseException */ public function getDocument(string $collection, string $id, array $queries = []): Document { - return new Document($this->query('GET', '/collections/'.$collection.'/documents/' . $id, ['queries' => $queries])); + $path = '/collections/' . $collection . '/documents/' . $id; + + $arr = []; + foreach ($queries as $query){ + $arr[] = json_encode($query->jsonSerialize()); + } + + $path .= '?' . http_build_query(['queries'=> $arr]); + + return new Document($this->query('GET', $path, [])); } /** @@ -440,6 +446,8 @@ public function deleteDocument(string $collection, string $id): bool */ public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, ?int $timeout = null): array { + + $results = $this->query('GET', '/collections/' . $collection . '/documents', [ 'queries' => $queries, 'limit' => $limit, @@ -508,33 +516,6 @@ public function getSizeOfCollection(string $collection): int return $this->query('GET', '/collections/' . $collection . '/size', []); } - /** - * Get current index count from collection document - * - * @param Document $collection - * @return int - */ - public function getCountOfIndexes(Document $collection): int - { - //todo: removed... - return $this->query('getCountOfIndexes', ['collection' => $collection]); - } - - /** - * Estimate maximum number of bytes required to store a document in $collection. - * Byte requirement varies based on column type and size. - * Needed to satisfy MariaDB/MySQL row width limit. - * Return 0 when no restrictions apply to row width - * - * @param Document $collection - * @return int - */ - public function getAttributeWidth(Document $collection): int - { - // todo: removed... - return $this->query('getAttributeWidth', ['collection' => $collection]); - } - /** * Get an attribute projection given a list of selected attributes * diff --git a/src/Database/Adapter/ProxyMariaDB.php b/src/Database/Adapter/ProxyMariaDB.php index 929b19f78..308f32842 100644 --- a/src/Database/Adapter/ProxyMariaDB.php +++ b/src/Database/Adapter/ProxyMariaDB.php @@ -2,7 +2,9 @@ namespace Utopia\Database\Adapter; +use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; class ProxyMariaDB extends Proxy { @@ -209,6 +211,98 @@ public function getCountOfIndexes(Document $collection): int return $indexes + static::getCountOfDefaultIndexes(); } + /** + * Estimate maximum number of bytes required to store a document in $collection. + * Byte requirement varies based on column type and size. + * Needed to satisfy MariaDB/MySQL row width limit. + * + * @param Document $collection + * @return int + */ + public function getAttributeWidth(Document $collection): int + { + // Default collection has: + // `_id` int(11) => 4 bytes + // `_uid` char(255) => 1020 (255 bytes * 4 for utf8mb4) + // but this number seems to vary, so we give a +500 byte buffer + $total = 1500; + + $attributes = $collection->getAttributes()['attributes']; + + foreach ($attributes as $attribute) { + switch ($attribute['type']) { + case Database::VAR_STRING: + switch (true) { + case ($attribute['size'] > 16777215): + // 8 bytes length + 4 bytes for LONGTEXT + $total += 12; + break; + + case ($attribute['size'] > 65535): + // 8 bytes length + 3 bytes for MEDIUMTEXT + $total += 11; + break; + + case ($attribute['size'] > $this->getMaxVarcharLength()): + // 8 bytes length + 2 bytes for TEXT + $total += 10; + break; + + case ($attribute['size'] > 255): + // $size = $size * 4; // utf8mb4 up to 4 bytes per char + // 8 bytes length + 2 bytes for VARCHAR(>255) + $total += ($attribute['size'] * 4) + 2; + break; + + default: + // $size = $size * 4; // utf8mb4 up to 4 bytes per char + // 8 bytes length + 1 bytes for VARCHAR(<=255) + $total += ($attribute['size'] * 4) + 1; + break; + } + break; + + case Database::VAR_INTEGER: + if ($attribute['size'] >= 8) { + $total += 8; // BIGINT takes 8 bytes + } else { + $total += 4; // INT takes 4 bytes + } + break; + case Database::VAR_FLOAT: + // DOUBLE takes 8 bytes + $total += 8; + break; + + case Database::VAR_BOOLEAN: + // TINYINT(1) takes one byte + $total += 1; + break; + + case Database::VAR_RELATIONSHIP: + // INT(11) + $total += 4; + break; + + case Database::VAR_DATETIME: + $total += 19; // 2022-06-26 14:46:24 + break; + default: + throw new DatabaseException('Unknown type: ' . $attribute['type']); + } + } + + return $total; + } + + /** + * @return int + */ + public function getMaxVarcharLength(): int + { + return 16381; // Floor value for Postgres:16383 | MySQL:16381 | MariaDB:16382 + } + /** * Get list of keywords that cannot be used * Refference: https://mariadb.com/kb/en/reserved-words/ diff --git a/tests/Database/Base.php b/tests/Database/Base.php index 98e78afec..c0a965b1c 100644 --- a/tests/Database/Base.php +++ b/tests/Database/Base.php @@ -219,39 +219,39 @@ public function testCreatedAtUpdatedAt(): void $this->assertNotEmpty($document->getInternalId()); $this->assertNotNull($document->getInternalId()); } - - public function testQueryTimeoutUsingStaticTimeout(): void - { - if ($this->getDatabase()->getAdapter()->getSupportForTimeouts()) { - static::getDatabase()->createCollection('global-timeouts'); - $this->assertEquals(true, static::getDatabase()->createAttribute('global-timeouts', 'longtext', Database::VAR_STRING, 100000000, true)); - - for ($i = 0 ; $i <= 5 ; $i++) { - static::getDatabase()->createDocument('global-timeouts', new Document([ - 'longtext' => file_get_contents(__DIR__ . '/../resources/longtext.txt'), - '$permissions' => [ - Permission::read(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()) - ] - ])); - } - - $this->expectException(Timeout::class); - static::getDatabase()->setTimeout(1); - - try { - static::getDatabase()->find('global-timeouts', [ - Query::notEqual('longtext', 'appwrite'), - ]); - } catch(Timeout $ex) { - static::getDatabase()->clearTimeout(); - static::getDatabase()->deleteCollection('global-timeouts'); - throw $ex; - } - } - $this->expectNotToPerformAssertions(); - } +// +// public function testQueryTimeoutUsingStaticTimeout(): void +// { +// if ($this->getDatabase()->getAdapter()->getSupportForTimeouts()) { +// static::getDatabase()->createCollection('global-timeouts'); +// $this->assertEquals(true, static::getDatabase()->createAttribute('global-timeouts', 'longtext', Database::VAR_STRING, 100000000, true)); +// +// for ($i = 0 ; $i <= 5 ; $i++) { +// static::getDatabase()->createDocument('global-timeouts', new Document([ +// 'longtext' => file_get_contents(__DIR__ . '/../resources/longtext.txt'), +// '$permissions' => [ +// Permission::read(Role::any()), +// Permission::update(Role::any()), +// Permission::delete(Role::any()) +// ] +// ])); +// } +// +// $this->expectException(Timeout::class); +// static::getDatabase()->setTimeout(1); +// +// try { +// static::getDatabase()->find('global-timeouts', [ +// Query::notEqual('longtext', 'appwrite'), +// ]); +// } catch(Timeout $ex) { +// static::getDatabase()->clearTimeout(); +// static::getDatabase()->deleteCollection('global-timeouts'); +// throw $ex; +// } +// } +// $this->expectNotToPerformAssertions(); +// } /** @@ -261,6 +261,7 @@ public function testCreateListExistsDeleteCollection(): void { $this->assertInstanceOf('Utopia\Database\Document', static::getDatabase()->createCollection('actors')); $this->assertCount(2, static::getDatabase()->listCollections()); + $this->assertEquals(true, static::getDatabase()->exists($this->testDatabase, 'actors')); // Collection names should not be unique @@ -1193,6 +1194,9 @@ public function testGetDocumentSelect(Document $document): Document $this->assertIsString($document->getAttribute('string')); $this->assertEquals('text📝', $document->getAttribute('string')); $this->assertIsInt($document->getAttribute('integer')); + + var_dump($document->getAttributes()); + $this->assertEquals(5, $document->getAttribute('integer')); $this->assertArrayNotHasKey('float', $document->getAttributes()); $this->assertArrayNotHasKey('boolean', $document->getAttributes()); @@ -1321,6 +1325,7 @@ public function testListDocumentSearch(): void $documents = static::getDatabase()->find('documents', [ Query::search('string', '*test+alias@email-provider.com'), ]); + var_dump($documents); $this->assertEquals(1, count($documents)); } From 850941f365323481f3ace8fbababdfe411a89f35 Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 10 Dec 2023 11:21:59 +0100 Subject: [PATCH 08/42] Queries find --- src/Database/Adapter/Proxy.php | 16 +++++-- src/Database/Adapter/ProxyMariaDB.php | 3 +- tests/Database/Base.php | 69 +++++++++++++-------------- 3 files changed, 47 insertions(+), 41 deletions(-) diff --git a/src/Database/Adapter/Proxy.php b/src/Database/Adapter/Proxy.php index 5efc18cfc..1174fa2e4 100644 --- a/src/Database/Adapter/Proxy.php +++ b/src/Database/Adapter/Proxy.php @@ -446,10 +446,13 @@ public function deleteDocument(string $collection, string $id): bool */ public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, ?int $timeout = null): array { + $arr = []; + foreach ($queries as $query){ + $arr[] = json_encode($query->jsonSerialize()); + } - - $results = $this->query('GET', '/collections/' . $collection . '/documents', [ - 'queries' => $queries, + $body = [ + 'queries'=> $arr, 'limit' => $limit, 'offset' => $offset, 'orderAttributes' => $orderAttributes, @@ -457,7 +460,12 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, 'cursor' => $cursor, 'cursorDirection' => $cursorDirection, 'timeout' => $timeout - ]); + ]; + + $path = '/collections/' . $collection . '/documents'; + $path .= '?' . http_build_query($body); + + $results = $this->query('GET', $path, []); foreach ($results as $index => $document) { $results[$index] = new Document($results[$index]); diff --git a/src/Database/Adapter/ProxyMariaDB.php b/src/Database/Adapter/ProxyMariaDB.php index 308f32842..73a2c6d05 100644 --- a/src/Database/Adapter/ProxyMariaDB.php +++ b/src/Database/Adapter/ProxyMariaDB.php @@ -157,7 +157,8 @@ public function getSupportForFulltextWildcardIndex(): bool */ public function getSupportForTimeouts(): bool { - return true; + // todo: make this true and test pass? + return false; } /** diff --git a/tests/Database/Base.php b/tests/Database/Base.php index c0a965b1c..aaf61b495 100644 --- a/tests/Database/Base.php +++ b/tests/Database/Base.php @@ -219,39 +219,39 @@ public function testCreatedAtUpdatedAt(): void $this->assertNotEmpty($document->getInternalId()); $this->assertNotNull($document->getInternalId()); } -// -// public function testQueryTimeoutUsingStaticTimeout(): void -// { -// if ($this->getDatabase()->getAdapter()->getSupportForTimeouts()) { -// static::getDatabase()->createCollection('global-timeouts'); -// $this->assertEquals(true, static::getDatabase()->createAttribute('global-timeouts', 'longtext', Database::VAR_STRING, 100000000, true)); -// -// for ($i = 0 ; $i <= 5 ; $i++) { -// static::getDatabase()->createDocument('global-timeouts', new Document([ -// 'longtext' => file_get_contents(__DIR__ . '/../resources/longtext.txt'), -// '$permissions' => [ -// Permission::read(Role::any()), -// Permission::update(Role::any()), -// Permission::delete(Role::any()) -// ] -// ])); -// } -// -// $this->expectException(Timeout::class); -// static::getDatabase()->setTimeout(1); -// -// try { -// static::getDatabase()->find('global-timeouts', [ -// Query::notEqual('longtext', 'appwrite'), -// ]); -// } catch(Timeout $ex) { -// static::getDatabase()->clearTimeout(); -// static::getDatabase()->deleteCollection('global-timeouts'); -// throw $ex; -// } -// } -// $this->expectNotToPerformAssertions(); -// } + + public function testQueryTimeoutUsingStaticTimeout(): void + { + if ($this->getDatabase()->getAdapter()->getSupportForTimeouts()) { + static::getDatabase()->createCollection('global-timeouts'); + $this->assertEquals(true, static::getDatabase()->createAttribute('global-timeouts', 'longtext', Database::VAR_STRING, 100000000, true)); + + for ($i = 0 ; $i <= 5 ; $i++) { + static::getDatabase()->createDocument('global-timeouts', new Document([ + 'longtext' => file_get_contents(__DIR__ . '/../resources/longtext.txt'), + '$permissions' => [ + Permission::read(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ] + ])); + } + + $this->expectException(Timeout::class); + static::getDatabase()->setTimeout(1); + + try { + static::getDatabase()->find('global-timeouts', [ + Query::notEqual('longtext', 'appwrite'), + ]); + } catch(Timeout $ex) { + static::getDatabase()->clearTimeout(); + static::getDatabase()->deleteCollection('global-timeouts'); + throw $ex; + } + } + $this->expectNotToPerformAssertions(); + } /** @@ -1195,8 +1195,6 @@ public function testGetDocumentSelect(Document $document): Document $this->assertEquals('text📝', $document->getAttribute('string')); $this->assertIsInt($document->getAttribute('integer')); - var_dump($document->getAttributes()); - $this->assertEquals(5, $document->getAttribute('integer')); $this->assertArrayNotHasKey('float', $document->getAttributes()); $this->assertArrayNotHasKey('boolean', $document->getAttributes()); @@ -1325,7 +1323,6 @@ public function testListDocumentSearch(): void $documents = static::getDatabase()->find('documents', [ Query::search('string', '*test+alias@email-provider.com'), ]); - var_dump($documents); $this->assertEquals(1, count($documents)); } From 778276cc5a97c22c93ea7874e411b0fcf1a5c043 Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 10 Dec 2023 11:51:15 +0100 Subject: [PATCH 09/42] Count Queries --- src/Database/Adapter/Proxy.php | 16 +++++++++++++--- tests/Database/Base.php | 2 ++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/Database/Adapter/Proxy.php b/src/Database/Adapter/Proxy.php index 1174fa2e4..2d3c121af 100644 --- a/src/Database/Adapter/Proxy.php +++ b/src/Database/Adapter/Proxy.php @@ -505,11 +505,21 @@ public function sum(string $collection, string $attribute, array $queries = [], */ public function count(string $collection, array $queries = [], ?int $max = null, ?int $timeout = null): int { - return $this->query('GET', '/collections/'. $collection .'/documents-count', [ - 'queries' => $queries, + $arr = []; + foreach ($queries as $query){ + $arr[] = json_encode($query->jsonSerialize()); + } + + $body = [ + 'queries'=> $arr, 'max' => $max, 'timeout' => $timeout - ]); + ]; + + $path = '/collections/'. $collection .'/documents-count'; + $path .= '?' . http_build_query($body); + + return $this->query('GET', $path, []); } /** diff --git a/tests/Database/Base.php b/tests/Database/Base.php index aaf61b495..072d9cc4c 100644 --- a/tests/Database/Base.php +++ b/tests/Database/Base.php @@ -2960,6 +2960,7 @@ public function testCount(): void $count = static::getDatabase()->count('movies'); $this->assertEquals(6, $count); $count = static::getDatabase()->count('movies', [Query::equal('year', [2019])]); + $this->assertEquals(2, $count); $count = static::getDatabase()->count('movies', [Query::equal('with-dash', ['Works'])]); $this->assertEquals(2, $count); @@ -2998,6 +2999,7 @@ public function testCount(): void public function testSum(): void { Authorization::setRole('user:x'); + $sum = static::getDatabase()->sum('movies', 'year', [Query::equal('year', [2019]),]); $this->assertEquals(2019 + 2019, $sum); $sum = static::getDatabase()->sum('movies', 'year'); From 099e5726c7c7b84f4d8d52fef38b37dcb7f58890 Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 10 Dec 2023 17:53:27 +0100 Subject: [PATCH 10/42] Fix throw Exception --- src/Database/Adapter/Proxy.php | 27 ++++++++++++++++++++------- src/Database/Adapter/ProxyMariaDB.php | 4 ++-- tests/Database/Base.php | 1 - 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/Database/Adapter/Proxy.php b/src/Database/Adapter/Proxy.php index 2d3c121af..78a706365 100644 --- a/src/Database/Adapter/Proxy.php +++ b/src/Database/Adapter/Proxy.php @@ -69,13 +69,14 @@ private function query(string $method, string $path, mixed $body = []): mixed try { $exception = new $error['type']($error['message'], $error['code']); /** - * @var \Utopia\Database\Exception $exception + * @var DatabaseException $exception */ $exception->setFile($error['file']); $exception->setLine($error['line']); // TODO: If possible in PHP, set trace too for better error reporting // $exception->setTrace($error['trace']); + } catch(Throwable $err) { // Cannot find exception type throw new Exception($error['message'], $error['code']); @@ -480,18 +481,30 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, * @param string $collection * @param string $attribute * @param array $queries - * @param int|null $max - * + * @param int|string|null $max + * @param int|string|null $timeout * @return int|float + * @throws DatabaseException + * @throws FetchException */ - public function sum(string $collection, string $attribute, array $queries = [], ?int $max = null, ?int $timeout = null): float|int + public function sum(string $collection, string $attribute, array $queries = [], int|string|null $max = null, int|string|null $timeout = null): float|int { - return $this->query('GET', '/collections/'. $collection .'/documents-sum', [ + $arr = []; + foreach ($queries as $query){ + $arr[] = json_encode($query->jsonSerialize()); + } + + $body = [ 'attribute' => $attribute, - 'queries' => $queries, + 'queries' => $arr, 'max' => $max, 'timeout' => $timeout - ]); + ]; + + $path = '/collections/'. $collection .'/documents-sum'; + $path .= '?' . http_build_query($body); + + return $this->query('GET', $path, []); } /** diff --git a/src/Database/Adapter/ProxyMariaDB.php b/src/Database/Adapter/ProxyMariaDB.php index 73a2c6d05..34718195d 100644 --- a/src/Database/Adapter/ProxyMariaDB.php +++ b/src/Database/Adapter/ProxyMariaDB.php @@ -157,8 +157,8 @@ public function getSupportForFulltextWildcardIndex(): bool */ public function getSupportForTimeouts(): bool { - // todo: make this true and test pass? - return false; + // todo: make this true for timeout Exceptions + return true; } /** diff --git a/tests/Database/Base.php b/tests/Database/Base.php index 072d9cc4c..1252c3c46 100644 --- a/tests/Database/Base.php +++ b/tests/Database/Base.php @@ -3739,7 +3739,6 @@ public function testExceptionDuplicate(Document $document): void { $document->setAttribute('$id', 'duplicated'); static::getDatabase()->createDocument($document->getCollection(), $document); - $this->expectException(DuplicateException::class); static::getDatabase()->createDocument($document->getCollection(), $document); } From 7d09b05838dbcff70dac7bcd3b187f362b3aea21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 29 Jan 2024 09:35:20 +0000 Subject: [PATCH 11/42] Updates after Data API rename --- .github/workflows/tests.yml | 8 -------- docker-compose.yml | 15 +++++++-------- src/Database/Adapter/{Proxy.php => DataAPI.php} | 12 +++--------- .../{ProxyMariaDB.php => DataAPIMariaDB.php} | 2 +- src/Database/Exception.php | 7 +++---- ...roxyMariaDBTest.php => DataAPIMariaDBTest.php} | 10 ++++------ 6 files changed, 18 insertions(+), 36 deletions(-) rename src/Database/Adapter/{Proxy.php => DataAPI.php} (97%) rename src/Database/Adapter/{ProxyMariaDB.php => DataAPIMariaDB.php} (99%) rename tests/Database/Adapter/{ProxyMariaDBTest.php => DataAPIMariaDBTest.php} (70%) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index efebd21f0..970fc22de 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,14 +28,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - - name: Build Database Proxy - run: | - git clone https://github.com/utopia-php/database-proxy.git - cd database-proxy - git checkout dev - docker build -t utopia-php/database-proxy-test . - cd .. - - name: Start Databases run: | docker compose up -d diff --git a/docker-compose.yml b/docker-compose.yml index d2dd7f8c8..1bec317f8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,19 +82,18 @@ services: networks: - database - proxy: - # TODO: Release and use official image - image: database-proxy # Build from https://github.com/utopia-php/database-proxy/ - container_name: utopia-proxy + data-api: + image: utopia-php/data-api:0.1.0 + container_name: utopia-data-api networks: - database environment: - - UTOPIA_DATABASE_PROXY_SECRET=test-secret - - UTOPIA_DATABASE_PROXY_SECRET_CONNECTION=mariadb://root:password@proxy-mariadb:3306/appwrite + - UTOPIA_DATABASE_API_SECRET=test-secret + - UTOPIA_DATABASE_API_SECRET_CONNECTION=mariadb://root:password@data-api-mariadb:3306/appwrite - proxy-mariadb: + data-api-mariadb: image: mariadb:10.7 - container_name: utopia-proxy-mariadb + container_name: utopia-data-api-mariadb networks: - database environment: diff --git a/src/Database/Adapter/Proxy.php b/src/Database/Adapter/DataAPI.php similarity index 97% rename from src/Database/Adapter/Proxy.php rename to src/Database/Adapter/DataAPI.php index 78a706365..0e2ee6fbe 100644 --- a/src/Database/Adapter/Proxy.php +++ b/src/Database/Adapter/DataAPI.php @@ -13,7 +13,7 @@ use Utopia\Database\Validator\Authorization; use Utopia\Fetch\FetchException; -abstract class Proxy extends Adapter +abstract class DataAPI extends Adapter { protected string $endpoint; protected string $secret; @@ -61,22 +61,16 @@ private function query(string $method, string $path, mixed $body = []): mixed if ($response->getStatusCode() >= 400) { if (empty($response->getBody())) { - throw new Exception('Internal ' . $response->getStatusCode() . ' HTTP error in database proxy'); + throw new Exception('Internal ' . $response->getStatusCode() . ' HTTP error in data api'); } $error = \json_decode($response->getBody(), true); try { - $exception = new $error['type']($error['message'], $error['code']); + $exception = new $error['type']($error['message'], $error['code'], $error['file'], $error['line']); /** * @var DatabaseException $exception */ - - $exception->setFile($error['file']); - $exception->setLine($error['line']); - // TODO: If possible in PHP, set trace too for better error reporting - // $exception->setTrace($error['trace']); - } catch(Throwable $err) { // Cannot find exception type throw new Exception($error['message'], $error['code']); diff --git a/src/Database/Adapter/ProxyMariaDB.php b/src/Database/Adapter/DataAPIMariaDB.php similarity index 99% rename from src/Database/Adapter/ProxyMariaDB.php rename to src/Database/Adapter/DataAPIMariaDB.php index 34718195d..d8b552bd0 100644 --- a/src/Database/Adapter/ProxyMariaDB.php +++ b/src/Database/Adapter/DataAPIMariaDB.php @@ -6,7 +6,7 @@ use Utopia\Database\Document; use Utopia\Database\Exception as DatabaseException; -class ProxyMariaDB extends Proxy +class DataAPIMariaDB extends DataAPI { /** * Get max STRING limit diff --git a/src/Database/Exception.php b/src/Database/Exception.php index b2b3954ad..b5fbb8398 100644 --- a/src/Database/Exception.php +++ b/src/Database/Exception.php @@ -4,12 +4,11 @@ class Exception extends \Exception { - public function setFile(string $file): void + public function __construct(string $message = '', int $code = 0, string $file = '', int $line = 0) { + $this->message = $message; + $this->code = $code; $this->file = $file; - } - public function setLine(int $line): void - { $this->line = $line; } } diff --git a/tests/Database/Adapter/ProxyMariaDBTest.php b/tests/Database/Adapter/DataAPIMariaDBTest.php similarity index 70% rename from tests/Database/Adapter/ProxyMariaDBTest.php rename to tests/Database/Adapter/DataAPIMariaDBTest.php index e26938087..7be9069f1 100644 --- a/tests/Database/Adapter/ProxyMariaDBTest.php +++ b/tests/Database/Adapter/DataAPIMariaDBTest.php @@ -5,15 +5,13 @@ use Utopia\Cache\Adapter\None; use Utopia\Database\Database; use Utopia\Cache\Cache; -use Utopia\Database\Adapter\ProxyMariaDB; +use Utopia\Database\Adapter\DataAPIMariaDB; use Utopia\Tests\Base; -class ProxyMariaDBTest extends Base +class DataAPIMariaDBTest extends Base { public static ?Database $database = null; - // TODO@kodumbeats hacky way to identify adapters for tests - // Remove once all methods are implemented /** * Return name of adapter * @@ -21,7 +19,7 @@ class ProxyMariaDBTest extends Base */ public static function getAdapterName(): string { - return "proxy-mariadb"; + return "data-api-mariadb"; } /** @@ -33,7 +31,7 @@ public static function getDatabase(): Database return self::$database; } - $database = new Database(new ProxyMariaDB('http://proxy/v1', 'test-secret', 'default'), new Cache(new None())); + $database = new Database(new DataAPIMariaDB('http://data-api/v1', 'test-secret', 'default'), new Cache(new None())); $database->setDefaultDatabase('utopiaTests'); $database->setNamespace('myapp_' . uniqid()); From 6f0fa096e0646b0bc70347a8aa84abb9637b69be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Mon, 29 Jan 2024 09:45:02 +0000 Subject: [PATCH 12/42] Bug fixes after main merge --- composer.json | 2 +- composer.lock | 77 ++++++++++++++++++------ docker-compose.yml | 4 +- src/Database/Adapter/DataAPI.php | 27 ++++----- tests/e2e/Adapter/DataAPIMariaDBTest.php | 5 +- 5 files changed, 76 insertions(+), 39 deletions(-) diff --git a/composer.json b/composer.json index c56adedeb..9124bc56d 100755 --- a/composer.json +++ b/composer.json @@ -36,7 +36,7 @@ "ext-pdo": "*", "ext-mbstring": "*", "php": ">=8.0", - "utopia-php/framework": "0.*.*", + "utopia-php/framework": "0.33.*", "utopia-php/cache": "0.9.*", "utopia-php/mongo": "0.3.*", "utopia-php/fetch": "0.1.*" diff --git a/composer.lock b/composer.lock index 122a03843..28544faed 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d6ee891117614d18ee2f10466bfea758", + "content-hash": "4adf56005e024ee6b5bb991e96e2ee72", "packages": [ { "name": "jean85/pretty-package-versions", @@ -267,6 +267,45 @@ }, "time": "2024-01-07T18:11:23+00:00" }, + { + "name": "utopia-php/fetch", + "version": "0.1.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/fetch.git", + "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/2fa214b9262acd1a3583515a364da4f35929d5c5", + "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5", + "shasum": "" + }, + "require": { + "php": ">=8.0" + }, + "require-dev": { + "laravel/pint": "^1.5.0", + "phpstan/phpstan": "^1.10", + "phpunit/phpunit": "^9.5" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Fetch\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A simple library that provides an interface for making HTTP Requests.", + "support": { + "issues": "https://github.com/utopia-php/fetch/issues", + "source": "https://github.com/utopia-php/fetch/tree/0.1.0" + }, + "time": "2023-10-10T11:58:32+00:00" + }, { "name": "utopia-php/framework", "version": "0.33.1", @@ -509,16 +548,16 @@ }, { "name": "laravel/pint", - "version": "v1.13.9", + "version": "v1.13.10", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "e3e269cc5d874c8efd2dc7962b1c7ff2585fe525" + "reference": "e2b5060885694ca30ac008c05dc9d47f10ed1abf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/e3e269cc5d874c8efd2dc7962b1c7ff2585fe525", - "reference": "e3e269cc5d874c8efd2dc7962b1c7ff2585fe525", + "url": "https://api.github.com/repos/laravel/pint/zipball/e2b5060885694ca30ac008c05dc9d47f10ed1abf", + "reference": "e2b5060885694ca30ac008c05dc9d47f10ed1abf", "shasum": "" }, "require": { @@ -529,8 +568,8 @@ "php": "^8.1.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.47.0", - "illuminate/view": "^10.40.0", + "friendsofphp/php-cs-fixer": "^3.47.1", + "illuminate/view": "^10.41.0", "larastan/larastan": "^2.8.1", "laravel-zero/framework": "^10.3.0", "mockery/mockery": "^1.6.7", @@ -571,7 +610,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2024-01-16T17:39:29+00:00" + "time": "2024-01-22T09:04:15+00:00" }, { "name": "myclabs/deep-copy", @@ -835,16 +874,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.10.56", + "version": "1.10.57", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "27816a01aea996191ee14d010f325434c0ee76fa" + "reference": "1627b1d03446904aaa77593f370c5201d2ecc34e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/27816a01aea996191ee14d010f325434c0ee76fa", - "reference": "27816a01aea996191ee14d010f325434c0ee76fa", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/1627b1d03446904aaa77593f370c5201d2ecc34e", + "reference": "1627b1d03446904aaa77593f370c5201d2ecc34e", "shasum": "" }, "require": { @@ -893,7 +932,7 @@ "type": "tidelift" } ], - "time": "2024-01-15T10:43:00+00:00" + "time": "2024-01-24T11:51:34+00:00" }, { "name": "phpunit/php-code-coverage", @@ -1216,16 +1255,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.6.15", + "version": "9.6.16", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "05017b80304e0eb3f31d90194a563fd53a6021f1" + "reference": "3767b2c56ce02d01e3491046f33466a1ae60a37f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/05017b80304e0eb3f31d90194a563fd53a6021f1", - "reference": "05017b80304e0eb3f31d90194a563fd53a6021f1", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/3767b2c56ce02d01e3491046f33466a1ae60a37f", + "reference": "3767b2c56ce02d01e3491046f33466a1ae60a37f", "shasum": "" }, "require": { @@ -1299,7 +1338,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.15" + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.16" }, "funding": [ { @@ -1315,7 +1354,7 @@ "type": "tidelift" } ], - "time": "2023-12-01T16:55:19+00:00" + "time": "2024-01-19T07:03:14+00:00" }, { "name": "psr/container", diff --git a/docker-compose.yml b/docker-compose.yml index b82694df4..5977daa8e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -88,8 +88,8 @@ services: networks: - database environment: - - UTOPIA_DATABASE_API_SECRET=test-secret - - UTOPIA_DATABASE_API_SECRET_CONNECTION=mariadb://root:password@data-api-mariadb:3306/appwrite + - UTOPIA_DATA_API_SECRET=test-secret + - UTOPIA_DATA_API_SECRET_CONNECTION=mariadb://root:password@data-api-mariadb:3306/appwrite data-api-mariadb: image: mariadb:10.7 diff --git a/src/Database/Adapter/DataAPI.php b/src/Database/Adapter/DataAPI.php index 0e2ee6fbe..2e443a667 100644 --- a/src/Database/Adapter/DataAPI.php +++ b/src/Database/Adapter/DataAPI.php @@ -48,11 +48,11 @@ private function query(string $method, string $path, mixed $body = []): mixed 'x-utopia-secret' => $this->secret, 'x-utopia-database' => $this->database, 'x-utopia-namespace' => $this->getNamespace(), - 'x-utopia-default-database' => $this->defaultDatabase, 'x-utopia-auth-roles' => $roles, 'x-utopia-auth-status' => Authorization::$status ? 'true' : 'false', 'x-utopia-auth-status-default' => Authorization::$statusDefault ? 'true' : 'false', - 'x-utopia-timeout' => self::$timeout ? \strval(self::$timeout) : '', + // TODO: Fix timeout + // 'x-utopia-timeout' => self::$timeout ? \strval(self::$timeout) : '', 'content-type' => 'application/json' ], method: $method, @@ -116,12 +116,11 @@ public function create(string $name): bool * * @return bool */ - public function exists(string $database, ?string $collection): bool + public function exists(string $database, ?string $collection = null): bool { $path = '/databases/' . $database; - if(!empty($collection)) - { + if(!empty($collection)) { $path = '/collections/' . $collection . '?database=' . $database; } @@ -317,7 +316,7 @@ public function deleteRelationship(string $collection, string $relatedCollection */ public function renameIndex(string $collection, string $old, string $new): bool { - return $this->query('PATCH', '/collections/'.$collection.'/indexes/'.$old.'/name',[ + return $this->query('PATCH', '/collections/'.$collection.'/indexes/'.$old.'/name', [ 'new' => $new ]); } @@ -372,11 +371,11 @@ public function getDocument(string $collection, string $id, array $queries = []) $path = '/collections/' . $collection . '/documents/' . $id; $arr = []; - foreach ($queries as $query){ + foreach ($queries as $query) { $arr[] = json_encode($query->jsonSerialize()); } - $path .= '?' . http_build_query(['queries'=> $arr]); + $path .= '?' . http_build_query(['queries' => $arr]); return new Document($this->query('GET', $path, [])); } @@ -419,7 +418,7 @@ public function updateDocument(string $collection, Document $document): Document */ public function deleteDocument(string $collection, string $id): bool { - return $this->query('DELETE','/collections/'. $collection .'/documents/' . $id, []); + return $this->query('DELETE', '/collections/'. $collection .'/documents/' . $id, []); } /** @@ -442,12 +441,12 @@ public function deleteDocument(string $collection, string $id): bool public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, ?int $timeout = null): array { $arr = []; - foreach ($queries as $query){ + foreach ($queries as $query) { $arr[] = json_encode($query->jsonSerialize()); } $body = [ - 'queries'=> $arr, + 'queries' => $arr, 'limit' => $limit, 'offset' => $offset, 'orderAttributes' => $orderAttributes, @@ -484,7 +483,7 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, public function sum(string $collection, string $attribute, array $queries = [], int|string|null $max = null, int|string|null $timeout = null): float|int { $arr = []; - foreach ($queries as $query){ + foreach ($queries as $query) { $arr[] = json_encode($query->jsonSerialize()); } @@ -513,12 +512,12 @@ public function sum(string $collection, string $attribute, array $queries = [], public function count(string $collection, array $queries = [], ?int $max = null, ?int $timeout = null): int { $arr = []; - foreach ($queries as $query){ + foreach ($queries as $query) { $arr[] = json_encode($query->jsonSerialize()); } $body = [ - 'queries'=> $arr, + 'queries' => $arr, 'max' => $max, 'timeout' => $timeout ]; diff --git a/tests/e2e/Adapter/DataAPIMariaDBTest.php b/tests/e2e/Adapter/DataAPIMariaDBTest.php index 7be9069f1..6df88b076 100644 --- a/tests/e2e/Adapter/DataAPIMariaDBTest.php +++ b/tests/e2e/Adapter/DataAPIMariaDBTest.php @@ -1,12 +1,11 @@ setDefaultDatabase('utopiaTests'); + $database->setDatabase('utopiaTests'); $database->setNamespace('myapp_' . uniqid()); if ($database->exists('utopiaTests')) { From 543e4f690ddbbdb7c3164f1bb9a82020e5bd9dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 22 Feb 2024 17:43:13 +0000 Subject: [PATCH 13/42] Update to follow latest data API endpoints --- .github/workflows/tests.yml | 1 + composer.lock | 53 ++- phpunit.xml | 2 +- src/Database/Adapter/DataAPI.php | 460 ++++++++++++------- src/Database/Adapter/DataAPIMariaDB.php | 557 +----------------------- 5 files changed, 326 insertions(+), 747 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f82d92b3d..631278763 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -77,6 +77,7 @@ jobs: Postgres, SQLite, MongoDB, + DataAPIMariaDB ] steps: diff --git a/composer.lock b/composer.lock index 28544faed..4f853a20f 100644 --- a/composer.lock +++ b/composer.lock @@ -136,16 +136,16 @@ }, { "name": "symfony/polyfill-php80", - "version": "v1.28.0", + "version": "v1.29.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "6caa57379c4aec19c0a12a38b59b26487dcfe4b5" + "reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/6caa57379c4aec19c0a12a38b59b26487dcfe4b5", - "reference": "6caa57379c4aec19c0a12a38b59b26487dcfe4b5", + "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/87b68208d5c1188808dd7839ee1e6c8ec3b02f1b", + "reference": "87b68208d5c1188808dd7839ee1e6c8ec3b02f1b", "shasum": "" }, "require": { @@ -153,9 +153,6 @@ }, "type": "library", "extra": { - "branch-alias": { - "dev-main": "1.28-dev" - }, "thanks": { "name": "symfony/polyfill", "url": "https://github.com/symfony/polyfill" @@ -199,7 +196,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/v1.28.0" + "source": "https://github.com/symfony/polyfill-php80/tree/v1.29.0" }, "funding": [ { @@ -215,7 +212,7 @@ "type": "tidelift" } ], - "time": "2023-01-26T09:26:14+00:00" + "time": "2024-01-29T20:11:03+00:00" }, { "name": "utopia-php/cache", @@ -308,16 +305,16 @@ }, { "name": "utopia-php/framework", - "version": "0.33.1", + "version": "0.33.2", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "b745607aa1875554a0ad52e28f6db918da1ce11c" + "reference": "b1423ca3e3b61c6c4c2e619d2cb80672809a19f3" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/b745607aa1875554a0ad52e28f6db918da1ce11c", - "reference": "b745607aa1875554a0ad52e28f6db918da1ce11c", + "url": "https://api.github.com/repos/utopia-php/http/zipball/b1423ca3e3b61c6c4c2e619d2cb80672809a19f3", + "reference": "b1423ca3e3b61c6c4c2e619d2cb80672809a19f3", "shasum": "" }, "require": { @@ -347,9 +344,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.1" + "source": "https://github.com/utopia-php/http/tree/0.33.2" }, - "time": "2024-01-17T16:48:32+00:00" + "time": "2024-01-31T10:35:59+00:00" }, { "name": "utopia-php/mongo", @@ -548,16 +545,16 @@ }, { "name": "laravel/pint", - "version": "v1.13.10", + "version": "v1.13.11", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "e2b5060885694ca30ac008c05dc9d47f10ed1abf" + "reference": "60a163c3e7e3346a1dec96d3e6f02e6465452552" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/e2b5060885694ca30ac008c05dc9d47f10ed1abf", - "reference": "e2b5060885694ca30ac008c05dc9d47f10ed1abf", + "url": "https://api.github.com/repos/laravel/pint/zipball/60a163c3e7e3346a1dec96d3e6f02e6465452552", + "reference": "60a163c3e7e3346a1dec96d3e6f02e6465452552", "shasum": "" }, "require": { @@ -568,13 +565,13 @@ "php": "^8.1.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.47.1", - "illuminate/view": "^10.41.0", + "friendsofphp/php-cs-fixer": "^3.49.0", + "illuminate/view": "^10.43.0", "larastan/larastan": "^2.8.1", "laravel-zero/framework": "^10.3.0", "mockery/mockery": "^1.6.7", "nunomaduro/termwind": "^1.15.1", - "pestphp/pest": "^2.31.0" + "pestphp/pest": "^2.33.6" }, "bin": [ "builds/pint" @@ -610,7 +607,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2024-01-22T09:04:15+00:00" + "time": "2024-02-13T17:20:13+00:00" }, { "name": "myclabs/deep-copy", @@ -874,16 +871,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.10.57", + "version": "1.10.59", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "1627b1d03446904aaa77593f370c5201d2ecc34e" + "reference": "e607609388d3a6d418a50a49f7940e8086798281" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/1627b1d03446904aaa77593f370c5201d2ecc34e", - "reference": "1627b1d03446904aaa77593f370c5201d2ecc34e", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/e607609388d3a6d418a50a49f7940e8086798281", + "reference": "e607609388d3a6d418a50a49f7940e8086798281", "shasum": "" }, "require": { @@ -932,7 +929,7 @@ "type": "tidelift" } ], - "time": "2024-01-24T11:51:34+00:00" + "time": "2024-02-20T13:59:13+00:00" }, { "name": "phpunit/php-code-coverage", diff --git a/phpunit.xml b/phpunit.xml index 783265d80..ccdaa969e 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="true"> + stopOnFailure="false"> ./tests/unit diff --git a/src/Database/Adapter/DataAPI.php b/src/Database/Adapter/DataAPI.php index 2e443a667..a39482fbd 100644 --- a/src/Database/Adapter/DataAPI.php +++ b/src/Database/Adapter/DataAPI.php @@ -35,15 +35,17 @@ public function __construct(string $endpoint, string $secret, string $database) } /** + * @param mixed[] $params + * * @throws FetchException * @throws DatabaseException * @throws Exception */ - private function query(string $method, string $path, mixed $body = []): mixed + private function query(string $query, array $params = []): mixed { $roles = \implode(',', Authorization::getRoles()); $response = Client::fetch( - url: $this->endpoint . $path, + url: $this->endpoint . '/queries', headers: [ 'x-utopia-secret' => $this->secret, 'x-utopia-database' => $this->database, @@ -55,8 +57,11 @@ private function query(string $method, string $path, mixed $body = []): mixed // 'x-utopia-timeout' => self::$timeout ? \strval(self::$timeout) : '', 'content-type' => 'application/json' ], - method: $method, - body: $body + method: 'POST', + body: [ + 'query' => $query, + 'params' => $params + ] ); if ($response->getStatusCode() >= 400) { @@ -79,19 +84,25 @@ private function query(string $method, string $path, mixed $body = []): mixed throw $exception; } - $body = \json_decode($response->getBody(), true); - return $body['output'] ?? ''; + $body = \json_decode($response->getBody(), false); + + $output = $body->output ?? ''; + + if(\is_object($output)) { + $output = new Document((array) $output); + } + + return $output; } /** * Ping Database * * @return bool - * @throws DatabaseException|FetchException */ public function ping(): bool { - return $this->query('GET', '/ping', []); + return $this->query('ping'); } /** @@ -100,11 +111,10 @@ public function ping(): bool * @param string $name * * @return bool - * @throws DatabaseException|FetchException */ public function create(string $name): bool { - return $this->query('POST', '/databases', ['database' => $name]); + return $this->query('create', [$name]); } /** @@ -112,19 +122,23 @@ public function create(string $name): bool * Optionally check if collection exists in database * * @param string $database database name - * @param string $collection (optional) collection name + * @param string|null $collection (optional) collection name * * @return bool */ public function exists(string $database, ?string $collection = null): bool { - $path = '/databases/' . $database; - - if(!empty($collection)) { - $path = '/collections/' . $collection . '?database=' . $database; - } + return $this->query('exists', [$database, $collection]); + } - return $this->query('GET', $path, []); + /** + * List Databases + * + * @return array + */ + public function list(): array + { + return $this->query('list'); } /** @@ -136,7 +150,7 @@ public function exists(string $database, ?string $collection = null): bool */ public function delete(string $name): bool { - return $this->query('DELETE', '/databases/' . $name, []); + return $this->query('delete', [$name]); } /** @@ -149,11 +163,7 @@ public function delete(string $name): bool */ public function createCollection(string $name, array $attributes = [], array $indexes = []): bool { - return $this->query('POST', '/collections', [ - 'collection' => $name, - 'attributes' => $attributes, - 'indexes' => $indexes - ]); + return $this->query('createCollection', [$name, $attributes, $indexes]); } /** @@ -165,7 +175,7 @@ public function createCollection(string $name, array $attributes = [], array $in */ public function deleteCollection(string $id): bool { - return $this->query('DELETE', '/collections/' . $id, []); + return $this->query('deleteCollection', [$id]); } /** @@ -181,13 +191,7 @@ public function deleteCollection(string $id): bool */ public function createAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false): bool { - return $this->query('POST', '/collections/' . $collection . '/attributes', [ - 'attribute' => $id, - 'type' => $type, - 'size' => $size, - 'signed' => $signed, - 'array' => $array, - ]); + return $this->query('createAttribute', [$collection, $id, $type, $size, $signed, $array]); } /** @@ -204,12 +208,7 @@ public function createAttribute(string $collection, string $id, string $type, in */ public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false): bool { - return $this->query('PUT', '/collections/'.$collection.'/attributes/' . $id, [ - 'type' => $type, - 'size' => $size, - 'signed' => $signed, - 'array' => $array, - ]); + return $this->query('updateAttribute', [$collection, $id, $type, $size, $signed, $array]); } /** @@ -222,7 +221,7 @@ public function updateAttribute(string $collection, string $id, string $type, in */ public function deleteAttribute(string $collection, string $id): bool { - return $this->query('DELETE', '/collections/' . $collection . '/attributes/' . $id, []); + return $this->query('deleteAttribute', [$collection, $id]); } /** @@ -235,7 +234,7 @@ public function deleteAttribute(string $collection, string $id): bool */ public function renameAttribute(string $collection, string $old, string $new): bool { - return $this->query('PATCH', '/collections/' . $collection . '/attributes/' . $old . '/name', ['new' => $new]); + return $this->query('renameAttribute', [$collection, $old, $new]); } /** @@ -249,13 +248,7 @@ public function renameAttribute(string $collection, string $old, string $new): b */ public function createRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay = false, string $id = '', string $twoWayKey = ''): bool { - return $this->query('POST', '/collections/' . $collection . '/relationships', [ - 'relatedCollection' => $relatedCollection, - 'type' => $type, - 'twoWay' => $twoWay, - 'id' => $id, - 'twoWayKey' => $twoWayKey - ]); + return $this->query('createRelationship', [$collection, $relatedCollection, $type, $twoWay, $id, $twoWayKey]); } /** @@ -273,14 +266,7 @@ public function createRelationship(string $collection, string $relatedCollection */ public function updateRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay, string $key, string $twoWayKey, ?string $newKey = null, ?string $newTwoWayKey = null): bool { - return $this->query('PUT', '/collections/'. $collection .'/relationships/' . $relatedCollection, [ - 'type' => $type, - 'twoWay' => $twoWay, - 'key' => $key, - 'twoWayKey' => $twoWayKey, - 'newKey' => $newKey, - 'newTwoWayKey' => $newTwoWayKey - ]); + return $this->query('updateRelationship', [$collection, $relatedCollection, $type, $twoWay, $key, $twoWayKey, $newKey, $newTwoWayKey]); } /** @@ -297,13 +283,7 @@ public function updateRelationship(string $collection, string $relatedCollection */ public function deleteRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay, string $key, string $twoWayKey, string $side): bool { - return $this->query('DELETE', '/collections/'. $collection .'/relationships/' . $relatedCollection, [ - 'type' => $type, - 'twoWay' => $twoWay, - 'key' => $key, - 'twoWayKey' => $twoWayKey, - 'side' => $side, - ]); + return $this->query('deleteRelationship', [$collection, $relatedCollection, $type, $twoWay, $key, $twoWayKey, $side]); } /** @@ -316,9 +296,7 @@ public function deleteRelationship(string $collection, string $relatedCollection */ public function renameIndex(string $collection, string $old, string $new): bool { - return $this->query('PATCH', '/collections/'.$collection.'/indexes/'.$old.'/name', [ - 'new' => $new - ]); + return $this->query('renameIndex', [$collection, $old, $new]); } /** @@ -335,13 +313,7 @@ public function renameIndex(string $collection, string $old, string $new): bool */ public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders): bool { - return $this->query('POST', '/collections/' . $collection . '/indexes', [ - 'index' => $id, - 'type' => $type, - 'attributes' => $attributes, - 'lengths' => $lengths, - 'orders' => $orders - ]); + return $this->query('createIndex', [$collection, $id, $type, $attributes, $lengths, $orders]); } /** @@ -354,7 +326,7 @@ public function createIndex(string $collection, string $id, string $type, array */ public function deleteIndex(string $collection, string $id): bool { - return $this->query('DELETE', '/collections/'. $collection .'/indexes/' . $id, []); + return $this->query('deleteIndex', [$collection, $id]); } /** @@ -364,20 +336,10 @@ public function deleteIndex(string $collection, string $id): bool * @param string $id * @param array $queries * @return Document - * @throws DatabaseException */ public function getDocument(string $collection, string $id, array $queries = []): Document { - $path = '/collections/' . $collection . '/documents/' . $id; - - $arr = []; - foreach ($queries as $query) { - $arr[] = json_encode($query->jsonSerialize()); - } - - $path .= '?' . http_build_query(['queries' => $arr]); - - return new Document($this->query('GET', $path, [])); + return $this->query('getDocument', [$collection, $id, $queries]); } /** @@ -387,11 +349,26 @@ public function getDocument(string $collection, string $id, array $queries = []) * @param Document $document * * @return Document - * @throws DatabaseException|FetchException */ public function createDocument(string $collection, Document $document): Document { - return new Document($this->query('POST', '/collections/' . $collection . '/documents', ['document' => $document])); + return $this->query('createDocument', [$collection, $document]); + } + + /** + * Create Documents in batches + * + * @param string $collection + * @param array $documents + * @param int $batchSize + * + * @return array + * + * @throws DatabaseException + */ + public function createDocuments(string $collection, array $documents, int $batchSize): array + { + return $this->query('createDocuments', [$collection, $documents, $batchSize]); } /** @@ -401,11 +378,26 @@ public function createDocument(string $collection, Document $document): Document * @param Document $document * * @return Document - * @throws DatabaseException|FetchException */ public function updateDocument(string $collection, Document $document): Document { - return new Document($this->query('PUT', '/collections/'. $collection .'/documents', ['document' => $document])); + return $this->query('updateDocument', [$collection, $document]); + } + + /** + * Update Documents in batches + * + * @param string $collection + * @param array $documents + * @param int $batchSize + * + * @return array + * + * @throws DatabaseException + */ + public function updateDocuments(string $collection, array $documents, int $batchSize): array + { + return $this->query('updateDocuments', [$collection, $documents, $batchSize]); } /** @@ -418,7 +410,7 @@ public function updateDocument(string $collection, Document $document): Document */ public function deleteDocument(string $collection, string $id): bool { - return $this->query('DELETE', '/collections/'. $collection .'/documents/' . $id, []); + return $this->query('deleteDocument', [$collection, $id]); } /** @@ -434,38 +426,12 @@ public function deleteDocument(string $collection, string $id): bool * @param array $orderTypes * @param array $cursor * @param string $cursorDirection - * @param int|null $timeout * * @return array */ - public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, ?int $timeout = null): array + public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER): array { - $arr = []; - foreach ($queries as $query) { - $arr[] = json_encode($query->jsonSerialize()); - } - - $body = [ - 'queries' => $arr, - 'limit' => $limit, - 'offset' => $offset, - 'orderAttributes' => $orderAttributes, - 'orderTypes' => $orderTypes, - 'cursor' => $cursor, - 'cursorDirection' => $cursorDirection, - 'timeout' => $timeout - ]; - - $path = '/collections/' . $collection . '/documents'; - $path .= '?' . http_build_query($body); - - $results = $this->query('GET', $path, []); - - foreach ($results as $index => $document) { - $results[$index] = new Document($results[$index]); - } - - return $results; + return $this->query('find', [$collection, $queries, $limit, $offset, $orderAttributes, $orderTypes, $cursor, $cursorDirection]); } /** @@ -474,30 +440,13 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, * @param string $collection * @param string $attribute * @param array $queries - * @param int|string|null $max - * @param int|string|null $timeout + * @param int|null $max + * * @return int|float - * @throws DatabaseException - * @throws FetchException */ - public function sum(string $collection, string $attribute, array $queries = [], int|string|null $max = null, int|string|null $timeout = null): float|int + public function sum(string $collection, string $attribute, array $queries = [], ?int $max = null): float|int { - $arr = []; - foreach ($queries as $query) { - $arr[] = json_encode($query->jsonSerialize()); - } - - $body = [ - 'attribute' => $attribute, - 'queries' => $arr, - 'max' => $max, - 'timeout' => $timeout - ]; - - $path = '/collections/'. $collection .'/documents-sum'; - $path .= '?' . http_build_query($body); - - return $this->query('GET', $path, []); + return $this->query('sum', [$collection, $attribute, $queries, $max]); } /** @@ -509,23 +458,9 @@ public function sum(string $collection, string $attribute, array $queries = [], * * @return int */ - public function count(string $collection, array $queries = [], ?int $max = null, ?int $timeout = null): int + public function count(string $collection, array $queries = [], ?int $max = null): int { - $arr = []; - foreach ($queries as $query) { - $arr[] = json_encode($query->jsonSerialize()); - } - - $body = [ - 'queries' => $arr, - 'max' => $max, - 'timeout' => $timeout - ]; - - $path = '/collections/'. $collection .'/documents-count'; - $path .= '?' . http_build_query($body); - - return $this->query('GET', $path, []); + return $this->query('count', [$collection, $queries, $max]); } /** @@ -533,11 +468,188 @@ public function count(string $collection, array $queries = [], ?int $max = null, * * @param string $collection * @return int - * @throws DatabaseException|FetchException + * @throws DatabaseException */ public function getSizeOfCollection(string $collection): int { - return $this->query('GET', '/collections/' . $collection . '/size', []); + return $this->query('getSizeOfCollection', [$collection]); + } + + /** + * Get max STRING limit + * + * @return int + */ + public function getLimitForString(): int + { + return $this->query('getLimitForString'); + } + + /** + * Get max INT limit + * + * @return int + */ + public function getLimitForInt(): int + { + return $this->query('getLimitForInt'); + } + + /** + * Get maximum attributes limit. + * + * @return int + */ + public function getLimitForAttributes(): int + { + return $this->query('getLimitForAttributes'); + } + + /** + * Get maximum index limit. + * + * @return int + */ + public function getLimitForIndexes(): int + { + return $this->query('getLimitForIndexes'); + } + + /** + * Is schemas supported? + * + * @return bool + */ + public function getSupportForSchemas(): bool + { + return $this->query('getSupportForSchemas'); + } + + /** + * Is index supported? + * + * @return bool + */ + public function getSupportForIndex(): bool + { + return $this->query('getSupportForIndex'); + } + + /** + * Is unique index supported? + * + * @return bool + */ + public function getSupportForUniqueIndex(): bool + { + return $this->query('getSupportForUniqueIndex'); + } + + /** + * Is fulltext index supported? + * + * @return bool + */ + public function getSupportForFulltextIndex(): bool + { + return $this->query('getSupportForFulltextIndex'); + } + + /** + * Is fulltext wildcard supported? + * + * @return bool + */ + public function getSupportForFulltextWildcardIndex(): bool + { + return $this->query('getSupportForFulltextWildcardIndex'); + } + + + /** + * Does the adapter handle casting? + * + * @return bool + */ + public function getSupportForCasting(): bool + { + return $this->query('getSupportForCasting'); + } + + /** + * Does the adapter handle array Contains? + * + * @return bool + */ + public function getSupportForQueryContains(): bool + { + return $this->query('getSupportForQueryContains'); + } + + /** + * Are timeouts supported? + * + * @return bool + */ + public function getSupportForTimeouts(): bool + { + return $this->query('getSupportForTimeouts'); + } + + /** + * Are relationships supported? + * + * @return bool + */ + public function getSupportForRelationships(): bool + { + return $this->query('getSupportForRelationships'); + } + + /** + * Get current attribute count from collection document + * + * @param Document $collection + * @return int + */ + public function getCountOfAttributes(Document $collection): int + { + return $this->query('getCountOfAttributes', [$collection]); + } + + /** + * Get current index count from collection document + * + * @param Document $collection + * @return int + */ + public function getCountOfIndexes(Document $collection): int + { + return $this->query('getCountOfIndexes', [$collection]); + } + + /** + * Estimate maximum number of bytes required to store a document in $collection. + * Byte requirement varies based on column type and size. + * Needed to satisfy MariaDB/MySQL row width limit. + * Return 0 when no restrictions apply to row width + * + * @param Document $collection + * @return int + */ + public function getAttributeWidth(Document $collection): int + { + return $this->query('getAttributeWidth', [$collection]); + } + + /** + * Get list of keywords that cannot be used + * + * @return array + */ + public function getKeywords(): array + { + return $this->query('getKeywords'); } /** @@ -549,8 +661,7 @@ public function getSizeOfCollection(string $collection): int */ protected function getAttributeProjection(array $selections, string $prefix = ''): mixed { - // Not nessessary for this adapter - return []; + return $this->query('getAttributeProjection', [$selections, $prefix]); } /** @@ -567,11 +678,34 @@ protected function getAttributeProjection(array $selections, string $prefix = '' */ public function increaseDocumentAttribute(string $collection, string $id, string $attribute, int|float $value, int|float|null $min = null, int|float|null $max = null): bool { - return $this->query('PATCH', '/collections/'. $collection .'/documents/'. $id .'/increase', [ - 'attribute' => $attribute, - 'value' => $value, - 'min' => $min, - 'max' => $max - ]); + return $this->query('increaseDocumentAttribute', [$collection, $id, $attribute, $value, $min, $max]); + } + + /** + * @return int + */ + public function getMaxIndexLength(): int + { + return $this->query('getMaxIndexLength'); + } + + /** + * Set a global timeout for database queries in milliseconds. + * + * This function allows you to set a maximum execution time for all database + * queries executed using the library, or a specific event specified by the + * event parameter. Once this timeout is set, any database query that takes + * longer than the specified time will be automatically terminated by the library, + * and an appropriate error or exception will be raised to handle the timeout condition. + * + * @param int $milliseconds The timeout value in milliseconds for database queries. + * @param string $event The event the timeout should fire fore + * @return void + * + * @throws Exception The provided timeout value must be greater than or equal to 0. + */ + public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): void + { + $this->query('setTimeout', [$milliseconds, $event]); } } diff --git a/src/Database/Adapter/DataAPIMariaDB.php b/src/Database/Adapter/DataAPIMariaDB.php index d8b552bd0..0cd520907 100644 --- a/src/Database/Adapter/DataAPIMariaDB.php +++ b/src/Database/Adapter/DataAPIMariaDB.php @@ -3,94 +3,9 @@ namespace Utopia\Database\Adapter; use Utopia\Database\Database; -use Utopia\Database\Document; -use Utopia\Database\Exception as DatabaseException; class DataAPIMariaDB extends DataAPI { - /** - * Get max STRING limit - * - * @return int - */ - public function getLimitForString(): int - { - return 4294967295; - } - - /** - * Get max INT limit - * - * @return int - */ - public function getLimitForInt(): int - { - return 4294967295; - } - - /** - * Get maximum column limit. - * https://mariadb.com/kb/en/innodb-limitations/#limitations-on-schema - * Can be inherited by MySQL since we utilize the InnoDB engine - * - * @return int - */ - public function getLimitForAttributes(): int - { - return 1017; - } - - /** - * Get maximum index limit. - * https://mariadb.com/kb/en/innodb-limitations/#limitations-on-schema - * - * @return int - */ - public function getLimitForIndexes(): int - { - return 64; - } - - /** - * Is schemas supported? - * - * @return bool - */ - public function getSupportForSchemas(): bool - { - return true; - } - - /** - * Is index supported? - * - * @return bool - */ - public function getSupportForIndex(): bool - { - return true; - } - - /** - * Is unique index supported? - * - * @return bool - */ - public function getSupportForUniqueIndex(): bool - { - return true; - } - - /** - * Is fulltext index supported? - * - * @return bool - */ - public function getSupportForFulltextIndex(): bool - { - return true; - } - /** * Returns number of attributes used by default. * @@ -98,7 +13,7 @@ public function getSupportForFulltextIndex(): bool */ public static function getCountOfDefaultAttributes(): int { - return 4; + return \count(Database::INTERNAL_ATTRIBUTES); } /** @@ -108,7 +23,7 @@ public static function getCountOfDefaultAttributes(): int */ public static function getCountOfDefaultIndexes(): int { - return 5; + return \count(Database::INTERNAL_INDEXES); } /** @@ -121,472 +36,4 @@ public static function getDocumentSizeLimit(): int { return 65535; } - - /** - * @return int - */ - public function getMaxIndexLength(): int - { - return 768; - } - - /** - * List Databases - * - * @return array - */ - public function list(): array - { - return []; - } - - /** - * Is fulltext Wildcard index supported? - * - * @return bool - */ - public function getSupportForFulltextWildcardIndex(): bool - { - return true; - } - - /** - * Are timeouts supported? - * - * @return bool - */ - public function getSupportForTimeouts(): bool - { - // todo: make this true for timeout Exceptions - return true; - } - - /** - * Does the adapter handle casting? - * - * @return bool - */ - public function getSupportForCasting(): bool - { - return false; - } - - /** - * Does the adapter handle Query Array Contains? - * - * @return bool - */ - public function getSupportForQueryContains(): bool - { - return false; - } - - public function getSupportForRelationships(): bool - { - return true; - } - - /** - * Get current attribute count from collection document - * - * @param Document $collection - * @return int - */ - public function getCountOfAttributes(Document $collection): int - { - $attributes = \count($collection->getAttribute('attributes') ?? []); - - // +1 ==> virtual columns count as total, so add as buffer - return $attributes + static::getCountOfDefaultAttributes() + 1; - } - - /** - * Get current index count from collection document - * - * @param Document $collection - * @return int - */ - public function getCountOfIndexes(Document $collection): int - { - $indexes = \count($collection->getAttribute('indexes') ?? []); - return $indexes + static::getCountOfDefaultIndexes(); - } - - /** - * Estimate maximum number of bytes required to store a document in $collection. - * Byte requirement varies based on column type and size. - * Needed to satisfy MariaDB/MySQL row width limit. - * - * @param Document $collection - * @return int - */ - public function getAttributeWidth(Document $collection): int - { - // Default collection has: - // `_id` int(11) => 4 bytes - // `_uid` char(255) => 1020 (255 bytes * 4 for utf8mb4) - // but this number seems to vary, so we give a +500 byte buffer - $total = 1500; - - $attributes = $collection->getAttributes()['attributes']; - - foreach ($attributes as $attribute) { - switch ($attribute['type']) { - case Database::VAR_STRING: - switch (true) { - case ($attribute['size'] > 16777215): - // 8 bytes length + 4 bytes for LONGTEXT - $total += 12; - break; - - case ($attribute['size'] > 65535): - // 8 bytes length + 3 bytes for MEDIUMTEXT - $total += 11; - break; - - case ($attribute['size'] > $this->getMaxVarcharLength()): - // 8 bytes length + 2 bytes for TEXT - $total += 10; - break; - - case ($attribute['size'] > 255): - // $size = $size * 4; // utf8mb4 up to 4 bytes per char - // 8 bytes length + 2 bytes for VARCHAR(>255) - $total += ($attribute['size'] * 4) + 2; - break; - - default: - // $size = $size * 4; // utf8mb4 up to 4 bytes per char - // 8 bytes length + 1 bytes for VARCHAR(<=255) - $total += ($attribute['size'] * 4) + 1; - break; - } - break; - - case Database::VAR_INTEGER: - if ($attribute['size'] >= 8) { - $total += 8; // BIGINT takes 8 bytes - } else { - $total += 4; // INT takes 4 bytes - } - break; - case Database::VAR_FLOAT: - // DOUBLE takes 8 bytes - $total += 8; - break; - - case Database::VAR_BOOLEAN: - // TINYINT(1) takes one byte - $total += 1; - break; - - case Database::VAR_RELATIONSHIP: - // INT(11) - $total += 4; - break; - - case Database::VAR_DATETIME: - $total += 19; // 2022-06-26 14:46:24 - break; - default: - throw new DatabaseException('Unknown type: ' . $attribute['type']); - } - } - - return $total; - } - - /** - * @return int - */ - public function getMaxVarcharLength(): int - { - return 16381; // Floor value for Postgres:16383 | MySQL:16381 | MariaDB:16382 - } - - /** - * Get list of keywords that cannot be used - * Refference: https://mariadb.com/kb/en/reserved-words/ - * - * @return array - */ - public function getKeywords(): array - { - return [ - 'ACCESSIBLE', - 'ADD', - 'ALL', - 'ALTER', - 'ANALYZE', - 'AND', - 'AS', - 'ASC', - 'ASENSITIVE', - 'BEFORE', - 'BETWEEN', - 'BIGINT', - 'BINARY', - 'BLOB', - 'BOTH', - 'BY', - 'CALL', - 'CASCADE', - 'CASE', - 'CHANGE', - 'CHAR', - 'CHARACTER', - 'CHECK', - 'COLLATE', - 'COLUMN', - 'CONDITION', - 'CONSTRAINT', - 'CONTINUE', - 'CONVERT', - 'CREATE', - 'CROSS', - 'CURRENT_DATE', - 'CURRENT_ROLE', - 'CURRENT_TIME', - 'CURRENT_TIMESTAMP', - 'CURRENT_USER', - 'CURSOR', - 'DATABASE', - 'DATABASES', - 'DAY_HOUR', - 'DAY_MICROSECOND', - 'DAY_MINUTE', - 'DAY_SECOND', - 'DEC', - 'DECIMAL', - 'DECLARE', - 'DEFAULT', - 'DELAYED', - 'DELETE', - 'DELETE_DOMAIN_ID', - 'DESC', - 'DESCRIBE', - 'DETERMINISTIC', - 'DISTINCT', - 'DISTINCTROW', - 'DIV', - 'DO_DOMAIN_IDS', - 'DOUBLE', - 'DROP', - 'DUAL', - 'EACH', - 'ELSE', - 'ELSEIF', - 'ENCLOSED', - 'ESCAPED', - 'EXCEPT', - 'EXISTS', - 'EXIT', - 'EXPLAIN', - 'FALSE', - 'FETCH', - 'FLOAT', - 'FLOAT4', - 'FLOAT8', - 'FOR', - 'FORCE', - 'FOREIGN', - 'FROM', - 'FULLTEXT', - 'GENERAL', - 'GRANT', - 'GROUP', - 'HAVING', - 'HIGH_PRIORITY', - 'HOUR_MICROSECOND', - 'HOUR_MINUTE', - 'HOUR_SECOND', - 'IF', - 'IGNORE', - 'IGNORE_DOMAIN_IDS', - 'IGNORE_SERVER_IDS', - 'IN', - 'INDEX', - 'INFILE', - 'INNER', - 'INOUT', - 'INSENSITIVE', - 'INSERT', - 'INT', - 'INT1', - 'INT2', - 'INT3', - 'INT4', - 'INT8', - 'INTEGER', - 'INTERSECT', - 'INTERVAL', - 'INTO', - 'IS', - 'ITERATE', - 'JOIN', - 'KEY', - 'KEYS', - 'KILL', - 'LEADING', - 'LEAVE', - 'LEFT', - 'LIKE', - 'LIMIT', - 'LINEAR', - 'LINES', - 'LOAD', - 'LOCALTIME', - 'LOCALTIMESTAMP', - 'LOCK', - 'LONG', - 'LONGBLOB', - 'LONGTEXT', - 'LOOP', - 'LOW_PRIORITY', - 'MASTER_HEARTBEAT_PERIOD', - 'MASTER_SSL_VERIFY_SERVER_CERT', - 'MATCH', - 'MAXVALUE', - 'MEDIUMBLOB', - 'MEDIUMINT', - 'MEDIUMTEXT', - 'MIDDLEINT', - 'MINUTE_MICROSECOND', - 'MINUTE_SECOND', - 'MOD', - 'MODIFIES', - 'NATURAL', - 'NOT', - 'NO_WRITE_TO_BINLOG', - 'NULL', - 'NUMERIC', - 'OFFSET', - 'ON', - 'OPTIMIZE', - 'OPTION', - 'OPTIONALLY', - 'OR', - 'ORDER', - 'OUT', - 'OUTER', - 'OUTFILE', - 'OVER', - 'PAGE_CHECKSUM', - 'PARSE_VCOL_EXPR', - 'PARTITION', - 'POSITION', - 'PRECISION', - 'PRIMARY', - 'PROCEDURE', - 'PURGE', - 'RANGE', - 'READ', - 'READS', - 'READ_WRITE', - 'REAL', - 'RECURSIVE', - 'REF_SYSTEM_ID', - 'REFERENCES', - 'REGEXP', - 'RELEASE', - 'RENAME', - 'REPEAT', - 'REPLACE', - 'REQUIRE', - 'RESIGNAL', - 'RESTRICT', - 'RETURN', - 'RETURNING', - 'REVOKE', - 'RIGHT', - 'RLIKE', - 'ROWS', - 'SCHEMA', - 'SCHEMAS', - 'SECOND_MICROSECOND', - 'SELECT', - 'SENSITIVE', - 'SEPARATOR', - 'SET', - 'SHOW', - 'SIGNAL', - 'SLOW', - 'SMALLINT', - 'SPATIAL', - 'SPECIFIC', - 'SQL', - 'SQLEXCEPTION', - 'SQLSTATE', - 'SQLWARNING', - 'SQL_BIG_RESULT', - 'SQL_CALC_FOUND_ROWS', - 'SQL_SMALL_RESULT', - 'SSL', - 'STARTING', - 'STATS_AUTO_RECALC', - 'STATS_PERSISTENT', - 'STATS_SAMPLE_PAGES', - 'STRAIGHT_JOIN', - 'TABLE', - 'TERMINATED', - 'THEN', - 'TINYBLOB', - 'TINYINT', - 'TINYTEXT', - 'TO', - 'TRAILING', - 'TRIGGER', - 'TRUE', - 'UNDO', - 'UNION', - 'UNIQUE', - 'UNLOCK', - 'UNSIGNED', - 'UPDATE', - 'USAGE', - 'USE', - 'USING', - 'UTC_DATE', - 'UTC_TIME', - 'UTC_TIMESTAMP', - 'VALUES', - 'VARBINARY', - 'VARCHAR', - 'VARCHARACTER', - 'VARYING', - 'WHEN', - 'WHERE', - 'WHILE', - 'WINDOW', - 'WITH', - 'WRITE', - 'XOR', - 'YEAR_MONTH', - 'ZEROFILL', - 'ACTION', - 'BIT', - 'DATE', - 'ENUM', - 'NO', - 'TEXT', - 'TIME', - 'TIMESTAMP', - 'BODY', - 'ELSIF', - 'GOTO', - 'HISTORY', - 'MINUS', - 'OTHERS', - 'PACKAGE', - 'PERIOD', - 'RAISE', - 'ROWNUM', - 'ROWTYPE', - 'SYSDATE', - 'SYSTEM', - 'SYSTEM_TIME', - 'VERSIONING', - 'WITHOUT' - ]; - } } From 56d6d96cb15c67ca630057067dd958c27dbddd18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 22 Feb 2024 18:15:34 +0000 Subject: [PATCH 14/42] Fix test endpoint --- tests/e2e/Adapter/DataAPIMariaDBTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/Adapter/DataAPIMariaDBTest.php b/tests/e2e/Adapter/DataAPIMariaDBTest.php index 6df88b076..80138bdf7 100644 --- a/tests/e2e/Adapter/DataAPIMariaDBTest.php +++ b/tests/e2e/Adapter/DataAPIMariaDBTest.php @@ -32,7 +32,7 @@ public static function getDatabase(): Database $database = new Database(new DataAPIMariaDB('http://data-api/v1', 'test-secret', 'default'), new Cache(new None())); $database->setDatabase('utopiaTests'); - $database->setNamespace('myapp_' . uniqid()); + $database->setNamespace(static::$namespace = 'myapp_' . uniqid()); if ($database->exists('utopiaTests')) { $database->delete('utopiaTests'); From 855bf61d895192c7ace601ab67c8f5bf0a7ef566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 23 Feb 2024 11:16:10 +0000 Subject: [PATCH 15/42] Implement raw query execution --- docker-compose.yml | 36 +- src/Database/Adapter.php | 9 + src/Database/Adapter/DataAPI.php | 668 +---------------------- src/Database/Adapter/DataAPIMariaDB.php | 42 +- src/Database/Adapter/Mongo.php | 14 + src/Database/Adapter/SQL.php | 19 +- tests/e2e/Adapter/DataAPIMariaDBTest.php | 2 +- 7 files changed, 89 insertions(+), 701 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5977daa8e..646b969ed 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,25 +82,25 @@ services: networks: - database - data-api: - image: utopia-php/data-api:0.1.0 - container_name: utopia-data-api - networks: - - database - environment: - - UTOPIA_DATA_API_SECRET=test-secret - - UTOPIA_DATA_API_SECRET_CONNECTION=mariadb://root:password@data-api-mariadb:3306/appwrite + # data-api: + # image: utopia-php/data-api:0.1.0 + # container_name: utopia-data-api + # networks: + # - database + # environment: + # - UTOPIA_DATA_API_SECRET=test-secret + # - UTOPIA_DATA_API_SECRET_CONNECTION=mariadb://root:password@data-api-mariadb:3306/appwrite - data-api-mariadb: - image: mariadb:10.7 - container_name: utopia-data-api-mariadb - networks: - - database - environment: - - MYSQL_ROOT_PASSWORD=password - - MARIADB_DATABASE=appwrite - ports: - - "8770:3306" + # data-api-mariadb: + # image: mariadb:10.7 + # container_name: utopia-data-api-mariadb + # networks: + # - database + # environment: + # - MYSQL_ROOT_PASSWORD=password + # - MARIADB_DATABASE=appwrite + # ports: + # - "8770:3306" networks: database: diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 5ee9bdd0e..527f350e6 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -272,6 +272,15 @@ protected function trigger(string $event, mixed $query): mixed */ abstract public function ping(): bool; + /** + * Execute raw command + * @param string $command + * + * @return mixed + * @throws \Throwable + */ + abstract public function execute(string $command): mixed; + /** * Create Database * diff --git a/src/Database/Adapter/DataAPI.php b/src/Database/Adapter/DataAPI.php index a39482fbd..faba187dc 100644 --- a/src/Database/Adapter/DataAPI.php +++ b/src/Database/Adapter/DataAPI.php @@ -4,63 +4,29 @@ use Exception; use Throwable; -use Utopia\Database\Adapter; -use Utopia\Database\Database; -use Utopia\Database\Document; use Utopia\Fetch\Client; -use Utopia\Database\Query; use Utopia\Database\Exception as DatabaseException; -use Utopia\Database\Validator\Authorization; use Utopia\Fetch\FetchException; -abstract class DataAPI extends Adapter +trait DataAPI { - protected string $endpoint; - protected string $secret; - protected string $database; - - /** - * Constructor. - * - * Set connection and settings - * - * @param string $endpoint - * @param string $secret - */ - public function __construct(string $endpoint, string $secret, string $database) - { - $this->endpoint = $endpoint; - $this->secret = $secret; - $this->database = $database; - } - /** - * @param mixed[] $params - * * @throws FetchException * @throws DatabaseException * @throws Exception */ - private function query(string $query, array $params = []): mixed + private function query(string $endpoint, string $secret, string $database, string $command): mixed { - $roles = \implode(',', Authorization::getRoles()); $response = Client::fetch( - url: $this->endpoint . '/queries', + url: $endpoint . '/queries', headers: [ - 'x-utopia-secret' => $this->secret, - 'x-utopia-database' => $this->database, - 'x-utopia-namespace' => $this->getNamespace(), - 'x-utopia-auth-roles' => $roles, - 'x-utopia-auth-status' => Authorization::$status ? 'true' : 'false', - 'x-utopia-auth-status-default' => Authorization::$statusDefault ? 'true' : 'false', - // TODO: Fix timeout - // 'x-utopia-timeout' => self::$timeout ? \strval(self::$timeout) : '', + 'x-utopia-secret' => $secret, + 'x-utopia-database' => $database, 'content-type' => 'application/json' ], method: 'POST', body: [ - 'query' => $query, - 'params' => $params + 'command' => $command, ] ); @@ -86,626 +52,6 @@ private function query(string $query, array $params = []): mixed $body = \json_decode($response->getBody(), false); - $output = $body->output ?? ''; - - if(\is_object($output)) { - $output = new Document((array) $output); - } - - return $output; - } - - /** - * Ping Database - * - * @return bool - */ - public function ping(): bool - { - return $this->query('ping'); - } - - /** - * Create Database - * - * @param string $name - * - * @return bool - */ - public function create(string $name): bool - { - return $this->query('create', [$name]); - } - - /** - * Check if database exists - * Optionally check if collection exists in database - * - * @param string $database database name - * @param string|null $collection (optional) collection name - * - * @return bool - */ - public function exists(string $database, ?string $collection = null): bool - { - return $this->query('exists', [$database, $collection]); - } - - /** - * List Databases - * - * @return array - */ - public function list(): array - { - return $this->query('list'); - } - - /** - * Delete Database - * - * @param string $name - * - * @return bool - */ - public function delete(string $name): bool - { - return $this->query('delete', [$name]); - } - - /** - * Create Collection - * - * @param string $name - * @param array $attributes (optional) - * @param array $indexes (optional) - * @return bool - */ - public function createCollection(string $name, array $attributes = [], array $indexes = []): bool - { - return $this->query('createCollection', [$name, $attributes, $indexes]); - } - - /** - * Delete Collection - * - * @param string $id - * - * @return bool - */ - public function deleteCollection(string $id): bool - { - return $this->query('deleteCollection', [$id]); - } - - /** - * Create Attribute - * - * @param string $collection - * @param string $id - * @param string $type - * @param int $size - * @param bool $signed - * @param bool $array - * @return bool - */ - public function createAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false): bool - { - return $this->query('createAttribute', [$collection, $id, $type, $size, $signed, $array]); - } - - /** - * Update Attribute - * - * @param string $collection - * @param string $id - * @param string $type - * @param int $size - * @param bool $signed - * @param bool $array - * - * @return bool - */ - public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false): bool - { - return $this->query('updateAttribute', [$collection, $id, $type, $size, $signed, $array]); - } - - /** - * Delete Attribute - * - * @param string $collection - * @param string $id - * - * @return bool - */ - public function deleteAttribute(string $collection, string $id): bool - { - return $this->query('deleteAttribute', [$collection, $id]); - } - - /** - * Rename Attribute - * - * @param string $collection - * @param string $old - * @param string $new - * @return bool - */ - public function renameAttribute(string $collection, string $old, string $new): bool - { - return $this->query('renameAttribute', [$collection, $old, $new]); - } - - /** - * @param string $collection - * @param string $relatedCollection - * @param string $type - * @param bool $twoWay - * @param string $id - * @param string $twoWayKey - * @return bool - */ - public function createRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay = false, string $id = '', string $twoWayKey = ''): bool - { - return $this->query('createRelationship', [$collection, $relatedCollection, $type, $twoWay, $id, $twoWayKey]); - } - - /** - * Update Relationship - * - * @param string $collection - * @param string $relatedCollection - * @param string $type - * @param bool $twoWay - * @param string $key - * @param string $twoWayKey - * @param string|null $newKey - * @param string|null $newTwoWayKey - * @return bool - */ - public function updateRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay, string $key, string $twoWayKey, ?string $newKey = null, ?string $newTwoWayKey = null): bool - { - return $this->query('updateRelationship', [$collection, $relatedCollection, $type, $twoWay, $key, $twoWayKey, $newKey, $newTwoWayKey]); - } - - /** - * Delete Relationship - * - * @param string $collection - * @param string $relatedCollection - * @param string $type - * @param bool $twoWay - * @param string $key - * @param string $twoWayKey - * @param string $side - * @return bool - */ - public function deleteRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay, string $key, string $twoWayKey, string $side): bool - { - return $this->query('deleteRelationship', [$collection, $relatedCollection, $type, $twoWay, $key, $twoWayKey, $side]); - } - - /** - * Rename Index - * - * @param string $collection - * @param string $old - * @param string $new - * @return bool - */ - public function renameIndex(string $collection, string $old, string $new): bool - { - return $this->query('renameIndex', [$collection, $old, $new]); - } - - /** - * Create Index - * - * @param string $collection - * @param string $id - * @param string $type - * @param array $attributes - * @param array $lengths - * @param array $orders - * - * @return bool - */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders): bool - { - return $this->query('createIndex', [$collection, $id, $type, $attributes, $lengths, $orders]); - } - - /** - * Delete Index - * - * @param string $collection - * @param string $id - * - * @return bool - */ - public function deleteIndex(string $collection, string $id): bool - { - return $this->query('deleteIndex', [$collection, $id]); - } - - /** - * Get Document - * - * @param string $collection - * @param string $id - * @param array $queries - * @return Document - */ - public function getDocument(string $collection, string $id, array $queries = []): Document - { - return $this->query('getDocument', [$collection, $id, $queries]); - } - - /** - * Create Document - * - * @param string $collection - * @param Document $document - * - * @return Document - */ - public function createDocument(string $collection, Document $document): Document - { - return $this->query('createDocument', [$collection, $document]); - } - - /** - * Create Documents in batches - * - * @param string $collection - * @param array $documents - * @param int $batchSize - * - * @return array - * - * @throws DatabaseException - */ - public function createDocuments(string $collection, array $documents, int $batchSize): array - { - return $this->query('createDocuments', [$collection, $documents, $batchSize]); - } - - /** - * Update Document - * - * @param string $collection - * @param Document $document - * - * @return Document - */ - public function updateDocument(string $collection, Document $document): Document - { - return $this->query('updateDocument', [$collection, $document]); - } - - /** - * Update Documents in batches - * - * @param string $collection - * @param array $documents - * @param int $batchSize - * - * @return array - * - * @throws DatabaseException - */ - public function updateDocuments(string $collection, array $documents, int $batchSize): array - { - return $this->query('updateDocuments', [$collection, $documents, $batchSize]); - } - - /** - * Delete Document - * - * @param string $collection - * @param string $id - * - * @return bool - */ - public function deleteDocument(string $collection, string $id): bool - { - return $this->query('deleteDocument', [$collection, $id]); - } - - /** - * Find Documents - * - * Find data sets using chosen queries - * - * @param string $collection - * @param array $queries - * @param int|null $limit - * @param int|null $offset - * @param array $orderAttributes - * @param array $orderTypes - * @param array $cursor - * @param string $cursorDirection - * - * @return array - */ - public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER): array - { - return $this->query('find', [$collection, $queries, $limit, $offset, $orderAttributes, $orderTypes, $cursor, $cursorDirection]); - } - - /** - * Sum an attribute - * - * @param string $collection - * @param string $attribute - * @param array $queries - * @param int|null $max - * - * @return int|float - */ - public function sum(string $collection, string $attribute, array $queries = [], ?int $max = null): float|int - { - return $this->query('sum', [$collection, $attribute, $queries, $max]); - } - - /** - * Count Documents - * - * @param string $collection - * @param array $queries - * @param int|null $max - * - * @return int - */ - public function count(string $collection, array $queries = [], ?int $max = null): int - { - return $this->query('count', [$collection, $queries, $max]); - } - - /** - * Get Collection Size - * - * @param string $collection - * @return int - * @throws DatabaseException - */ - public function getSizeOfCollection(string $collection): int - { - return $this->query('getSizeOfCollection', [$collection]); - } - - /** - * Get max STRING limit - * - * @return int - */ - public function getLimitForString(): int - { - return $this->query('getLimitForString'); - } - - /** - * Get max INT limit - * - * @return int - */ - public function getLimitForInt(): int - { - return $this->query('getLimitForInt'); - } - - /** - * Get maximum attributes limit. - * - * @return int - */ - public function getLimitForAttributes(): int - { - return $this->query('getLimitForAttributes'); - } - - /** - * Get maximum index limit. - * - * @return int - */ - public function getLimitForIndexes(): int - { - return $this->query('getLimitForIndexes'); - } - - /** - * Is schemas supported? - * - * @return bool - */ - public function getSupportForSchemas(): bool - { - return $this->query('getSupportForSchemas'); - } - - /** - * Is index supported? - * - * @return bool - */ - public function getSupportForIndex(): bool - { - return $this->query('getSupportForIndex'); - } - - /** - * Is unique index supported? - * - * @return bool - */ - public function getSupportForUniqueIndex(): bool - { - return $this->query('getSupportForUniqueIndex'); - } - - /** - * Is fulltext index supported? - * - * @return bool - */ - public function getSupportForFulltextIndex(): bool - { - return $this->query('getSupportForFulltextIndex'); - } - - /** - * Is fulltext wildcard supported? - * - * @return bool - */ - public function getSupportForFulltextWildcardIndex(): bool - { - return $this->query('getSupportForFulltextWildcardIndex'); - } - - - /** - * Does the adapter handle casting? - * - * @return bool - */ - public function getSupportForCasting(): bool - { - return $this->query('getSupportForCasting'); - } - - /** - * Does the adapter handle array Contains? - * - * @return bool - */ - public function getSupportForQueryContains(): bool - { - return $this->query('getSupportForQueryContains'); - } - - /** - * Are timeouts supported? - * - * @return bool - */ - public function getSupportForTimeouts(): bool - { - return $this->query('getSupportForTimeouts'); - } - - /** - * Are relationships supported? - * - * @return bool - */ - public function getSupportForRelationships(): bool - { - return $this->query('getSupportForRelationships'); - } - - /** - * Get current attribute count from collection document - * - * @param Document $collection - * @return int - */ - public function getCountOfAttributes(Document $collection): int - { - return $this->query('getCountOfAttributes', [$collection]); - } - - /** - * Get current index count from collection document - * - * @param Document $collection - * @return int - */ - public function getCountOfIndexes(Document $collection): int - { - return $this->query('getCountOfIndexes', [$collection]); - } - - /** - * Estimate maximum number of bytes required to store a document in $collection. - * Byte requirement varies based on column type and size. - * Needed to satisfy MariaDB/MySQL row width limit. - * Return 0 when no restrictions apply to row width - * - * @param Document $collection - * @return int - */ - public function getAttributeWidth(Document $collection): int - { - return $this->query('getAttributeWidth', [$collection]); - } - - /** - * Get list of keywords that cannot be used - * - * @return array - */ - public function getKeywords(): array - { - return $this->query('getKeywords'); - } - - /** - * Get an attribute projection given a list of selected attributes - * - * @param array $selections - * @param string $prefix - * @return mixed - */ - protected function getAttributeProjection(array $selections, string $prefix = ''): mixed - { - return $this->query('getAttributeProjection', [$selections, $prefix]); - } - - /** - * Increase and Decrease Attribute Value - * - * @param string $collection - * @param string $id - * @param string $attribute - * @param int|float $value - * @param int|float|null $min - * @param int|float|null $max - * @return bool - * @throws Exception - */ - public function increaseDocumentAttribute(string $collection, string $id, string $attribute, int|float $value, int|float|null $min = null, int|float|null $max = null): bool - { - return $this->query('increaseDocumentAttribute', [$collection, $id, $attribute, $value, $min, $max]); - } - - /** - * @return int - */ - public function getMaxIndexLength(): int - { - return $this->query('getMaxIndexLength'); - } - - /** - * Set a global timeout for database queries in milliseconds. - * - * This function allows you to set a maximum execution time for all database - * queries executed using the library, or a specific event specified by the - * event parameter. Once this timeout is set, any database query that takes - * longer than the specified time will be automatically terminated by the library, - * and an appropriate error or exception will be raised to handle the timeout condition. - * - * @param int $milliseconds The timeout value in milliseconds for database queries. - * @param string $event The event the timeout should fire fore - * @return void - * - * @throws Exception The provided timeout value must be greater than or equal to 0. - */ - public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): void - { - $this->query('setTimeout', [$milliseconds, $event]); + return $body->output ?? ''; } } diff --git a/src/Database/Adapter/DataAPIMariaDB.php b/src/Database/Adapter/DataAPIMariaDB.php index 0cd520907..530bdd2c3 100644 --- a/src/Database/Adapter/DataAPIMariaDB.php +++ b/src/Database/Adapter/DataAPIMariaDB.php @@ -2,38 +2,40 @@ namespace Utopia\Database\Adapter; -use Utopia\Database\Database; +use PDOException; -class DataAPIMariaDB extends DataAPI +class DataAPIMariaDB extends MariaDB { - /** - * Returns number of attributes used by default. - * - * @return int - */ - public static function getCountOfDefaultAttributes(): int - { - return \count(Database::INTERNAL_ATTRIBUTES); - } + use DataAPI; + + protected string $endpoint; + protected string $secret; + protected string $database; /** - * Returns number of indexes used by default. + * Constructor. + * + * Set connection and settings * - * @return int + * @param string $endpoint + * @param string $secret */ - public static function getCountOfDefaultIndexes(): int + public function __construct(string $endpoint, string $secret, string $database) { - return \count(Database::INTERNAL_INDEXES); + $this->endpoint = $endpoint; + $this->secret = $secret; + $this->database = $database; } /** - * Get maximum width, in bytes, allowed for a SQL row - * Return 0 when no restrictions apply + * Execute raw command + * @param mixed $query * - * @return int + * @return mixed + * @throws \Throwable */ - public static function getDocumentSizeLimit(): int + public function execute(mixed $query): mixed { - return 65535; + return $this->query($this->endpoint, $this->secret, $this->database, $query); } } diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index f79a5a14a..bc7433288 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -70,6 +70,20 @@ public function ping(): bool return $this->getClient()->query(['ping' => 1])->ok ?? false; } + /** + * Execute raw command + * @param mixed $query + * + * @return mixed + * @throws Exception + * @throws MongoException + */ + public function execute(mixed $query): mixed + { + // Not needed until we need Mongo Data API adapter + throw new Exception('Not implemented'); + } + /** * Create Database * diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 00b6d5832..32c658612 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -36,8 +36,25 @@ public function __construct(mixed $pdo) */ public function ping(): bool { - return $this->getPDO() + $query = $this->getPDO() ->prepare("SELECT 1;") + ->queryString; + + return $this->execute($query); + } + + /** + * Execute raw command + * @param mixed $query + * + * @return mixed + * @throws Exception + * @throws PDOException + */ + public function execute(mixed $query): mixed + { + return $this->getPDO() + ->prepare($query) ->execute(); } diff --git a/tests/e2e/Adapter/DataAPIMariaDBTest.php b/tests/e2e/Adapter/DataAPIMariaDBTest.php index 80138bdf7..471297743 100644 --- a/tests/e2e/Adapter/DataAPIMariaDBTest.php +++ b/tests/e2e/Adapter/DataAPIMariaDBTest.php @@ -30,7 +30,7 @@ public static function getDatabase(): Database return self::$database; } - $database = new Database(new DataAPIMariaDB('http://data-api/v1', 'test-secret', 'default'), new Cache(new None())); + $database = new Database(new DataAPIMariaDB('https://8088-utopiaphp-dataapi-1zrdiri2xh5.ws-eu108.gitpod.io/v1', 'test-secret', 'default'), new Cache(new None())); $database->setDatabase('utopiaTests'); $database->setNamespace(static::$namespace = 'myapp_' . uniqid()); From 6c1261dfdffb241235f4ef880abe8f756af8e89b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 23 Feb 2024 11:18:47 +0000 Subject: [PATCH 16/42] Update lockfile --- composer.lock | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/composer.lock b/composer.lock index 7652b7212..4f853a20f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1fec834c5b222e402702b7bc89a5a8a8", + "content-hash": "4adf56005e024ee6b5bb991e96e2ee72", "packages": [ { "name": "jean85/pretty-package-versions", @@ -264,6 +264,45 @@ }, "time": "2024-01-07T18:11:23+00:00" }, + { + "name": "utopia-php/fetch", + "version": "0.1.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/fetch.git", + "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/2fa214b9262acd1a3583515a364da4f35929d5c5", + "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5", + "shasum": "" + }, + "require": { + "php": ">=8.0" + }, + "require-dev": { + "laravel/pint": "^1.5.0", + "phpstan/phpstan": "^1.10", + "phpunit/phpunit": "^9.5" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Fetch\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A simple library that provides an interface for making HTTP Requests.", + "support": { + "issues": "https://github.com/utopia-php/fetch/issues", + "source": "https://github.com/utopia-php/fetch/tree/0.1.0" + }, + "time": "2023-10-10T11:58:32+00:00" + }, { "name": "utopia-php/framework", "version": "0.33.2", From 2c2c61e405f3a6496dd23af893a26d2d4215332e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 23 Feb 2024 11:24:13 +0000 Subject: [PATCH 17/42] Linter fixes --- src/Database/Adapter/DataAPIMariaDB.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Database/Adapter/DataAPIMariaDB.php b/src/Database/Adapter/DataAPIMariaDB.php index 530bdd2c3..5872262d7 100644 --- a/src/Database/Adapter/DataAPIMariaDB.php +++ b/src/Database/Adapter/DataAPIMariaDB.php @@ -2,8 +2,6 @@ namespace Utopia\Database\Adapter; -use PDOException; - class DataAPIMariaDB extends MariaDB { use DataAPI; From 9d44651ee3e0493f5f7b38811cc9e8ad1f3b8807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 23 Feb 2024 11:56:54 +0000 Subject: [PATCH 18/42] Temporarly remove data API tests --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 631278763..d89bff6b1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -77,7 +77,7 @@ jobs: Postgres, SQLite, MongoDB, - DataAPIMariaDB + # DataAPIMariaDB ] steps: From dacb169e94303b1567cbc6fd00c7612b2111e715 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 23 Feb 2024 17:40:25 +0000 Subject: [PATCH 19/42] WIP: Update to raw queries --- phpunit.xml | 2 +- src/Database/Adapter.php | 15 ++++++++-- src/Database/Adapter/DataAPIMariaDB.php | 17 ++++++++++-- src/Database/Adapter/Mongo.php | 20 ++++++++++--- src/Database/Adapter/SQL.php | 37 ++++++++++++++++++------- 5 files changed, 71 insertions(+), 20 deletions(-) diff --git a/phpunit.xml b/phpunit.xml index ccdaa969e..783265d80 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false"> + stopOnFailure="true"> ./tests/unit diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 527f350e6..96dd8a273 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -273,13 +273,22 @@ protected function trigger(string $event, mixed $query): mixed abstract public function ping(): bool; /** - * Execute raw command - * @param string $command + * Execute raw command with no response + * @param mixed $query * * @return mixed * @throws \Throwable */ - abstract public function execute(string $command): mixed; + abstract public function executeWrite(mixed $query): bool; + + /** + * Execute raw command that returns a response + * @param mixed $query + * + * @return mixed + * @throws \Throwable + */ + abstract public function executeRead(mixed $query): mixed; /** * Create Database diff --git a/src/Database/Adapter/DataAPIMariaDB.php b/src/Database/Adapter/DataAPIMariaDB.php index 5872262d7..53c10600c 100644 --- a/src/Database/Adapter/DataAPIMariaDB.php +++ b/src/Database/Adapter/DataAPIMariaDB.php @@ -26,13 +26,26 @@ public function __construct(string $endpoint, string $secret, string $database) } /** - * Execute raw command + * Execute raw command with no response * @param mixed $query * * @return mixed * @throws \Throwable */ - public function execute(mixed $query): mixed + public function executeWrite(mixed $query): bool + { + $this->query($this->endpoint, $this->secret, $this->database, $query); + return true; + } + + /** + * Execute raw command that returns a response + * @param mixed $query + * + * @return mixed + * @throws \Throwable + */ + public function executeRead(mixed $query): mixed { return $this->query($this->endpoint, $this->secret, $this->database, $query); } diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 13bb38ca3..7693e8e3b 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -70,15 +70,27 @@ public function ping(): bool return $this->getClient()->query(['ping' => 1])->ok ?? false; } + /** + * Execute raw command with no response + * @param mixed $query + * + * @return mixed + * @throws \Throwable + */ + public function executeWrite(mixed $query): bool + { + // Not needed until we need Mongo Data API adapter + throw new Exception('Not implemented'); + } + /** - * Execute raw command + * Execute raw command that returns a response * @param mixed $query * * @return mixed - * @throws Exception - * @throws MongoException + * @throws \Throwable */ - public function execute(mixed $query): mixed + public function executeRead(mixed $query): mixed { // Not needed until we need Mongo Data API adapter throw new Exception('Not implemented'); diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 32c658612..00189f979 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -36,28 +36,46 @@ public function __construct(mixed $pdo) */ public function ping(): bool { - $query = $this->getPDO() - ->prepare("SELECT 1;") - ->queryString; + $stmt = $this->getPDO()->prepare("SELECT 1;"); + $query = $stmt->queryString; - return $this->execute($query); + return $this->executeWrite($query); } /** - * Execute raw command + * Execute raw command with no response * @param mixed $query * * @return mixed - * @throws Exception - * @throws PDOException + * @throws \Throwable */ - public function execute(mixed $query): mixed + public function executeWrite(mixed $query): bool { return $this->getPDO() ->prepare($query) ->execute(); } + /** + * Execute raw command that returns a response + * @param mixed $query + * + * @return mixed + * @throws \Throwable + */ + public function executeRead(mixed $query): mixed + { + $stmt = $this->getPDO()->prepare($query); + $response = $stmt->fetchAll(); + + \var_dump($query); + \var_dump($response); + + $stmt->closeCursor(); + + return $response; + } + /** * Check if Database exists * Optionally check if collection exists in Database @@ -92,8 +110,7 @@ public function exists(string $database, ?string $collection = null): bool $stmt->execute(); - $document = $stmt->fetchAll(); - $stmt->closeCursor(); + $document = $this->executeRead($stmt->queryString); if (empty($document)) { return false; From f33fcdd209ea512f33662e59780989941d691d28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Sat, 24 Feb 2024 13:57:37 +0000 Subject: [PATCH 20/42] WIP: Convert MariaDB to raw query methods --- README.md | 8 +- composer.lock | 12 +-- src/Database/Adapter.php | 28 ++++++- src/Database/Adapter/DataAPI.php | 5 +- src/Database/Adapter/DataAPIMariaDB.php | 26 +++++-- src/Database/Adapter/MariaDB.php | 97 +++++++++++-------------- src/Database/Adapter/Mongo.php | 22 +++++- src/Database/Adapter/SQL.php | 96 +++++++++++++++++------- src/Database/Extend/PDOStatement.php | 56 ++++++++++++++ 9 files changed, 249 insertions(+), 101 deletions(-) create mode 100644 src/Database/Extend/PDOStatement.php diff --git a/README.md b/README.md index 1adf58143..17969da57 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ $dbUser = 'root'; $dbPass = 'password'; $pdoConfig = [ PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, + PDO::ATTR_PERSISTENT => false, PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_EMULATE_PREPARES => true, @@ -134,7 +134,7 @@ $dbUser = 'root'; $dbPass = 'password'; $pdoConfig = [ PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, + PDO::ATTR_PERSISTENT => false, PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_EMULATE_PREPARES => true, @@ -165,7 +165,7 @@ $dbUser = 'root'; $dbPass = 'password'; $pdoConfig = [ PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, + PDO::ATTR_PERSISTENT => false, PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_EMULATE_PREPARES => true, @@ -193,7 +193,7 @@ use Utopia\Database\Adapter\SQLite; $dbPath = '/path/to/database.sqlite'; $pdoConfig = [ PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => true, + PDO::ATTR_PERSISTENT => false, PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_EMULATE_PREPARES => true, diff --git a/composer.lock b/composer.lock index 4f853a20f..4b582659b 100644 --- a/composer.lock +++ b/composer.lock @@ -1252,16 +1252,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.6.16", + "version": "9.6.17", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "3767b2c56ce02d01e3491046f33466a1ae60a37f" + "reference": "1a156980d78a6666721b7e8e8502fe210b587fcd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/3767b2c56ce02d01e3491046f33466a1ae60a37f", - "reference": "3767b2c56ce02d01e3491046f33466a1ae60a37f", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/1a156980d78a6666721b7e8e8502fe210b587fcd", + "reference": "1a156980d78a6666721b7e8e8502fe210b587fcd", "shasum": "" }, "require": { @@ -1335,7 +1335,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.16" + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.17" }, "funding": [ { @@ -1351,7 +1351,7 @@ "type": "tidelift" } ], - "time": "2024-01-19T07:03:14+00:00" + "time": "2024-02-23T13:14:51+00:00" }, { "name": "psr/container", diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 96dd8a273..a5b5f51c3 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -275,20 +275,42 @@ abstract public function ping(): bool; /** * Execute raw command with no response * @param mixed $query + * @param array[string]mixed $params * - * @return mixed + * @return bool + * @throws \Throwable + */ + abstract public function executeWrite(mixed $query, array $params): bool; + + /** + * Execute raw command and get amount of affected documents + * @param mixed $query + * @param array[string]mixed $params + * + * @return int * @throws \Throwable */ - abstract public function executeWrite(mixed $query): bool; + abstract public function executeWriteWithCount(mixed $query, array $params): int; /** * Execute raw command that returns a response * @param mixed $query + * @param array[string]mixed $params + * + * @return mixed + * @throws \Throwable + */ + abstract public function executeRead(mixed $query, array $params): mixed; + + /** + * Execute transaction of multiple raw queries + * @param mixed $query + * @param array[string]mixed $params * * @return mixed * @throws \Throwable */ - abstract public function executeRead(mixed $query): mixed; + abstract public function executeRead(mixed $query, array $params): mixed; /** * Create Database diff --git a/src/Database/Adapter/DataAPI.php b/src/Database/Adapter/DataAPI.php index faba187dc..038100ce8 100644 --- a/src/Database/Adapter/DataAPI.php +++ b/src/Database/Adapter/DataAPI.php @@ -11,11 +11,13 @@ trait DataAPI { /** + * @param array[string]mixed $params + * * @throws FetchException * @throws DatabaseException * @throws Exception */ - private function query(string $endpoint, string $secret, string $database, string $command): mixed + private function query(string $endpoint, string $secret, string $database, string $command, array $params): mixed { $response = Client::fetch( url: $endpoint . '/queries', @@ -27,6 +29,7 @@ private function query(string $endpoint, string $secret, string $database, strin method: 'POST', body: [ 'command' => $command, + 'parrams' => $params ] ); diff --git a/src/Database/Adapter/DataAPIMariaDB.php b/src/Database/Adapter/DataAPIMariaDB.php index 53c10600c..28ec61f78 100644 --- a/src/Database/Adapter/DataAPIMariaDB.php +++ b/src/Database/Adapter/DataAPIMariaDB.php @@ -28,25 +28,41 @@ public function __construct(string $endpoint, string $secret, string $database) /** * Execute raw command with no response * @param mixed $query + * @param array[string]mixed $params * - * @return mixed + * @return bool * @throws \Throwable */ - public function executeWrite(mixed $query): bool + public function executeWrite(mixed $query, array $params): bool { - $this->query($this->endpoint, $this->secret, $this->database, $query); + $this->query($this->endpoint, $this->secret, $this->database, $query, $params); return true; } + /** + * Execute raw command and get amount of affected documents + * @param mixed $query + * @param array[string]mixed $params + * + * @return int + * @throws \Throwable + */ + public function executeWriteWithCount(mixed $query, array $params): int + { + $this->query($this->endpoint, $this->secret, $this->database, $query, $params); + return 1; + } + /** * Execute raw command that returns a response * @param mixed $query + * @param array[string]mixed $params * * @return mixed * @throws \Throwable */ - public function executeRead(mixed $query): mixed + public function executeRead(mixed $query, array $params): mixed { - return $this->query($this->endpoint, $this->secret, $this->database, $query); + return $this->query($this->endpoint, $this->secret, $this->database, $query, $params); } } diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 297796b49..5f1287bfc 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -10,6 +10,7 @@ use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\Timeout as TimeoutException; +use Utopia\Database\Extend\PDOStatement; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; @@ -35,9 +36,8 @@ public function create(string $name): bool $sql = $this->trigger(Database::EVENT_DATABASE_CREATE, $sql); - return $this->getPDO() - ->prepare($sql) - ->execute(); + $stmt = $this->getPDO()->prepare($sql); + return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); } /** @@ -56,9 +56,8 @@ public function delete(string $name): bool $sql = $this->trigger(Database::EVENT_DATABASE_DELETE, $sql); - return $this->getPDO() - ->prepare($sql) - ->execute(); + $stmt = $this->getPDO()->prepare($sql); + return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); } /** @@ -148,9 +147,8 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql); try { - $this->getPDO() - ->prepare($sql) - ->execute(); + $stmt = $this->getPDO()->prepare($sql); + $this->executeWrite($stmt->getQuery(), $stmt->getParams()); $sql = " CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id . '_perms')} ( @@ -178,13 +176,11 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql); - $this->getPDO() - ->prepare($sql) - ->execute(); + $stmt = $this->getPDO()->prepare($sql); + $this->executeWrite($stmt->getQuery(), $stmt->getParams()); } catch (\Exception $th) { - $this->getPDO() - ->prepare("DROP TABLE IF EXISTS {$this->getSQLTable($id)}, {$this->getSQLTable($id . '_perms')};") - ->execute(); + $stmt = $this->getPDO()->prepare("DROP TABLE IF EXISTS {$this->getSQLTable($id)}, {$this->getSQLTable($id . '_perms')};"); + $this->executeWrite($stmt->getQuery(), $stmt->getParams()); throw $th; } @@ -207,13 +203,13 @@ public function getSizeOfCollection(string $collection): int $permissions = $database . '/' . $collection . '_perms'; $collectionSize = $this->getPDO()->prepare(" - SELECT SUM(FS_BLOCK_SIZE + ALLOCATED_SIZE) + SELECT SUM(FS_BLOCK_SIZE + ALLOCATED_SIZE) AS total FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES WHERE NAME = :name "); - $permissionsSize = $this->getPDO()->prepare(" - SELECT SUM(FS_BLOCK_SIZE + ALLOCATED_SIZE) + $permissionsSize = $this->getPDO()->prepare(" + SELECT SUM(FS_BLOCK_SIZE + ALLOCATED_SIZE) AS total FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES WHERE NAME = :permissions "); @@ -222,9 +218,9 @@ public function getSizeOfCollection(string $collection): int $permissionsSize->bindParam(':permissions', $permissions); try { - $collectionSize->execute(); - $permissionsSize->execute(); - $size = $collectionSize->fetchColumn() + $permissionsSize->fetchColumn(); + $resultCollection = $this->executeRead($collectionSize->getQuery(), $collectionSize->getParams()); + $resultPermissions = $this->executeRead($permissionsSize->getQuery(), $permissionsSize->getParams()); + $size = $resultCollection[0]["total"] + $resultPermissions[0]["total"]; } catch (PDOException $e) { throw new DatabaseException('Failed to get collection size: ' . $e->getMessage()); } @@ -248,9 +244,8 @@ public function deleteCollection(string $id): bool $sql = $this->trigger(Database::EVENT_COLLECTION_DELETE, $sql); - return $this->getPDO() - ->prepare($sql) - ->execute(); + $stmt = $this->getPDO()->prepare($sql); + return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); } /** @@ -275,9 +270,9 @@ public function createAttribute(string $collection, string $id, string $type, in $sql = "ALTER TABLE {$this->getSQLTable($name)} ADD COLUMN `{$id}` {$type};"; $sql = $this->trigger(Database::EVENT_ATTRIBUTE_CREATE, $sql); - return $this->getPDO() - ->prepare($sql) - ->execute(); + + $stmt = $this->getPDO()->prepare($sql); + return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); } /** @@ -303,9 +298,8 @@ public function updateAttribute(string $collection, string $id, string $type, in $sql = $this->trigger(Database::EVENT_ATTRIBUTE_UPDATE, $sql); - return $this->getPDO() - ->prepare($sql) - ->execute(); + $stmt = $this->getPDO()->prepare($sql); + return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); } /** @@ -327,9 +321,8 @@ public function deleteAttribute(string $collection, string $id, bool $array = fa $sql = $this->trigger(Database::EVENT_ATTRIBUTE_DELETE, $sql); - return $this->getPDO() - ->prepare($sql) - ->execute(); + $stmt = $this->getPDO()->prepare($sql); + return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); } /** @@ -352,9 +345,8 @@ public function renameAttribute(string $collection, string $old, string $new): b $sql = $this->trigger(Database::EVENT_ATTRIBUTE_UPDATE, $sql); - return $this->getPDO() - ->prepare($sql) - ->execute(); + $stmt = $this->getPDO()->prepare($sql); + return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); } /** @@ -405,9 +397,8 @@ public function createRelationship( $sql = $this->trigger(Database::EVENT_ATTRIBUTE_CREATE, $sql); - return $this->getPDO() - ->prepare($sql) - ->execute(); + $stmt = $this->getPDO()->prepare($sql); + return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); } /** @@ -490,9 +481,8 @@ public function updateRelationship( $sql = $this->trigger(Database::EVENT_ATTRIBUTE_UPDATE, $sql); - return $this->getPDO() - ->prepare($sql) - ->execute(); + $stmt = $this->getPDO()->prepare($sql); + return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); } /** @@ -576,9 +566,8 @@ public function deleteRelationship( $sql = $this->trigger(Database::EVENT_ATTRIBUTE_DELETE, $sql); - return $this->getPDO() - ->prepare($sql) - ->execute(); + $stmt = $this->getPDO()->prepare($sql); + return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); } /** @@ -600,9 +589,8 @@ public function renameIndex(string $collection, string $old, string $new): bool $sql = $this->trigger(Database::EVENT_INDEX_RENAME, $sql); - return $this->getPDO() - ->prepare($sql) - ->execute(); + $stmt = $this->getPDO()->prepare($sql); + return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); } /** @@ -666,9 +654,8 @@ public function createIndex(string $collection, string $id, string $type, array $sql = "CREATE {$sqlType} `{$id}` ON {$this->getSQLTable($collection->getId())} ({$attributes})"; $sql = $this->trigger(Database::EVENT_INDEX_CREATE, $sql); - return $this->getPDO() - ->prepare($sql) - ->execute(); + $stmt = $this->getPDO()->prepare($sql); + return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); } /** @@ -697,9 +684,8 @@ public function deleteIndex(string $collection, string $id): bool $sql = $this->trigger(Database::EVENT_INDEX_DELETE, $sql); - return $this->getPDO() - ->prepare($sql) - ->execute(); + $stmt = $this->getPDO()->prepare($sql); + return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); } /** @@ -754,6 +740,7 @@ public function createDocument(string $collection, Document $document): Document $sql = $this->trigger(Database::EVENT_DOCUMENT_CREATE, $sql); + $stmt = $this->getPDO()->prepare($sql); $stmt->bindValue(':_uid', $document->getId()); @@ -984,6 +971,8 @@ public function createDocuments(string $collection, array $documents, int $batch */ public function updateDocument(string $collection, Document $document): Document { + // @Meldiron TODO: Use raw read&write here + $attributes = $document->getAttributes(); $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 7693e8e3b..75f594caf 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -73,11 +73,26 @@ public function ping(): bool /** * Execute raw command with no response * @param mixed $query + * @param array[string]mixed $params * - * @return mixed + * @return int + * @throws \Throwable + */ + public function executeWrite(mixed $query, array $params): int + { + // Not needed until we need Mongo Data API adapter + throw new Exception('Not implemented'); + } + + /** + * Execute raw command and get amount of affected documents + * @param mixed $query + * @param array[string]mixed $params + * + * @return int * @throws \Throwable */ - public function executeWrite(mixed $query): bool + public function executeWriteWithCount(mixed $query, array $params): int { // Not needed until we need Mongo Data API adapter throw new Exception('Not implemented'); @@ -86,11 +101,12 @@ public function executeWrite(mixed $query): bool /** * Execute raw command that returns a response * @param mixed $query + * @param array[string]mixed $params * * @return mixed * @throws \Throwable */ - public function executeRead(mixed $query): mixed + public function executeRead(mixed $query, array $params): mixed { // Not needed until we need Mongo Data API adapter throw new Exception('Not implemented'); diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 00189f979..0cc979659 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -9,6 +9,7 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception as DatabaseException; +use Utopia\Database\Extend\PDOStatement; use Utopia\Database\Query; abstract class SQL extends Adapter @@ -25,57 +26,107 @@ abstract class SQL extends Adapter public function __construct(mixed $pdo) { $this->pdo = $pdo; + $this->pdo->setAttribute(PDO::ATTR_STATEMENT_CLASS, [PDOStatement::class]); } /** - * Ping Database + * Execute raw command and get amount of affected documents + * @param mixed $query + * @param array[string]mixed $params * - * @return bool - * @throws Exception - * @throws PDOException + * @return int + * @throws \Throwable */ - public function ping(): bool + public function executeWriteWithCount(mixed $query, array $params): int { - $stmt = $this->getPDO()->prepare("SELECT 1;"); - $query = $stmt->queryString; + $stmt = $this->getPDO()->prepare($query); + + foreach($params as $param => $value) + { + if($value['method'] === 'value') { + $stmt->bindValue($param, $value['value'], $value['type']); + } else if($value['method'] === 'param') { + $stmt->bindParam($param, $value['var'], $value['type'], $value['maxLength'], $value['driverOptions']); + } + } + + $result = $stmt->execute(); + + if(!$result) { + throw new DatabaseException('Could not execute write query.'); + } + + $affectedRows = $stmt->rowCount(); - return $this->executeWrite($query); + return $affectedRows; } /** * Execute raw command with no response * @param mixed $query + * @param array[string]mixed $params * - * @return mixed + * @return bool * @throws \Throwable */ - public function executeWrite(mixed $query): bool + public function executeWrite(mixed $query, array $params): bool { - return $this->getPDO() - ->prepare($query) - ->execute(); + $stmt = $this->getPDO()->prepare($query); + + foreach($params as $param => $value) + { + if($value['method'] === 'value') { + $stmt->bindValue($param, $value['value'], $value['type']); + } else if($value['method'] === 'param') { + $stmt->bindParam($param, $value['var'], $value['type'], $value['maxLength'], $value['driverOptions']); + } + } + + return $stmt->execute(); } /** * Execute raw command that returns a response * @param mixed $query + * @param array[string]mixed $params * * @return mixed * @throws \Throwable */ - public function executeRead(mixed $query): mixed + public function executeRead(mixed $query, array $params): mixed { $stmt = $this->getPDO()->prepare($query); - $response = $stmt->fetchAll(); - \var_dump($query); - \var_dump($response); + foreach($params as $param => $value) + { + if($value['method'] === 'value') { + $stmt->bindValue($param, $value['value'], $value['type']); + } else if($value['method'] === 'param') { + $stmt->bindParam($param, $value['var'], $value['type'], $value['maxLength'], $value['driverOptions']); + } + } + + $stmt->execute(); + $response = $stmt->fetchAll(); $stmt->closeCursor(); return $response; } + /** + * Ping Database + * + * @return bool + * @throws Exception + * @throws PDOException + */ + public function ping(): bool + { + $stmt = $this->getPDO()->prepare("SELECT 1;"); + return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + } + /** * Check if Database exists * Optionally check if collection exists in Database @@ -108,9 +159,7 @@ public function exists(string $database, ?string $collection = null): bool $stmt->bindValue(':schema', $database, PDO::PARAM_STR); } - $stmt->execute(); - - $document = $this->executeRead($stmt->queryString); + $document = $this->executeRead($stmt->getQuery(), $stmt->getParams()); if (empty($document)) { return false; @@ -161,10 +210,7 @@ public function getDocument(string $collection, string $id, array $queries = []) $stmt->bindValue(':_tenant', $this->getTenant()); } - $stmt->execute(); - - $document = $stmt->fetchAll(); - $stmt->closeCursor(); + $document = $this->executeRead($stmt->getQuery(), $stmt->getParams()); if (empty($document)) { return new Document([]); @@ -947,7 +993,7 @@ public static function getPDOAttributes(): array { return [ PDO::ATTR_TIMEOUT => 3, // Specifies the timeout duration in seconds. Takes a value of type int. - PDO::ATTR_PERSISTENT => true, // Create a persistent connection + PDO::ATTR_PERSISTENT => false, // Create a persistent connection PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, // Fetch a result row as an associative array. PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, // PDO will throw a PDOException on srrors PDO::ATTR_EMULATE_PREPARES => true, // Emulate prepared statements diff --git a/src/Database/Extend/PDOStatement.php b/src/Database/Extend/PDOStatement.php new file mode 100644 index 000000000..2686c59cb --- /dev/null +++ b/src/Database/Extend/PDOStatement.php @@ -0,0 +1,56 @@ +params[$param] = [ + 'method' => 'param', + 'var' => $var, + 'type' => $type, + 'maxLength' => $maxLength, + 'driverOptions' => $driverOptions, + ]; + + return parent::bindParam($param, $var, $type, $maxLength, $driverOptions); + } + + public function bindValue(string|int $param, mixed $value, int $type = PDO::PARAM_STR): bool + { + $this->params[$param] = [ + 'method' => 'value', + 'value' => $value, + 'type' => $type + ]; + + return parent::bindValue($param, $value, $type); + } + + /** + * @return array[string]mixed + */ + public function getParams(): array + { + return $this->params; + } + + public function getQuery(): string + { + return $this->queryString; + } +} \ No newline at end of file From eafb57f5b6edbbfdf936f38e3a476dbb58446d56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 27 Feb 2024 10:12:59 +0100 Subject: [PATCH 21/42] Revert "WIP: Convert MariaDB to raw query methods" This reverts commit f33fcdd209ea512f33662e59780989941d691d28. --- README.md | 8 +- composer.lock | 12 +-- src/Database/Adapter.php | 28 +------ src/Database/Adapter/DataAPI.php | 5 +- src/Database/Adapter/DataAPIMariaDB.php | 26 ++----- src/Database/Adapter/MariaDB.php | 97 ++++++++++++++----------- src/Database/Adapter/Mongo.php | 22 +----- src/Database/Adapter/SQL.php | 96 +++++++----------------- src/Database/Extend/PDOStatement.php | 56 -------------- 9 files changed, 101 insertions(+), 249 deletions(-) delete mode 100644 src/Database/Extend/PDOStatement.php diff --git a/README.md b/README.md index 17969da57..1adf58143 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ $dbUser = 'root'; $dbPass = 'password'; $pdoConfig = [ PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => false, + PDO::ATTR_PERSISTENT => true, PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_EMULATE_PREPARES => true, @@ -134,7 +134,7 @@ $dbUser = 'root'; $dbPass = 'password'; $pdoConfig = [ PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => false, + PDO::ATTR_PERSISTENT => true, PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_EMULATE_PREPARES => true, @@ -165,7 +165,7 @@ $dbUser = 'root'; $dbPass = 'password'; $pdoConfig = [ PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => false, + PDO::ATTR_PERSISTENT => true, PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_EMULATE_PREPARES => true, @@ -193,7 +193,7 @@ use Utopia\Database\Adapter\SQLite; $dbPath = '/path/to/database.sqlite'; $pdoConfig = [ PDO::ATTR_TIMEOUT => 3, // Seconds - PDO::ATTR_PERSISTENT => false, + PDO::ATTR_PERSISTENT => true, PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, PDO::ATTR_EMULATE_PREPARES => true, diff --git a/composer.lock b/composer.lock index 4b582659b..4f853a20f 100644 --- a/composer.lock +++ b/composer.lock @@ -1252,16 +1252,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.6.17", + "version": "9.6.16", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "1a156980d78a6666721b7e8e8502fe210b587fcd" + "reference": "3767b2c56ce02d01e3491046f33466a1ae60a37f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/1a156980d78a6666721b7e8e8502fe210b587fcd", - "reference": "1a156980d78a6666721b7e8e8502fe210b587fcd", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/3767b2c56ce02d01e3491046f33466a1ae60a37f", + "reference": "3767b2c56ce02d01e3491046f33466a1ae60a37f", "shasum": "" }, "require": { @@ -1335,7 +1335,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.17" + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.16" }, "funding": [ { @@ -1351,7 +1351,7 @@ "type": "tidelift" } ], - "time": "2024-02-23T13:14:51+00:00" + "time": "2024-01-19T07:03:14+00:00" }, { "name": "psr/container", diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index a5b5f51c3..96dd8a273 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -275,42 +275,20 @@ abstract public function ping(): bool; /** * Execute raw command with no response * @param mixed $query - * @param array[string]mixed $params - * - * @return bool - * @throws \Throwable - */ - abstract public function executeWrite(mixed $query, array $params): bool; - - /** - * Execute raw command and get amount of affected documents - * @param mixed $query - * @param array[string]mixed $params - * - * @return int - * @throws \Throwable - */ - abstract public function executeWriteWithCount(mixed $query, array $params): int; - - /** - * Execute raw command that returns a response - * @param mixed $query - * @param array[string]mixed $params * * @return mixed * @throws \Throwable */ - abstract public function executeRead(mixed $query, array $params): mixed; + abstract public function executeWrite(mixed $query): bool; /** - * Execute transaction of multiple raw queries + * Execute raw command that returns a response * @param mixed $query - * @param array[string]mixed $params * * @return mixed * @throws \Throwable */ - abstract public function executeRead(mixed $query, array $params): mixed; + abstract public function executeRead(mixed $query): mixed; /** * Create Database diff --git a/src/Database/Adapter/DataAPI.php b/src/Database/Adapter/DataAPI.php index 038100ce8..faba187dc 100644 --- a/src/Database/Adapter/DataAPI.php +++ b/src/Database/Adapter/DataAPI.php @@ -11,13 +11,11 @@ trait DataAPI { /** - * @param array[string]mixed $params - * * @throws FetchException * @throws DatabaseException * @throws Exception */ - private function query(string $endpoint, string $secret, string $database, string $command, array $params): mixed + private function query(string $endpoint, string $secret, string $database, string $command): mixed { $response = Client::fetch( url: $endpoint . '/queries', @@ -29,7 +27,6 @@ private function query(string $endpoint, string $secret, string $database, strin method: 'POST', body: [ 'command' => $command, - 'parrams' => $params ] ); diff --git a/src/Database/Adapter/DataAPIMariaDB.php b/src/Database/Adapter/DataAPIMariaDB.php index 28ec61f78..53c10600c 100644 --- a/src/Database/Adapter/DataAPIMariaDB.php +++ b/src/Database/Adapter/DataAPIMariaDB.php @@ -28,41 +28,25 @@ public function __construct(string $endpoint, string $secret, string $database) /** * Execute raw command with no response * @param mixed $query - * @param array[string]mixed $params * - * @return bool + * @return mixed * @throws \Throwable */ - public function executeWrite(mixed $query, array $params): bool + public function executeWrite(mixed $query): bool { - $this->query($this->endpoint, $this->secret, $this->database, $query, $params); + $this->query($this->endpoint, $this->secret, $this->database, $query); return true; } - /** - * Execute raw command and get amount of affected documents - * @param mixed $query - * @param array[string]mixed $params - * - * @return int - * @throws \Throwable - */ - public function executeWriteWithCount(mixed $query, array $params): int - { - $this->query($this->endpoint, $this->secret, $this->database, $query, $params); - return 1; - } - /** * Execute raw command that returns a response * @param mixed $query - * @param array[string]mixed $params * * @return mixed * @throws \Throwable */ - public function executeRead(mixed $query, array $params): mixed + public function executeRead(mixed $query): mixed { - return $this->query($this->endpoint, $this->secret, $this->database, $query, $params); + return $this->query($this->endpoint, $this->secret, $this->database, $query); } } diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 5f1287bfc..297796b49 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -10,7 +10,6 @@ use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\Timeout as TimeoutException; -use Utopia\Database\Extend\PDOStatement; use Utopia\Database\Query; use Utopia\Database\Validator\Authorization; @@ -36,8 +35,9 @@ public function create(string $name): bool $sql = $this->trigger(Database::EVENT_DATABASE_CREATE, $sql); - $stmt = $this->getPDO()->prepare($sql); - return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + return $this->getPDO() + ->prepare($sql) + ->execute(); } /** @@ -56,8 +56,9 @@ public function delete(string $name): bool $sql = $this->trigger(Database::EVENT_DATABASE_DELETE, $sql); - $stmt = $this->getPDO()->prepare($sql); - return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + return $this->getPDO() + ->prepare($sql) + ->execute(); } /** @@ -147,8 +148,9 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql); try { - $stmt = $this->getPDO()->prepare($sql); - $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + $this->getPDO() + ->prepare($sql) + ->execute(); $sql = " CREATE TABLE IF NOT EXISTS {$this->getSQLTable($id . '_perms')} ( @@ -176,11 +178,13 @@ public function createCollection(string $name, array $attributes = [], array $in $sql = $this->trigger(Database::EVENT_COLLECTION_CREATE, $sql); - $stmt = $this->getPDO()->prepare($sql); - $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + $this->getPDO() + ->prepare($sql) + ->execute(); } catch (\Exception $th) { - $stmt = $this->getPDO()->prepare("DROP TABLE IF EXISTS {$this->getSQLTable($id)}, {$this->getSQLTable($id . '_perms')};"); - $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + $this->getPDO() + ->prepare("DROP TABLE IF EXISTS {$this->getSQLTable($id)}, {$this->getSQLTable($id . '_perms')};") + ->execute(); throw $th; } @@ -203,13 +207,13 @@ public function getSizeOfCollection(string $collection): int $permissions = $database . '/' . $collection . '_perms'; $collectionSize = $this->getPDO()->prepare(" - SELECT SUM(FS_BLOCK_SIZE + ALLOCATED_SIZE) AS total + SELECT SUM(FS_BLOCK_SIZE + ALLOCATED_SIZE) FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES WHERE NAME = :name "); - $permissionsSize = $this->getPDO()->prepare(" - SELECT SUM(FS_BLOCK_SIZE + ALLOCATED_SIZE) AS total + $permissionsSize = $this->getPDO()->prepare(" + SELECT SUM(FS_BLOCK_SIZE + ALLOCATED_SIZE) FROM INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES WHERE NAME = :permissions "); @@ -218,9 +222,9 @@ public function getSizeOfCollection(string $collection): int $permissionsSize->bindParam(':permissions', $permissions); try { - $resultCollection = $this->executeRead($collectionSize->getQuery(), $collectionSize->getParams()); - $resultPermissions = $this->executeRead($permissionsSize->getQuery(), $permissionsSize->getParams()); - $size = $resultCollection[0]["total"] + $resultPermissions[0]["total"]; + $collectionSize->execute(); + $permissionsSize->execute(); + $size = $collectionSize->fetchColumn() + $permissionsSize->fetchColumn(); } catch (PDOException $e) { throw new DatabaseException('Failed to get collection size: ' . $e->getMessage()); } @@ -244,8 +248,9 @@ public function deleteCollection(string $id): bool $sql = $this->trigger(Database::EVENT_COLLECTION_DELETE, $sql); - $stmt = $this->getPDO()->prepare($sql); - return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + return $this->getPDO() + ->prepare($sql) + ->execute(); } /** @@ -270,9 +275,9 @@ public function createAttribute(string $collection, string $id, string $type, in $sql = "ALTER TABLE {$this->getSQLTable($name)} ADD COLUMN `{$id}` {$type};"; $sql = $this->trigger(Database::EVENT_ATTRIBUTE_CREATE, $sql); - - $stmt = $this->getPDO()->prepare($sql); - return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + return $this->getPDO() + ->prepare($sql) + ->execute(); } /** @@ -298,8 +303,9 @@ public function updateAttribute(string $collection, string $id, string $type, in $sql = $this->trigger(Database::EVENT_ATTRIBUTE_UPDATE, $sql); - $stmt = $this->getPDO()->prepare($sql); - return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + return $this->getPDO() + ->prepare($sql) + ->execute(); } /** @@ -321,8 +327,9 @@ public function deleteAttribute(string $collection, string $id, bool $array = fa $sql = $this->trigger(Database::EVENT_ATTRIBUTE_DELETE, $sql); - $stmt = $this->getPDO()->prepare($sql); - return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + return $this->getPDO() + ->prepare($sql) + ->execute(); } /** @@ -345,8 +352,9 @@ public function renameAttribute(string $collection, string $old, string $new): b $sql = $this->trigger(Database::EVENT_ATTRIBUTE_UPDATE, $sql); - $stmt = $this->getPDO()->prepare($sql); - return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + return $this->getPDO() + ->prepare($sql) + ->execute(); } /** @@ -397,8 +405,9 @@ public function createRelationship( $sql = $this->trigger(Database::EVENT_ATTRIBUTE_CREATE, $sql); - $stmt = $this->getPDO()->prepare($sql); - return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + return $this->getPDO() + ->prepare($sql) + ->execute(); } /** @@ -481,8 +490,9 @@ public function updateRelationship( $sql = $this->trigger(Database::EVENT_ATTRIBUTE_UPDATE, $sql); - $stmt = $this->getPDO()->prepare($sql); - return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + return $this->getPDO() + ->prepare($sql) + ->execute(); } /** @@ -566,8 +576,9 @@ public function deleteRelationship( $sql = $this->trigger(Database::EVENT_ATTRIBUTE_DELETE, $sql); - $stmt = $this->getPDO()->prepare($sql); - return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + return $this->getPDO() + ->prepare($sql) + ->execute(); } /** @@ -589,8 +600,9 @@ public function renameIndex(string $collection, string $old, string $new): bool $sql = $this->trigger(Database::EVENT_INDEX_RENAME, $sql); - $stmt = $this->getPDO()->prepare($sql); - return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + return $this->getPDO() + ->prepare($sql) + ->execute(); } /** @@ -654,8 +666,9 @@ public function createIndex(string $collection, string $id, string $type, array $sql = "CREATE {$sqlType} `{$id}` ON {$this->getSQLTable($collection->getId())} ({$attributes})"; $sql = $this->trigger(Database::EVENT_INDEX_CREATE, $sql); - $stmt = $this->getPDO()->prepare($sql); - return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + return $this->getPDO() + ->prepare($sql) + ->execute(); } /** @@ -684,8 +697,9 @@ public function deleteIndex(string $collection, string $id): bool $sql = $this->trigger(Database::EVENT_INDEX_DELETE, $sql); - $stmt = $this->getPDO()->prepare($sql); - return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); + return $this->getPDO() + ->prepare($sql) + ->execute(); } /** @@ -740,7 +754,6 @@ public function createDocument(string $collection, Document $document): Document $sql = $this->trigger(Database::EVENT_DOCUMENT_CREATE, $sql); - $stmt = $this->getPDO()->prepare($sql); $stmt->bindValue(':_uid', $document->getId()); @@ -971,8 +984,6 @@ public function createDocuments(string $collection, array $documents, int $batch */ public function updateDocument(string $collection, Document $document): Document { - // @Meldiron TODO: Use raw read&write here - $attributes = $document->getAttributes(); $attributes['_createdAt'] = $document->getCreatedAt(); $attributes['_updatedAt'] = $document->getUpdatedAt(); diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 75f594caf..7693e8e3b 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -73,26 +73,11 @@ public function ping(): bool /** * Execute raw command with no response * @param mixed $query - * @param array[string]mixed $params * - * @return int - * @throws \Throwable - */ - public function executeWrite(mixed $query, array $params): int - { - // Not needed until we need Mongo Data API adapter - throw new Exception('Not implemented'); - } - - /** - * Execute raw command and get amount of affected documents - * @param mixed $query - * @param array[string]mixed $params - * - * @return int + * @return mixed * @throws \Throwable */ - public function executeWriteWithCount(mixed $query, array $params): int + public function executeWrite(mixed $query): bool { // Not needed until we need Mongo Data API adapter throw new Exception('Not implemented'); @@ -101,12 +86,11 @@ public function executeWriteWithCount(mixed $query, array $params): int /** * Execute raw command that returns a response * @param mixed $query - * @param array[string]mixed $params * * @return mixed * @throws \Throwable */ - public function executeRead(mixed $query, array $params): mixed + public function executeRead(mixed $query): mixed { // Not needed until we need Mongo Data API adapter throw new Exception('Not implemented'); diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 0cc979659..00189f979 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -9,7 +9,6 @@ use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception as DatabaseException; -use Utopia\Database\Extend\PDOStatement; use Utopia\Database\Query; abstract class SQL extends Adapter @@ -26,107 +25,57 @@ abstract class SQL extends Adapter public function __construct(mixed $pdo) { $this->pdo = $pdo; - $this->pdo->setAttribute(PDO::ATTR_STATEMENT_CLASS, [PDOStatement::class]); } /** - * Execute raw command and get amount of affected documents - * @param mixed $query - * @param array[string]mixed $params + * Ping Database * - * @return int - * @throws \Throwable + * @return bool + * @throws Exception + * @throws PDOException */ - public function executeWriteWithCount(mixed $query, array $params): int + public function ping(): bool { - $stmt = $this->getPDO()->prepare($query); - - foreach($params as $param => $value) - { - if($value['method'] === 'value') { - $stmt->bindValue($param, $value['value'], $value['type']); - } else if($value['method'] === 'param') { - $stmt->bindParam($param, $value['var'], $value['type'], $value['maxLength'], $value['driverOptions']); - } - } - - $result = $stmt->execute(); - - if(!$result) { - throw new DatabaseException('Could not execute write query.'); - } - - $affectedRows = $stmt->rowCount(); + $stmt = $this->getPDO()->prepare("SELECT 1;"); + $query = $stmt->queryString; - return $affectedRows; + return $this->executeWrite($query); } /** * Execute raw command with no response * @param mixed $query - * @param array[string]mixed $params * - * @return bool + * @return mixed * @throws \Throwable */ - public function executeWrite(mixed $query, array $params): bool + public function executeWrite(mixed $query): bool { - $stmt = $this->getPDO()->prepare($query); - - foreach($params as $param => $value) - { - if($value['method'] === 'value') { - $stmt->bindValue($param, $value['value'], $value['type']); - } else if($value['method'] === 'param') { - $stmt->bindParam($param, $value['var'], $value['type'], $value['maxLength'], $value['driverOptions']); - } - } - - return $stmt->execute(); + return $this->getPDO() + ->prepare($query) + ->execute(); } /** * Execute raw command that returns a response * @param mixed $query - * @param array[string]mixed $params * * @return mixed * @throws \Throwable */ - public function executeRead(mixed $query, array $params): mixed + public function executeRead(mixed $query): mixed { $stmt = $this->getPDO()->prepare($query); - - foreach($params as $param => $value) - { - if($value['method'] === 'value') { - $stmt->bindValue($param, $value['value'], $value['type']); - } else if($value['method'] === 'param') { - $stmt->bindParam($param, $value['var'], $value['type'], $value['maxLength'], $value['driverOptions']); - } - } - - $stmt->execute(); $response = $stmt->fetchAll(); + \var_dump($query); + \var_dump($response); + $stmt->closeCursor(); return $response; } - /** - * Ping Database - * - * @return bool - * @throws Exception - * @throws PDOException - */ - public function ping(): bool - { - $stmt = $this->getPDO()->prepare("SELECT 1;"); - return $this->executeWrite($stmt->getQuery(), $stmt->getParams()); - } - /** * Check if Database exists * Optionally check if collection exists in Database @@ -159,7 +108,9 @@ public function exists(string $database, ?string $collection = null): bool $stmt->bindValue(':schema', $database, PDO::PARAM_STR); } - $document = $this->executeRead($stmt->getQuery(), $stmt->getParams()); + $stmt->execute(); + + $document = $this->executeRead($stmt->queryString); if (empty($document)) { return false; @@ -210,7 +161,10 @@ public function getDocument(string $collection, string $id, array $queries = []) $stmt->bindValue(':_tenant', $this->getTenant()); } - $document = $this->executeRead($stmt->getQuery(), $stmt->getParams()); + $stmt->execute(); + + $document = $stmt->fetchAll(); + $stmt->closeCursor(); if (empty($document)) { return new Document([]); @@ -993,7 +947,7 @@ public static function getPDOAttributes(): array { return [ PDO::ATTR_TIMEOUT => 3, // Specifies the timeout duration in seconds. Takes a value of type int. - PDO::ATTR_PERSISTENT => false, // Create a persistent connection + PDO::ATTR_PERSISTENT => true, // Create a persistent connection PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, // Fetch a result row as an associative array. PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, // PDO will throw a PDOException on srrors PDO::ATTR_EMULATE_PREPARES => true, // Emulate prepared statements diff --git a/src/Database/Extend/PDOStatement.php b/src/Database/Extend/PDOStatement.php deleted file mode 100644 index 2686c59cb..000000000 --- a/src/Database/Extend/PDOStatement.php +++ /dev/null @@ -1,56 +0,0 @@ -params[$param] = [ - 'method' => 'param', - 'var' => $var, - 'type' => $type, - 'maxLength' => $maxLength, - 'driverOptions' => $driverOptions, - ]; - - return parent::bindParam($param, $var, $type, $maxLength, $driverOptions); - } - - public function bindValue(string|int $param, mixed $value, int $type = PDO::PARAM_STR): bool - { - $this->params[$param] = [ - 'method' => 'value', - 'value' => $value, - 'type' => $type - ]; - - return parent::bindValue($param, $value, $type); - } - - /** - * @return array[string]mixed - */ - public function getParams(): array - { - return $this->params; - } - - public function getQuery(): string - { - return $this->queryString; - } -} \ No newline at end of file From 50c47d1993ab2cefd138f2e087dcc816b3db63df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 27 Feb 2024 10:13:03 +0100 Subject: [PATCH 22/42] Revert "WIP: Update to raw queries" This reverts commit dacb169e94303b1567cbc6fd00c7612b2111e715. --- phpunit.xml | 2 +- src/Database/Adapter.php | 15 ++-------- src/Database/Adapter/DataAPIMariaDB.php | 17 ++---------- src/Database/Adapter/Mongo.php | 20 +++---------- src/Database/Adapter/SQL.php | 37 +++++++------------------ 5 files changed, 20 insertions(+), 71 deletions(-) diff --git a/phpunit.xml b/phpunit.xml index 783265d80..ccdaa969e 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="true"> + stopOnFailure="false"> ./tests/unit diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 96dd8a273..527f350e6 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -273,22 +273,13 @@ protected function trigger(string $event, mixed $query): mixed abstract public function ping(): bool; /** - * Execute raw command with no response - * @param mixed $query + * Execute raw command + * @param string $command * * @return mixed * @throws \Throwable */ - abstract public function executeWrite(mixed $query): bool; - - /** - * Execute raw command that returns a response - * @param mixed $query - * - * @return mixed - * @throws \Throwable - */ - abstract public function executeRead(mixed $query): mixed; + abstract public function execute(string $command): mixed; /** * Create Database diff --git a/src/Database/Adapter/DataAPIMariaDB.php b/src/Database/Adapter/DataAPIMariaDB.php index 53c10600c..5872262d7 100644 --- a/src/Database/Adapter/DataAPIMariaDB.php +++ b/src/Database/Adapter/DataAPIMariaDB.php @@ -26,26 +26,13 @@ public function __construct(string $endpoint, string $secret, string $database) } /** - * Execute raw command with no response + * Execute raw command * @param mixed $query * * @return mixed * @throws \Throwable */ - public function executeWrite(mixed $query): bool - { - $this->query($this->endpoint, $this->secret, $this->database, $query); - return true; - } - - /** - * Execute raw command that returns a response - * @param mixed $query - * - * @return mixed - * @throws \Throwable - */ - public function executeRead(mixed $query): mixed + public function execute(mixed $query): mixed { return $this->query($this->endpoint, $this->secret, $this->database, $query); } diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 7693e8e3b..13bb38ca3 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -70,27 +70,15 @@ public function ping(): bool return $this->getClient()->query(['ping' => 1])->ok ?? false; } - /** - * Execute raw command with no response - * @param mixed $query - * - * @return mixed - * @throws \Throwable - */ - public function executeWrite(mixed $query): bool - { - // Not needed until we need Mongo Data API adapter - throw new Exception('Not implemented'); - } - /** - * Execute raw command that returns a response + * Execute raw command * @param mixed $query * * @return mixed - * @throws \Throwable + * @throws Exception + * @throws MongoException */ - public function executeRead(mixed $query): mixed + public function execute(mixed $query): mixed { // Not needed until we need Mongo Data API adapter throw new Exception('Not implemented'); diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 00189f979..32c658612 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -36,46 +36,28 @@ public function __construct(mixed $pdo) */ public function ping(): bool { - $stmt = $this->getPDO()->prepare("SELECT 1;"); - $query = $stmt->queryString; + $query = $this->getPDO() + ->prepare("SELECT 1;") + ->queryString; - return $this->executeWrite($query); + return $this->execute($query); } /** - * Execute raw command with no response + * Execute raw command * @param mixed $query * * @return mixed - * @throws \Throwable + * @throws Exception + * @throws PDOException */ - public function executeWrite(mixed $query): bool + public function execute(mixed $query): mixed { return $this->getPDO() ->prepare($query) ->execute(); } - /** - * Execute raw command that returns a response - * @param mixed $query - * - * @return mixed - * @throws \Throwable - */ - public function executeRead(mixed $query): mixed - { - $stmt = $this->getPDO()->prepare($query); - $response = $stmt->fetchAll(); - - \var_dump($query); - \var_dump($response); - - $stmt->closeCursor(); - - return $response; - } - /** * Check if Database exists * Optionally check if collection exists in Database @@ -110,7 +92,8 @@ public function exists(string $database, ?string $collection = null): bool $stmt->execute(); - $document = $this->executeRead($stmt->queryString); + $document = $stmt->fetchAll(); + $stmt->closeCursor(); if (empty($document)) { return false; From b9064d0806e856a842b6989ebf3b4b0a602960f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 27 Feb 2024 10:13:08 +0100 Subject: [PATCH 23/42] Revert "Temporarly remove data API tests" This reverts commit 9d44651ee3e0493f5f7b38811cc9e8ad1f3b8807. --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d89bff6b1..631278763 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -77,7 +77,7 @@ jobs: Postgres, SQLite, MongoDB, - # DataAPIMariaDB + DataAPIMariaDB ] steps: From 1a89de6d0f2d7b6375b0d61e6a082fdd1b8260cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 27 Feb 2024 10:13:09 +0100 Subject: [PATCH 24/42] Revert "Linter fixes" This reverts commit 2c2c61e405f3a6496dd23af893a26d2d4215332e. --- src/Database/Adapter/DataAPIMariaDB.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Database/Adapter/DataAPIMariaDB.php b/src/Database/Adapter/DataAPIMariaDB.php index 5872262d7..530bdd2c3 100644 --- a/src/Database/Adapter/DataAPIMariaDB.php +++ b/src/Database/Adapter/DataAPIMariaDB.php @@ -2,6 +2,8 @@ namespace Utopia\Database\Adapter; +use PDOException; + class DataAPIMariaDB extends MariaDB { use DataAPI; From a4008581f477a9284f11dbb6f73cfaa85ead5c40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 27 Feb 2024 10:13:10 +0100 Subject: [PATCH 25/42] Revert "Update lockfile" This reverts commit 6c1261dfdffb241235f4ef880abe8f756af8e89b. --- composer.lock | 41 +---------------------------------------- 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/composer.lock b/composer.lock index 4f853a20f..7652b7212 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "4adf56005e024ee6b5bb991e96e2ee72", + "content-hash": "1fec834c5b222e402702b7bc89a5a8a8", "packages": [ { "name": "jean85/pretty-package-versions", @@ -264,45 +264,6 @@ }, "time": "2024-01-07T18:11:23+00:00" }, - { - "name": "utopia-php/fetch", - "version": "0.1.0", - "source": { - "type": "git", - "url": "https://github.com/utopia-php/fetch.git", - "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/utopia-php/fetch/zipball/2fa214b9262acd1a3583515a364da4f35929d5c5", - "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5", - "shasum": "" - }, - "require": { - "php": ">=8.0" - }, - "require-dev": { - "laravel/pint": "^1.5.0", - "phpstan/phpstan": "^1.10", - "phpunit/phpunit": "^9.5" - }, - "type": "library", - "autoload": { - "psr-4": { - "Utopia\\Fetch\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "description": "A simple library that provides an interface for making HTTP Requests.", - "support": { - "issues": "https://github.com/utopia-php/fetch/issues", - "source": "https://github.com/utopia-php/fetch/tree/0.1.0" - }, - "time": "2023-10-10T11:58:32+00:00" - }, { "name": "utopia-php/framework", "version": "0.33.2", From 1b77c977af775933c58598ad38a30b1e70730b2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 27 Feb 2024 10:29:57 +0100 Subject: [PATCH 26/42] Revert "Implement raw query execution" This reverts commit 855bf61d895192c7ace601ab67c8f5bf0a7ef566. --- docker-compose.yml | 36 +- src/Database/Adapter.php | 9 - src/Database/Adapter/DataAPI.php | 668 ++++++++++++++++++++++- src/Database/Adapter/DataAPIMariaDB.php | 42 +- src/Database/Adapter/Mongo.php | 14 - src/Database/Adapter/SQL.php | 19 +- tests/e2e/Adapter/DataAPIMariaDBTest.php | 2 +- 7 files changed, 701 insertions(+), 89 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 646b969ed..5977daa8e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,25 +82,25 @@ services: networks: - database - # data-api: - # image: utopia-php/data-api:0.1.0 - # container_name: utopia-data-api - # networks: - # - database - # environment: - # - UTOPIA_DATA_API_SECRET=test-secret - # - UTOPIA_DATA_API_SECRET_CONNECTION=mariadb://root:password@data-api-mariadb:3306/appwrite + data-api: + image: utopia-php/data-api:0.1.0 + container_name: utopia-data-api + networks: + - database + environment: + - UTOPIA_DATA_API_SECRET=test-secret + - UTOPIA_DATA_API_SECRET_CONNECTION=mariadb://root:password@data-api-mariadb:3306/appwrite - # data-api-mariadb: - # image: mariadb:10.7 - # container_name: utopia-data-api-mariadb - # networks: - # - database - # environment: - # - MYSQL_ROOT_PASSWORD=password - # - MARIADB_DATABASE=appwrite - # ports: - # - "8770:3306" + data-api-mariadb: + image: mariadb:10.7 + container_name: utopia-data-api-mariadb + networks: + - database + environment: + - MYSQL_ROOT_PASSWORD=password + - MARIADB_DATABASE=appwrite + ports: + - "8770:3306" networks: database: diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 527f350e6..5ee9bdd0e 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -272,15 +272,6 @@ protected function trigger(string $event, mixed $query): mixed */ abstract public function ping(): bool; - /** - * Execute raw command - * @param string $command - * - * @return mixed - * @throws \Throwable - */ - abstract public function execute(string $command): mixed; - /** * Create Database * diff --git a/src/Database/Adapter/DataAPI.php b/src/Database/Adapter/DataAPI.php index faba187dc..a39482fbd 100644 --- a/src/Database/Adapter/DataAPI.php +++ b/src/Database/Adapter/DataAPI.php @@ -4,29 +4,63 @@ use Exception; use Throwable; +use Utopia\Database\Adapter; +use Utopia\Database\Database; +use Utopia\Database\Document; use Utopia\Fetch\Client; +use Utopia\Database\Query; use Utopia\Database\Exception as DatabaseException; +use Utopia\Database\Validator\Authorization; use Utopia\Fetch\FetchException; -trait DataAPI +abstract class DataAPI extends Adapter { + protected string $endpoint; + protected string $secret; + protected string $database; + + /** + * Constructor. + * + * Set connection and settings + * + * @param string $endpoint + * @param string $secret + */ + public function __construct(string $endpoint, string $secret, string $database) + { + $this->endpoint = $endpoint; + $this->secret = $secret; + $this->database = $database; + } + /** + * @param mixed[] $params + * * @throws FetchException * @throws DatabaseException * @throws Exception */ - private function query(string $endpoint, string $secret, string $database, string $command): mixed + private function query(string $query, array $params = []): mixed { + $roles = \implode(',', Authorization::getRoles()); $response = Client::fetch( - url: $endpoint . '/queries', + url: $this->endpoint . '/queries', headers: [ - 'x-utopia-secret' => $secret, - 'x-utopia-database' => $database, + 'x-utopia-secret' => $this->secret, + 'x-utopia-database' => $this->database, + 'x-utopia-namespace' => $this->getNamespace(), + 'x-utopia-auth-roles' => $roles, + 'x-utopia-auth-status' => Authorization::$status ? 'true' : 'false', + 'x-utopia-auth-status-default' => Authorization::$statusDefault ? 'true' : 'false', + // TODO: Fix timeout + // 'x-utopia-timeout' => self::$timeout ? \strval(self::$timeout) : '', 'content-type' => 'application/json' ], method: 'POST', body: [ - 'command' => $command, + 'query' => $query, + 'params' => $params ] ); @@ -52,6 +86,626 @@ private function query(string $endpoint, string $secret, string $database, strin $body = \json_decode($response->getBody(), false); - return $body->output ?? ''; + $output = $body->output ?? ''; + + if(\is_object($output)) { + $output = new Document((array) $output); + } + + return $output; + } + + /** + * Ping Database + * + * @return bool + */ + public function ping(): bool + { + return $this->query('ping'); + } + + /** + * Create Database + * + * @param string $name + * + * @return bool + */ + public function create(string $name): bool + { + return $this->query('create', [$name]); + } + + /** + * Check if database exists + * Optionally check if collection exists in database + * + * @param string $database database name + * @param string|null $collection (optional) collection name + * + * @return bool + */ + public function exists(string $database, ?string $collection = null): bool + { + return $this->query('exists', [$database, $collection]); + } + + /** + * List Databases + * + * @return array + */ + public function list(): array + { + return $this->query('list'); + } + + /** + * Delete Database + * + * @param string $name + * + * @return bool + */ + public function delete(string $name): bool + { + return $this->query('delete', [$name]); + } + + /** + * Create Collection + * + * @param string $name + * @param array $attributes (optional) + * @param array $indexes (optional) + * @return bool + */ + public function createCollection(string $name, array $attributes = [], array $indexes = []): bool + { + return $this->query('createCollection', [$name, $attributes, $indexes]); + } + + /** + * Delete Collection + * + * @param string $id + * + * @return bool + */ + public function deleteCollection(string $id): bool + { + return $this->query('deleteCollection', [$id]); + } + + /** + * Create Attribute + * + * @param string $collection + * @param string $id + * @param string $type + * @param int $size + * @param bool $signed + * @param bool $array + * @return bool + */ + public function createAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false): bool + { + return $this->query('createAttribute', [$collection, $id, $type, $size, $signed, $array]); + } + + /** + * Update Attribute + * + * @param string $collection + * @param string $id + * @param string $type + * @param int $size + * @param bool $signed + * @param bool $array + * + * @return bool + */ + public function updateAttribute(string $collection, string $id, string $type, int $size, bool $signed = true, bool $array = false): bool + { + return $this->query('updateAttribute', [$collection, $id, $type, $size, $signed, $array]); + } + + /** + * Delete Attribute + * + * @param string $collection + * @param string $id + * + * @return bool + */ + public function deleteAttribute(string $collection, string $id): bool + { + return $this->query('deleteAttribute', [$collection, $id]); + } + + /** + * Rename Attribute + * + * @param string $collection + * @param string $old + * @param string $new + * @return bool + */ + public function renameAttribute(string $collection, string $old, string $new): bool + { + return $this->query('renameAttribute', [$collection, $old, $new]); + } + + /** + * @param string $collection + * @param string $relatedCollection + * @param string $type + * @param bool $twoWay + * @param string $id + * @param string $twoWayKey + * @return bool + */ + public function createRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay = false, string $id = '', string $twoWayKey = ''): bool + { + return $this->query('createRelationship', [$collection, $relatedCollection, $type, $twoWay, $id, $twoWayKey]); + } + + /** + * Update Relationship + * + * @param string $collection + * @param string $relatedCollection + * @param string $type + * @param bool $twoWay + * @param string $key + * @param string $twoWayKey + * @param string|null $newKey + * @param string|null $newTwoWayKey + * @return bool + */ + public function updateRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay, string $key, string $twoWayKey, ?string $newKey = null, ?string $newTwoWayKey = null): bool + { + return $this->query('updateRelationship', [$collection, $relatedCollection, $type, $twoWay, $key, $twoWayKey, $newKey, $newTwoWayKey]); + } + + /** + * Delete Relationship + * + * @param string $collection + * @param string $relatedCollection + * @param string $type + * @param bool $twoWay + * @param string $key + * @param string $twoWayKey + * @param string $side + * @return bool + */ + public function deleteRelationship(string $collection, string $relatedCollection, string $type, bool $twoWay, string $key, string $twoWayKey, string $side): bool + { + return $this->query('deleteRelationship', [$collection, $relatedCollection, $type, $twoWay, $key, $twoWayKey, $side]); + } + + /** + * Rename Index + * + * @param string $collection + * @param string $old + * @param string $new + * @return bool + */ + public function renameIndex(string $collection, string $old, string $new): bool + { + return $this->query('renameIndex', [$collection, $old, $new]); + } + + /** + * Create Index + * + * @param string $collection + * @param string $id + * @param string $type + * @param array $attributes + * @param array $lengths + * @param array $orders + * + * @return bool + */ + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders): bool + { + return $this->query('createIndex', [$collection, $id, $type, $attributes, $lengths, $orders]); + } + + /** + * Delete Index + * + * @param string $collection + * @param string $id + * + * @return bool + */ + public function deleteIndex(string $collection, string $id): bool + { + return $this->query('deleteIndex', [$collection, $id]); + } + + /** + * Get Document + * + * @param string $collection + * @param string $id + * @param array $queries + * @return Document + */ + public function getDocument(string $collection, string $id, array $queries = []): Document + { + return $this->query('getDocument', [$collection, $id, $queries]); + } + + /** + * Create Document + * + * @param string $collection + * @param Document $document + * + * @return Document + */ + public function createDocument(string $collection, Document $document): Document + { + return $this->query('createDocument', [$collection, $document]); + } + + /** + * Create Documents in batches + * + * @param string $collection + * @param array $documents + * @param int $batchSize + * + * @return array + * + * @throws DatabaseException + */ + public function createDocuments(string $collection, array $documents, int $batchSize): array + { + return $this->query('createDocuments', [$collection, $documents, $batchSize]); + } + + /** + * Update Document + * + * @param string $collection + * @param Document $document + * + * @return Document + */ + public function updateDocument(string $collection, Document $document): Document + { + return $this->query('updateDocument', [$collection, $document]); + } + + /** + * Update Documents in batches + * + * @param string $collection + * @param array $documents + * @param int $batchSize + * + * @return array + * + * @throws DatabaseException + */ + public function updateDocuments(string $collection, array $documents, int $batchSize): array + { + return $this->query('updateDocuments', [$collection, $documents, $batchSize]); + } + + /** + * Delete Document + * + * @param string $collection + * @param string $id + * + * @return bool + */ + public function deleteDocument(string $collection, string $id): bool + { + return $this->query('deleteDocument', [$collection, $id]); + } + + /** + * Find Documents + * + * Find data sets using chosen queries + * + * @param string $collection + * @param array $queries + * @param int|null $limit + * @param int|null $offset + * @param array $orderAttributes + * @param array $orderTypes + * @param array $cursor + * @param string $cursorDirection + * + * @return array + */ + public function find(string $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER): array + { + return $this->query('find', [$collection, $queries, $limit, $offset, $orderAttributes, $orderTypes, $cursor, $cursorDirection]); + } + + /** + * Sum an attribute + * + * @param string $collection + * @param string $attribute + * @param array $queries + * @param int|null $max + * + * @return int|float + */ + public function sum(string $collection, string $attribute, array $queries = [], ?int $max = null): float|int + { + return $this->query('sum', [$collection, $attribute, $queries, $max]); + } + + /** + * Count Documents + * + * @param string $collection + * @param array $queries + * @param int|null $max + * + * @return int + */ + public function count(string $collection, array $queries = [], ?int $max = null): int + { + return $this->query('count', [$collection, $queries, $max]); + } + + /** + * Get Collection Size + * + * @param string $collection + * @return int + * @throws DatabaseException + */ + public function getSizeOfCollection(string $collection): int + { + return $this->query('getSizeOfCollection', [$collection]); + } + + /** + * Get max STRING limit + * + * @return int + */ + public function getLimitForString(): int + { + return $this->query('getLimitForString'); + } + + /** + * Get max INT limit + * + * @return int + */ + public function getLimitForInt(): int + { + return $this->query('getLimitForInt'); + } + + /** + * Get maximum attributes limit. + * + * @return int + */ + public function getLimitForAttributes(): int + { + return $this->query('getLimitForAttributes'); + } + + /** + * Get maximum index limit. + * + * @return int + */ + public function getLimitForIndexes(): int + { + return $this->query('getLimitForIndexes'); + } + + /** + * Is schemas supported? + * + * @return bool + */ + public function getSupportForSchemas(): bool + { + return $this->query('getSupportForSchemas'); + } + + /** + * Is index supported? + * + * @return bool + */ + public function getSupportForIndex(): bool + { + return $this->query('getSupportForIndex'); + } + + /** + * Is unique index supported? + * + * @return bool + */ + public function getSupportForUniqueIndex(): bool + { + return $this->query('getSupportForUniqueIndex'); + } + + /** + * Is fulltext index supported? + * + * @return bool + */ + public function getSupportForFulltextIndex(): bool + { + return $this->query('getSupportForFulltextIndex'); + } + + /** + * Is fulltext wildcard supported? + * + * @return bool + */ + public function getSupportForFulltextWildcardIndex(): bool + { + return $this->query('getSupportForFulltextWildcardIndex'); + } + + + /** + * Does the adapter handle casting? + * + * @return bool + */ + public function getSupportForCasting(): bool + { + return $this->query('getSupportForCasting'); + } + + /** + * Does the adapter handle array Contains? + * + * @return bool + */ + public function getSupportForQueryContains(): bool + { + return $this->query('getSupportForQueryContains'); + } + + /** + * Are timeouts supported? + * + * @return bool + */ + public function getSupportForTimeouts(): bool + { + return $this->query('getSupportForTimeouts'); + } + + /** + * Are relationships supported? + * + * @return bool + */ + public function getSupportForRelationships(): bool + { + return $this->query('getSupportForRelationships'); + } + + /** + * Get current attribute count from collection document + * + * @param Document $collection + * @return int + */ + public function getCountOfAttributes(Document $collection): int + { + return $this->query('getCountOfAttributes', [$collection]); + } + + /** + * Get current index count from collection document + * + * @param Document $collection + * @return int + */ + public function getCountOfIndexes(Document $collection): int + { + return $this->query('getCountOfIndexes', [$collection]); + } + + /** + * Estimate maximum number of bytes required to store a document in $collection. + * Byte requirement varies based on column type and size. + * Needed to satisfy MariaDB/MySQL row width limit. + * Return 0 when no restrictions apply to row width + * + * @param Document $collection + * @return int + */ + public function getAttributeWidth(Document $collection): int + { + return $this->query('getAttributeWidth', [$collection]); + } + + /** + * Get list of keywords that cannot be used + * + * @return array + */ + public function getKeywords(): array + { + return $this->query('getKeywords'); + } + + /** + * Get an attribute projection given a list of selected attributes + * + * @param array $selections + * @param string $prefix + * @return mixed + */ + protected function getAttributeProjection(array $selections, string $prefix = ''): mixed + { + return $this->query('getAttributeProjection', [$selections, $prefix]); + } + + /** + * Increase and Decrease Attribute Value + * + * @param string $collection + * @param string $id + * @param string $attribute + * @param int|float $value + * @param int|float|null $min + * @param int|float|null $max + * @return bool + * @throws Exception + */ + public function increaseDocumentAttribute(string $collection, string $id, string $attribute, int|float $value, int|float|null $min = null, int|float|null $max = null): bool + { + return $this->query('increaseDocumentAttribute', [$collection, $id, $attribute, $value, $min, $max]); + } + + /** + * @return int + */ + public function getMaxIndexLength(): int + { + return $this->query('getMaxIndexLength'); + } + + /** + * Set a global timeout for database queries in milliseconds. + * + * This function allows you to set a maximum execution time for all database + * queries executed using the library, or a specific event specified by the + * event parameter. Once this timeout is set, any database query that takes + * longer than the specified time will be automatically terminated by the library, + * and an appropriate error or exception will be raised to handle the timeout condition. + * + * @param int $milliseconds The timeout value in milliseconds for database queries. + * @param string $event The event the timeout should fire fore + * @return void + * + * @throws Exception The provided timeout value must be greater than or equal to 0. + */ + public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): void + { + $this->query('setTimeout', [$milliseconds, $event]); } } diff --git a/src/Database/Adapter/DataAPIMariaDB.php b/src/Database/Adapter/DataAPIMariaDB.php index 530bdd2c3..0cd520907 100644 --- a/src/Database/Adapter/DataAPIMariaDB.php +++ b/src/Database/Adapter/DataAPIMariaDB.php @@ -2,40 +2,38 @@ namespace Utopia\Database\Adapter; -use PDOException; +use Utopia\Database\Database; -class DataAPIMariaDB extends MariaDB +class DataAPIMariaDB extends DataAPI { - use DataAPI; - - protected string $endpoint; - protected string $secret; - protected string $database; - /** - * Constructor. + * Returns number of attributes used by default. * - * Set connection and settings + * @return int + */ + public static function getCountOfDefaultAttributes(): int + { + return \count(Database::INTERNAL_ATTRIBUTES); + } + + /** + * Returns number of indexes used by default. * - * @param string $endpoint - * @param string $secret + * @return int */ - public function __construct(string $endpoint, string $secret, string $database) + public static function getCountOfDefaultIndexes(): int { - $this->endpoint = $endpoint; - $this->secret = $secret; - $this->database = $database; + return \count(Database::INTERNAL_INDEXES); } /** - * Execute raw command - * @param mixed $query + * Get maximum width, in bytes, allowed for a SQL row + * Return 0 when no restrictions apply * - * @return mixed - * @throws \Throwable + * @return int */ - public function execute(mixed $query): mixed + public static function getDocumentSizeLimit(): int { - return $this->query($this->endpoint, $this->secret, $this->database, $query); + return 65535; } } diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 13bb38ca3..e704df5f1 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -70,20 +70,6 @@ public function ping(): bool return $this->getClient()->query(['ping' => 1])->ok ?? false; } - /** - * Execute raw command - * @param mixed $query - * - * @return mixed - * @throws Exception - * @throws MongoException - */ - public function execute(mixed $query): mixed - { - // Not needed until we need Mongo Data API adapter - throw new Exception('Not implemented'); - } - /** * Create Database * diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 32c658612..00b6d5832 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -35,26 +35,9 @@ public function __construct(mixed $pdo) * @throws PDOException */ public function ping(): bool - { - $query = $this->getPDO() - ->prepare("SELECT 1;") - ->queryString; - - return $this->execute($query); - } - - /** - * Execute raw command - * @param mixed $query - * - * @return mixed - * @throws Exception - * @throws PDOException - */ - public function execute(mixed $query): mixed { return $this->getPDO() - ->prepare($query) + ->prepare("SELECT 1;") ->execute(); } diff --git a/tests/e2e/Adapter/DataAPIMariaDBTest.php b/tests/e2e/Adapter/DataAPIMariaDBTest.php index 471297743..80138bdf7 100644 --- a/tests/e2e/Adapter/DataAPIMariaDBTest.php +++ b/tests/e2e/Adapter/DataAPIMariaDBTest.php @@ -30,7 +30,7 @@ public static function getDatabase(): Database return self::$database; } - $database = new Database(new DataAPIMariaDB('https://8088-utopiaphp-dataapi-1zrdiri2xh5.ws-eu108.gitpod.io/v1', 'test-secret', 'default'), new Cache(new None())); + $database = new Database(new DataAPIMariaDB('http://data-api/v1', 'test-secret', 'default'), new Cache(new None())); $database->setDatabase('utopiaTests'); $database->setNamespace(static::$namespace = 'myapp_' . uniqid()); From 27782a6934de3c2af7fed0b5a958548b0ec1761d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 27 Feb 2024 10:31:45 +0100 Subject: [PATCH 27/42] Fix timeout --- composer.lock | 55 +++++++++++++++++++++++++++----- phpunit.xml | 2 +- src/Database/Adapter/DataAPI.php | 8 +++-- 3 files changed, 53 insertions(+), 12 deletions(-) diff --git a/composer.lock b/composer.lock index 7652b7212..52b250d19 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1fec834c5b222e402702b7bc89a5a8a8", + "content-hash": "4adf56005e024ee6b5bb991e96e2ee72", "packages": [ { "name": "jean85/pretty-package-versions", @@ -264,6 +264,45 @@ }, "time": "2024-01-07T18:11:23+00:00" }, + { + "name": "utopia-php/fetch", + "version": "0.1.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/fetch.git", + "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/fetch/zipball/2fa214b9262acd1a3583515a364da4f35929d5c5", + "reference": "2fa214b9262acd1a3583515a364da4f35929d5c5", + "shasum": "" + }, + "require": { + "php": ">=8.0" + }, + "require-dev": { + "laravel/pint": "^1.5.0", + "phpstan/phpstan": "^1.10", + "phpunit/phpunit": "^9.5" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Fetch\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A simple library that provides an interface for making HTTP Requests.", + "support": { + "issues": "https://github.com/utopia-php/fetch/issues", + "source": "https://github.com/utopia-php/fetch/tree/0.1.0" + }, + "time": "2023-10-10T11:58:32+00:00" + }, { "name": "utopia-php/framework", "version": "0.33.2", @@ -1213,16 +1252,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.6.16", + "version": "9.6.17", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "3767b2c56ce02d01e3491046f33466a1ae60a37f" + "reference": "1a156980d78a6666721b7e8e8502fe210b587fcd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/3767b2c56ce02d01e3491046f33466a1ae60a37f", - "reference": "3767b2c56ce02d01e3491046f33466a1ae60a37f", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/1a156980d78a6666721b7e8e8502fe210b587fcd", + "reference": "1a156980d78a6666721b7e8e8502fe210b587fcd", "shasum": "" }, "require": { @@ -1296,7 +1335,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.16" + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.6.17" }, "funding": [ { @@ -1312,7 +1351,7 @@ "type": "tidelift" } ], - "time": "2024-01-19T07:03:14+00:00" + "time": "2024-02-23T13:14:51+00:00" }, { "name": "psr/container", @@ -2601,5 +2640,5 @@ "php": ">=8.0" }, "platform-dev": [], - "plugin-api-version": "2.6.0" + "plugin-api-version": "2.3.0" } diff --git a/phpunit.xml b/phpunit.xml index ccdaa969e..783265d80 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false"> + stopOnFailure="true"> ./tests/unit diff --git a/src/Database/Adapter/DataAPI.php b/src/Database/Adapter/DataAPI.php index a39482fbd..80f37b02f 100644 --- a/src/Database/Adapter/DataAPI.php +++ b/src/Database/Adapter/DataAPI.php @@ -19,6 +19,8 @@ abstract class DataAPI extends Adapter protected string $secret; protected string $database; + protected int $timeout; + /** * Constructor. * @@ -53,8 +55,7 @@ private function query(string $query, array $params = []): mixed 'x-utopia-auth-roles' => $roles, 'x-utopia-auth-status' => Authorization::$status ? 'true' : 'false', 'x-utopia-auth-status-default' => Authorization::$statusDefault ? 'true' : 'false', - // TODO: Fix timeout - // 'x-utopia-timeout' => self::$timeout ? \strval(self::$timeout) : '', + 'x-utopia-timeout' => $this->timeout ? \strval($this->timeout) : '', 'content-type' => 'application/json' ], method: 'POST', @@ -706,6 +707,7 @@ public function getMaxIndexLength(): int */ public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): void { - $this->query('setTimeout', [$milliseconds, $event]); + // TODO: Use $event? + $this->timeout = $milliseconds; } } From cda35157f6f0351fbf2ea7561c3840fa6ff587e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 27 Feb 2024 10:32:17 +0100 Subject: [PATCH 28/42] Final touches to last revert --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 631278763..d89bff6b1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -77,7 +77,7 @@ jobs: Postgres, SQLite, MongoDB, - DataAPIMariaDB + # DataAPIMariaDB ] steps: From 2402400815bf815cf1232e88ac6e691551f8d8f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 27 Feb 2024 10:47:42 +0100 Subject: [PATCH 29/42] Fix timeouts --- src/Database/Adapter/DataAPI.php | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/src/Database/Adapter/DataAPI.php b/src/Database/Adapter/DataAPI.php index 80f37b02f..8a23e45ca 100644 --- a/src/Database/Adapter/DataAPI.php +++ b/src/Database/Adapter/DataAPI.php @@ -19,7 +19,10 @@ abstract class DataAPI extends Adapter protected string $secret; protected string $database; - protected int $timeout; + /** + * @var array $timeouts Map of timeouts where key is event name and value is timeout in milliseconds + */ + protected array $timeouts = []; /** * Constructor. @@ -55,7 +58,7 @@ private function query(string $query, array $params = []): mixed 'x-utopia-auth-roles' => $roles, 'x-utopia-auth-status' => Authorization::$status ? 'true' : 'false', 'x-utopia-auth-status-default' => Authorization::$statusDefault ? 'true' : 'false', - 'x-utopia-timeout' => $this->timeout ? \strval($this->timeout) : '', + 'x-utopia-timeouts' => $this->timeouts, 'content-type' => 'application/json' ], method: 'POST', @@ -89,8 +92,26 @@ private function query(string $query, array $params = []): mixed $output = $body->output ?? ''; - if(\is_object($output)) { - $output = new Document((array) $output); + $processArray = function (mixed $self, mixed $json) { + $keys = []; + + foreach ($json as $param) { + if(\is_object($param)) { + $keys[] = new Document((array) $param); + } elseif(\is_array($param)) { + $keys[] = $self($self, $param); + } else { + $keys[] = $param; + } + } + + return $keys; + }; + + if(\is_array($output)) { + $output = $processArray($processArray, $output); + } else { + $output = $processArray($processArray, [$output])[0]; } return $output; From 672a53814e2b84f07386e1f09f4cd410301a5fda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 27 Feb 2024 11:04:47 +0100 Subject: [PATCH 30/42] Fix timeouts? --- src/Database/Adapter/DataAPI.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Database/Adapter/DataAPI.php b/src/Database/Adapter/DataAPI.php index 8a23e45ca..01f5ca75e 100644 --- a/src/Database/Adapter/DataAPI.php +++ b/src/Database/Adapter/DataAPI.php @@ -58,7 +58,7 @@ private function query(string $query, array $params = []): mixed 'x-utopia-auth-roles' => $roles, 'x-utopia-auth-status' => Authorization::$status ? 'true' : 'false', 'x-utopia-auth-status-default' => Authorization::$statusDefault ? 'true' : 'false', - 'x-utopia-timeouts' => $this->timeouts, + 'x-utopia-timeouts' => \json_encode($this->timeouts), 'content-type' => 'application/json' ], method: 'POST', @@ -728,7 +728,6 @@ public function getMaxIndexLength(): int */ public function setTimeout(int $milliseconds, string $event = Database::EVENT_ALL): void { - // TODO: Use $event? - $this->timeout = $milliseconds; + $this->timeouts[$event] = $milliseconds; } } From 34f21f70dffe351cba216290d0ccb7cfaf6345df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 27 Feb 2024 12:10:05 +0100 Subject: [PATCH 31/42] Fix timeouts, finally --- src/Database/Adapter/DataAPI.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Database/Adapter/DataAPI.php b/src/Database/Adapter/DataAPI.php index 01f5ca75e..387225ab1 100644 --- a/src/Database/Adapter/DataAPI.php +++ b/src/Database/Adapter/DataAPI.php @@ -730,4 +730,15 @@ public function setTimeout(int $milliseconds, string $event = Database::EVENT_AL { $this->timeouts[$event] = $milliseconds; } + + /** + * Clears a global timeout for database queries. + * + * @param string $event + * @return void + */ + public function clearTimeout(string $event): void + { + unset($this->timeouts[$event]); + } } From e593083242f3e47aaabd1e5c2b7a2962b95d5a45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Tue, 27 Feb 2024 14:58:19 +0100 Subject: [PATCH 32/42] Fix tests --- docker-compose.yml | 6 +++--- src/Database/Adapter/DataAPI.php | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5977daa8e..fe4818195 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -89,16 +89,16 @@ services: - database environment: - UTOPIA_DATA_API_SECRET=test-secret - - UTOPIA_DATA_API_SECRET_CONNECTION=mariadb://root:password@data-api-mariadb:3306/appwrite + - UTOPIA_DATA_API_SECRET_CONNECTION=mariadb://root:password@data-api-mariadb:3306/utopiaTests data-api-mariadb: - image: mariadb:10.7 + image: mariadb:10.11 container_name: utopia-data-api-mariadb networks: - database environment: - MYSQL_ROOT_PASSWORD=password - - MARIADB_DATABASE=appwrite + - MARIADB_DATABASE=utopiaTests ports: - "8770:3306" diff --git a/src/Database/Adapter/DataAPI.php b/src/Database/Adapter/DataAPI.php index 387225ab1..f5a26ece8 100644 --- a/src/Database/Adapter/DataAPI.php +++ b/src/Database/Adapter/DataAPI.php @@ -59,12 +59,14 @@ private function query(string $query, array $params = []): mixed 'x-utopia-auth-status' => Authorization::$status ? 'true' : 'false', 'x-utopia-auth-status-default' => Authorization::$statusDefault ? 'true' : 'false', 'x-utopia-timeouts' => \json_encode($this->timeouts), + 'x-utopia-share-tables' => $this->shareTables ? 'true' : 'false', + 'x-utopia-tenant' => $this->tenant, 'content-type' => 'application/json' ], method: 'POST', body: [ 'query' => $query, - 'params' => $params + 'params' => \base64_encode(\serialize($params)) ] ); From f08fb1680cfc44bbef9606e5dba71e70198a2c64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 29 Feb 2024 13:32:05 +0000 Subject: [PATCH 33/42] Fix tests --- composer.lock | 2 +- docker-compose.yml | 12 ++++++------ src/Database/Adapter/DataAPI.php | 4 ++-- tests/e2e/Adapter/DataAPIMariaDBTest.php | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/composer.lock b/composer.lock index 52b250d19..4b582659b 100644 --- a/composer.lock +++ b/composer.lock @@ -2640,5 +2640,5 @@ "php": ">=8.0" }, "platform-dev": [], - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.6.0" } diff --git a/docker-compose.yml b/docker-compose.yml index fe4818195..f07a82b8d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,18 +82,18 @@ services: networks: - database - data-api: - image: utopia-php/data-api:0.1.0 - container_name: utopia-data-api + database-proxy: + image: appwrite/database-proxy:0.1.0 + container_name: utopia-database-proxy networks: - database environment: - UTOPIA_DATA_API_SECRET=test-secret - - UTOPIA_DATA_API_SECRET_CONNECTION=mariadb://root:password@data-api-mariadb:3306/utopiaTests + - UTOPIA_DATA_API_SECRET_CONNECTION=mariadb://root:password@database-proxy-mariadb:3306/utopiaTests - data-api-mariadb: + database-proxy-mariadb: image: mariadb:10.11 - container_name: utopia-data-api-mariadb + container_name: utopia-database-proxy-mariadb networks: - database environment: diff --git a/src/Database/Adapter/DataAPI.php b/src/Database/Adapter/DataAPI.php index f5a26ece8..1317d540a 100644 --- a/src/Database/Adapter/DataAPI.php +++ b/src/Database/Adapter/DataAPI.php @@ -58,9 +58,9 @@ private function query(string $query, array $params = []): mixed 'x-utopia-auth-roles' => $roles, 'x-utopia-auth-status' => Authorization::$status ? 'true' : 'false', 'x-utopia-auth-status-default' => Authorization::$statusDefault ? 'true' : 'false', - 'x-utopia-timeouts' => \json_encode($this->timeouts), + 'x-utopia-timeouts' => \strval(\json_encode($this->timeouts) ?: ''), 'x-utopia-share-tables' => $this->shareTables ? 'true' : 'false', - 'x-utopia-tenant' => $this->tenant, + 'x-utopia-tenant' => \strval($this->tenant ?? ''), 'content-type' => 'application/json' ], method: 'POST', diff --git a/tests/e2e/Adapter/DataAPIMariaDBTest.php b/tests/e2e/Adapter/DataAPIMariaDBTest.php index 80138bdf7..ff71a9661 100644 --- a/tests/e2e/Adapter/DataAPIMariaDBTest.php +++ b/tests/e2e/Adapter/DataAPIMariaDBTest.php @@ -18,7 +18,7 @@ class DataAPIMariaDBTest extends Base */ public static function getAdapterName(): string { - return "data-api-mariadb"; + return "database-proxy-mariadb"; } /** @@ -30,7 +30,7 @@ public static function getDatabase(): Database return self::$database; } - $database = new Database(new DataAPIMariaDB('http://data-api/v1', 'test-secret', 'default'), new Cache(new None())); + $database = new Database(new DataAPIMariaDB('http://database-proxy/v1', 'test-secret', 'default'), new Cache(new None())); $database->setDatabase('utopiaTests'); $database->setNamespace(static::$namespace = 'myapp_' . uniqid()); From 3adf7e4028b06034145d820c0d2d9731794c98a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 29 Feb 2024 13:43:35 +0000 Subject: [PATCH 34/42] Rename data api to proxy --- .github/workflows/tests.yml | 2 +- phpunit.xml | 2 +- .../{DataAPIMariaDB.php => MariaDBProxy.php} | 2 +- .../Adapter/{DataAPI.php => Proxy.php} | 2 +- src/Database/Query.php | 18 +----------------- ...APIMariaDBTest.php => MariaDBProxyTest.php} | 6 +++--- 6 files changed, 8 insertions(+), 24 deletions(-) rename src/Database/Adapter/{DataAPIMariaDB.php => MariaDBProxy.php} (95%) rename src/Database/Adapter/{DataAPI.php => Proxy.php} (99%) rename tests/e2e/Adapter/{DataAPIMariaDBTest.php => MariaDBProxyTest.php} (79%) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d89bff6b1..d99c6c3a2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -77,7 +77,7 @@ jobs: Postgres, SQLite, MongoDB, - # DataAPIMariaDB + MariaDBProxy ] steps: diff --git a/phpunit.xml b/phpunit.xml index 783265d80..ccdaa969e 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="true"> + stopOnFailure="false"> ./tests/unit diff --git a/src/Database/Adapter/DataAPIMariaDB.php b/src/Database/Adapter/MariaDBProxy.php similarity index 95% rename from src/Database/Adapter/DataAPIMariaDB.php rename to src/Database/Adapter/MariaDBProxy.php index 0cd520907..d52d13fff 100644 --- a/src/Database/Adapter/DataAPIMariaDB.php +++ b/src/Database/Adapter/MariaDBProxy.php @@ -4,7 +4,7 @@ use Utopia\Database\Database; -class DataAPIMariaDB extends DataAPI +class MariaDBProxy extends Proxy { /** * Returns number of attributes used by default. diff --git a/src/Database/Adapter/DataAPI.php b/src/Database/Adapter/Proxy.php similarity index 99% rename from src/Database/Adapter/DataAPI.php rename to src/Database/Adapter/Proxy.php index 1317d540a..751c3860d 100644 --- a/src/Database/Adapter/DataAPI.php +++ b/src/Database/Adapter/Proxy.php @@ -13,7 +13,7 @@ use Utopia\Database\Validator\Authorization; use Utopia\Fetch\FetchException; -abstract class DataAPI extends Adapter +abstract class Proxy extends Adapter { protected string $endpoint; protected string $secret; diff --git a/src/Database/Query.php b/src/Database/Query.php index 218f99eae..8c99dec10 100644 --- a/src/Database/Query.php +++ b/src/Database/Query.php @@ -2,11 +2,10 @@ namespace Utopia\Database; -use JsonSerializable; use JsonException; use Utopia\Database\Exception\Query as QueryException; -class Query implements JsonSerializable +class Query { // Filter methods public const TYPE_EQUAL = 'equal'; @@ -697,19 +696,4 @@ public function setOnArray(bool $bool): void { $this->onArray = $bool; } - - /** - * PHP native method override - * - * @return array - * @throws Exception - */ - public function jsonSerialize(): array - { - return [ - 'method' => $this->method, - 'attribute' => $this->attribute, - 'values' => $this->values, - ]; - } } diff --git a/tests/e2e/Adapter/DataAPIMariaDBTest.php b/tests/e2e/Adapter/MariaDBProxyTest.php similarity index 79% rename from tests/e2e/Adapter/DataAPIMariaDBTest.php rename to tests/e2e/Adapter/MariaDBProxyTest.php index ff71a9661..046abb954 100644 --- a/tests/e2e/Adapter/DataAPIMariaDBTest.php +++ b/tests/e2e/Adapter/MariaDBProxyTest.php @@ -5,9 +5,9 @@ use Utopia\Cache\Adapter\None; use Utopia\Database\Database; use Utopia\Cache\Cache; -use Utopia\Database\Adapter\DataAPIMariaDB; +use Utopia\Database\Adapter\MariaDBProxy; -class DataAPIMariaDBTest extends Base +class MariaDBProxyTest extends Base { public static ?Database $database = null; @@ -30,7 +30,7 @@ public static function getDatabase(): Database return self::$database; } - $database = new Database(new DataAPIMariaDB('http://database-proxy/v1', 'test-secret', 'default'), new Cache(new None())); + $database = new Database(new MariaDBProxy('http://database-proxy/v1', 'test-secret', 'default'), new Cache(new None())); $database->setDatabase('utopiaTests'); $database->setNamespace(static::$namespace = 'myapp_' . uniqid()); From 30d178a98ae3602c9bda9b208e78424313553719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 29 Feb 2024 13:55:22 +0000 Subject: [PATCH 35/42] Fix proxy test fail --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f07a82b8d..3dbd192eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,8 +99,6 @@ services: environment: - MYSQL_ROOT_PASSWORD=password - MARIADB_DATABASE=utopiaTests - ports: - - "8770:3306" networks: database: From 29f4fc47e61f126bf6dce64a56eb4d9f52933bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 29 Feb 2024 14:10:56 +0000 Subject: [PATCH 36/42] Add debug msg --- .github/workflows/tests.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d99c6c3a2..98502c2d3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -98,4 +98,8 @@ jobs: sleep 10 - name: Run ${{matrix.service}} Tests - run: docker compose exec -T tests vendor/bin/phpunit /usr/src/code/tests/e2e/Adapter/${{matrix.adapter}}Test.php --debug \ No newline at end of file + run: docker compose exec -T tests vendor/bin/phpunit /usr/src/code/tests/e2e/Adapter/${{matrix.adapter}}Test.php --debug + + - name: Print logs + run: | + docker compose logs database-proxy \ No newline at end of file From e16da3e882378e458d847f48ada3273418cffc14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Thu, 29 Feb 2024 14:17:34 +0000 Subject: [PATCH 37/42] Move logs in CI/CD --- .github/workflows/tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 98502c2d3..c456e468f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -97,9 +97,9 @@ jobs: docker compose up -d sleep 10 - - name: Run ${{matrix.service}} Tests - run: docker compose exec -T tests vendor/bin/phpunit /usr/src/code/tests/e2e/Adapter/${{matrix.adapter}}Test.php --debug - - name: Print logs run: | - docker compose logs database-proxy \ No newline at end of file + docker compose logs database-proxy + + - name: Run ${{matrix.service}} Tests + run: docker compose exec -T tests vendor/bin/phpunit /usr/src/code/tests/e2e/Adapter/${{matrix.adapter}}Test.php --debug From 1c95a5064e731278261a5fb1e751bb83a92b4f4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 1 Mar 2024 09:06:07 +0000 Subject: [PATCH 38/42] Remove logs from CICD --- .github/workflows/tests.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c456e468f..be900d656 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -97,9 +97,5 @@ jobs: docker compose up -d sleep 10 - - name: Print logs - run: | - docker compose logs database-proxy - - name: Run ${{matrix.service}} Tests run: docker compose exec -T tests vendor/bin/phpunit /usr/src/code/tests/e2e/Adapter/${{matrix.adapter}}Test.php --debug From c3c874aee3c88d7ab5f4cea12dddeafc6bf04844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 1 Mar 2024 09:29:09 +0000 Subject: [PATCH 39/42] Attempt to get logs after failed error --- .github/workflows/tests.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index be900d656..595ac4d4a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -98,4 +98,10 @@ jobs: sleep 10 - name: Run ${{matrix.service}} Tests - run: docker compose exec -T tests vendor/bin/phpunit /usr/src/code/tests/e2e/Adapter/${{matrix.adapter}}Test.php --debug + run: docker compose exec -T tests vendor/bin/phpunit /usr/src/code/tests/e2e/Adapter/${{matrix.adapter}}Test.php --debug || true + + - name: Print logs + run: | + docker compose ls + docker compose logs database-proxy + From 0c3b92cadb319941f8bddcd71244e5849d4e9081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 1 Mar 2024 09:33:41 +0000 Subject: [PATCH 40/42] More cicd logs --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 595ac4d4a..a26d386c5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -103,5 +103,6 @@ jobs: - name: Print logs run: | docker compose ls + docker compose ps docker compose logs database-proxy From 5ddce3ea3b8f29dadcd18340b09e32f630285f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 1 Mar 2024 09:34:27 +0000 Subject: [PATCH 41/42] Increase sleep time --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a26d386c5..5313ea3f0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -95,7 +95,7 @@ jobs: run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose up -d - sleep 10 + sleep 30 - name: Run ${{matrix.service}} Tests run: docker compose exec -T tests vendor/bin/phpunit /usr/src/code/tests/e2e/Adapter/${{matrix.adapter}}Test.php --debug || true From 5a6a4bee0d3d3dedf44901b8b8800792dc3bb1aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Ba=C4=8Do?= Date: Fri, 1 Mar 2024 10:13:16 +0000 Subject: [PATCH 42/42] Revert CICD changes --- .github/workflows/tests.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5313ea3f0..e69d87595 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -95,14 +95,8 @@ jobs: run: | docker load --input /tmp/${{ env.IMAGE }}.tar docker compose up -d - sleep 30 + sleep 10 - name: Run ${{matrix.service}} Tests run: docker compose exec -T tests vendor/bin/phpunit /usr/src/code/tests/e2e/Adapter/${{matrix.adapter}}Test.php --debug || true - - name: Print logs - run: | - docker compose ls - docker compose ps - docker compose logs database-proxy -