From 03e893fa8615d8266b8e8f93808784761f405114 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 4 Sep 2023 11:30:52 +0300 Subject: [PATCH 01/18] s3 scan dir --- src/Storage/Device/S3.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 625056b5..1c342c8a 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -656,12 +656,13 @@ public function getPartitionTotalSpace(): float /** * Get all files and directories inside a directory. * - * @param string $dir Directory to scan + * @param string $dir Directory to scan * @return string[] + * @throws Exception */ public function getFiles(string $dir): array { - throw new Exception('Not implemented.'); + return $this->listObjects($dir); } /** From 7ecad9daaeb2b1e11d5a281f7a80c91193d5a165 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 4 Sep 2023 14:50:27 +0300 Subject: [PATCH 02/18] s3 scan test --- composer.lock | 108 +++++++++++++++++++------------------- src/Storage/Device.php | 4 +- src/Storage/Device/S3.php | 12 +++-- tests/Storage/S3Base.php | 17 ++++++ 4 files changed, 80 insertions(+), 61 deletions(-) diff --git a/composer.lock b/composer.lock index 54da72ab..4e63d44b 100644 --- a/composer.lock +++ b/composer.lock @@ -74,16 +74,16 @@ }, { "name": "utopia-php/framework", - "version": "0.30.0", + "version": "0.31.0", "source": { "type": "git", "url": "https://github.com/utopia-php/framework.git", - "reference": "0969d429997996ceade53e477fce31221b415651" + "reference": "207f77378965fca9a9bc3783ea379d3549f86bc0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/framework/zipball/0969d429997996ceade53e477fce31221b415651", - "reference": "0969d429997996ceade53e477fce31221b415651", + "url": "https://api.github.com/repos/utopia-php/framework/zipball/207f77378965fca9a9bc3783ea379d3549f86bc0", + "reference": "207f77378965fca9a9bc3783ea379d3549f86bc0", "shasum": "" }, "require": { @@ -113,22 +113,22 @@ ], "support": { "issues": "https://github.com/utopia-php/framework/issues", - "source": "https://github.com/utopia-php/framework/tree/0.30.0" + "source": "https://github.com/utopia-php/framework/tree/0.31.0" }, - "time": "2023-08-07T07:55:48+00:00" + "time": "2023-08-30T16:10:04+00:00" }, { "name": "utopia-php/system", - "version": "0.7.0", + "version": "0.7.1", "source": { "type": "git", "url": "https://github.com/utopia-php/system.git", - "reference": "d385e9521ca3f68aa007ec1cadd11c4dbd871b90" + "reference": "01bf0d283aded0ee0a7a6e5ff540acf64270ab27" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/system/zipball/d385e9521ca3f68aa007ec1cadd11c4dbd871b90", - "reference": "d385e9521ca3f68aa007ec1cadd11c4dbd871b90", + "url": "https://api.github.com/repos/utopia-php/system/zipball/01bf0d283aded0ee0a7a6e5ff540acf64270ab27", + "reference": "01bf0d283aded0ee0a7a6e5ff540acf64270ab27", "shasum": "" }, "require": { @@ -170,9 +170,9 @@ ], "support": { "issues": "https://github.com/utopia-php/system/issues", - "source": "https://github.com/utopia-php/system/tree/0.7.0" + "source": "https://github.com/utopia-php/system/tree/0.7.1" }, - "time": "2023-08-07T09:34:26+00:00" + "time": "2023-08-30T09:14:37+00:00" } ], "packages-dev": [ @@ -422,12 +422,12 @@ "source": { "type": "git", "url": "https://github.com/composer/semver.git", - "reference": "fa1ec24f0ab1efe642671ec15c51a3ab879f59bf" + "reference": "1d09200268e7d1052ded8e5da9c73c96a63d18f5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/semver/zipball/fa1ec24f0ab1efe642671ec15c51a3ab879f59bf", - "reference": "fa1ec24f0ab1efe642671ec15c51a3ab879f59bf", + "url": "https://api.github.com/repos/composer/semver/zipball/1d09200268e7d1052ded8e5da9c73c96a63d18f5", + "reference": "1d09200268e7d1052ded8e5da9c73c96a63d18f5", "shasum": "" }, "require": { @@ -496,7 +496,7 @@ "type": "tidelift" } ], - "time": "2023-01-13T15:47:53+00:00" + "time": "2023-08-31T12:20:31+00:00" }, { "name": "composer/xdebug-handler", @@ -653,12 +653,12 @@ "source": { "type": "git", "url": "https://github.com/doctrine/instantiator.git", - "reference": "40fc8fcd149f6ccff851e767c3fc5ded4e6a96c0" + "reference": "f49f6a836a816609c853718ba2ea422dc18a8e4a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/instantiator/zipball/40fc8fcd149f6ccff851e767c3fc5ded4e6a96c0", - "reference": "40fc8fcd149f6ccff851e767c3fc5ded4e6a96c0", + "url": "https://api.github.com/repos/doctrine/instantiator/zipball/f49f6a836a816609c853718ba2ea422dc18a8e4a", + "reference": "f49f6a836a816609c853718ba2ea422dc18a8e4a", "shasum": "" }, "require": { @@ -716,7 +716,7 @@ "type": "tidelift" } ], - "time": "2023-07-30T10:00:15+00:00" + "time": "2023-08-24T20:23:35+00:00" }, { "name": "felixfbecker/advanced-json-rpc", @@ -938,12 +938,12 @@ "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "cfc54e30a4f5e5af5f9d9ce86697cfcc5f7e7c24" + "reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/cfc54e30a4f5e5af5f9d9ce86697cfcc5f7e7c24", - "reference": "cfc54e30a4f5e5af5f9d9ce86697cfcc5f7e7c24", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d", + "reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d", "shasum": "" }, "require": { @@ -985,9 +985,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/4.x" + "source": "https://github.com/nikic/PHP-Parser/tree/v4.17.1" }, - "time": "2023-07-30T21:38:32+00:00" + "time": "2023-08-13T19:53:39+00:00" }, { "name": "openlss/lib-array2xml", @@ -1284,12 +1284,12 @@ "source": { "type": "git", "url": "https://github.com/phpDocumentor/TypeResolver.git", - "reference": "07100e65d09fd50608d649fc656cae1c921a2495" + "reference": "3219c6ee25c9ea71e3d9bbaf39c67c9ebd499419" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/07100e65d09fd50608d649fc656cae1c921a2495", - "reference": "07100e65d09fd50608d649fc656cae1c921a2495", + "url": "https://api.github.com/repos/phpDocumentor/TypeResolver/zipball/3219c6ee25c9ea71e3d9bbaf39c67c9ebd499419", + "reference": "3219c6ee25c9ea71e3d9bbaf39c67c9ebd499419", "shasum": "" }, "require": { @@ -1333,9 +1333,9 @@ "description": "A PSR-5 based resolver of Class names, Types and Structural Element Names", "support": { "issues": "https://github.com/phpDocumentor/TypeResolver/issues", - "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.x" + "source": "https://github.com/phpDocumentor/TypeResolver/tree/1.7.3" }, - "time": "2023-07-20T19:57:33+00:00" + "time": "2023-08-12T11:01:26+00:00" }, { "name": "phpstan/phpdoc-parser", @@ -1343,12 +1343,12 @@ "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "846ae76eef31c6d7790fac9bc399ecee45160b26" + "reference": "e540adcd37d57d1f46e2b6e172d6140d005226cb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/846ae76eef31c6d7790fac9bc399ecee45160b26", - "reference": "846ae76eef31c6d7790fac9bc399ecee45160b26", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/e540adcd37d57d1f46e2b6e172d6140d005226cb", + "reference": "e540adcd37d57d1f46e2b6e172d6140d005226cb", "shasum": "" }, "require": { @@ -1381,9 +1381,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.23.1" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.23.x" }, - "time": "2023-08-03T16:32:59+00:00" + "time": "2023-08-16T11:40:53+00:00" }, { "name": "phpunit/php-code-coverage", @@ -1391,12 +1391,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "9e1baee3a7525530f5600035759ce754a8b3750b" + "reference": "55a161bb93e6a22a21c529515f7ddbb4d489b460" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/9e1baee3a7525530f5600035759ce754a8b3750b", - "reference": "9e1baee3a7525530f5600035759ce754a8b3750b", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/55a161bb93e6a22a21c529515f7ddbb4d489b460", + "reference": "55a161bb93e6a22a21c529515f7ddbb4d489b460", "shasum": "" }, "require": { @@ -1461,7 +1461,7 @@ "type": "github" } ], - "time": "2023-08-02T09:00:29+00:00" + "time": "2023-09-04T05:23:23+00:00" }, { "name": "phpunit/php-file-iterator", @@ -1710,12 +1710,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "6904cfa5f4a6211e759de8ef36fb51fc939a83fa" + "reference": "255265958b65d7c8ac2e301dbdc0a800d8dc7885" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/6904cfa5f4a6211e759de8ef36fb51fc939a83fa", - "reference": "6904cfa5f4a6211e759de8ef36fb51fc939a83fa", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/255265958b65d7c8ac2e301dbdc0a800d8dc7885", + "reference": "255265958b65d7c8ac2e301dbdc0a800d8dc7885", "shasum": "" }, "require": { @@ -1805,7 +1805,7 @@ "type": "tidelift" } ], - "time": "2023-08-02T08:56:58+00:00" + "time": "2023-09-04T05:22:44+00:00" }, { "name": "psr/container", @@ -2881,12 +2881,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "b504a3d266ad2bb632f196c0936ef2af5ff6e273" + "reference": "f4f71842f24c2023b91237c72a365306f3c58827" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/b504a3d266ad2bb632f196c0936ef2af5ff6e273", - "reference": "b504a3d266ad2bb632f196c0936ef2af5ff6e273", + "url": "https://api.github.com/repos/symfony/console/zipball/f4f71842f24c2023b91237c72a365306f3c58827", + "reference": "f4f71842f24c2023b91237c72a365306f3c58827", "shasum": "" }, "require": { @@ -2972,7 +2972,7 @@ "type": "tidelift" } ], - "time": "2023-07-19T20:11:33+00:00" + "time": "2023-08-07T06:12:30+00:00" }, { "name": "symfony/deprecation-contracts", @@ -3107,7 +3107,7 @@ "portable" ], "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/1.x" + "source": "https://github.com/symfony/polyfill-ctype/tree/v1.28.0" }, "funding": [ { @@ -3189,7 +3189,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/1.x" + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.28.0" }, "funding": [ { @@ -3274,7 +3274,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/1.x" + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.28.0" }, "funding": [ { @@ -3358,7 +3358,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/1.x" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.28.0" }, "funding": [ { @@ -3438,7 +3438,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php73/tree/1.x" + "source": "https://github.com/symfony/polyfill-php73/tree/v1.28.0" }, "funding": [ { @@ -3522,7 +3522,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/1.x" + "source": "https://github.com/symfony/polyfill-php80/tree/v1.28.0" }, "funding": [ { @@ -4036,5 +4036,5 @@ "php": ">=8.0" }, "platform-dev": [], - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.2.0" } diff --git a/src/Storage/Device.php b/src/Storage/Device.php index fdc73483..79c285fb 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -270,8 +270,8 @@ abstract public function getPartitionTotalSpace(): float; /** * Get all files and directories inside a directory. * - * @param string $dir Directory to scan - * @return string[] + * @param string $dir Directory to scan + * @return array */ abstract public function getFiles(string $dir): array; diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 1c342c8a..f2262382 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -474,12 +474,14 @@ public function delete(string $path, bool $recursive = false): bool /** * Get list of objects in the given path. * - * @param string $path + * @param string $prefix + * @param int $maxKeys + * @param string $continuationToken * @return array * - * @throws \Exception + * @throws Exception */ - private function listObjects($prefix = '', $maxKeys = 1000, $continuationToken = '') + private function listObjects(string $prefix = '', int $maxKeys = 1000, string $continuationToken = ''): array { $uri = '/'; $prefix = ltrim($prefix, '/'); /** S3 specific requirement that prefix should never contain a leading slash */ @@ -657,12 +659,12 @@ public function getPartitionTotalSpace(): float * Get all files and directories inside a directory. * * @param string $dir Directory to scan - * @return string[] + * @return array * @throws Exception */ public function getFiles(string $dir): array { - return $this->listObjects($dir); + return $this->listObjects($dir, 9999); } /** diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index a06c8aa9..729ed7c1 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -57,6 +57,23 @@ public function tearDown(): void $this->removeTestFiles(); } + public function testGetFiles() + { + $path = $this->object->getPath('testing/'); + $files = $this->object->getFiles($path); + $this->assertEquals('4', $files['KeyCount']); + $this->assertEquals('false', $files['IsTruncated']); + $this->assertIsArray($files['Contents']); + + $file = $files['Contents'][0]; + + $this->assertArrayHasKey('LastModified', $file); + $this->assertArrayHasKey('ETag', $file); + $this->assertArrayHasKey('Size', $file); + $this->assertArrayHasKey('StorageClass', $file); + $this->assertArrayHasKey('Type', $file); + } + public function testName() { $this->assertEquals($this->getAdapterName(), $this->object->getName()); From 43ca43bda0677bce846d1d263f66e8b601b90ce6 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 4 Sep 2023 14:52:12 +0300 Subject: [PATCH 03/18] s3 scan test --- src/Storage/Device.php | 2 +- src/Storage/Device/S3.php | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Storage/Device.php b/src/Storage/Device.php index 79c285fb..74617c9e 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -270,7 +270,7 @@ abstract public function getPartitionTotalSpace(): float; /** * Get all files and directories inside a directory. * - * @param string $dir Directory to scan + * @param string $dir Directory to scan * @return array */ abstract public function getFiles(string $dir): array; diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index f2262382..3582e514 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -474,9 +474,9 @@ public function delete(string $path, bool $recursive = false): bool /** * Get list of objects in the given path. * - * @param string $prefix - * @param int $maxKeys - * @param string $continuationToken + * @param string $prefix + * @param int $maxKeys + * @param string $continuationToken * @return array * * @throws Exception @@ -658,8 +658,9 @@ public function getPartitionTotalSpace(): float /** * Get all files and directories inside a directory. * - * @param string $dir Directory to scan + * @param string $dir Directory to scan * @return array + * * @throws Exception */ public function getFiles(string $dir): array From 7831cc061f4be65f72444ecf127e1293da1839f7 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 4 Sep 2023 16:29:51 +0300 Subject: [PATCH 04/18] Remove type test --- tests/Storage/S3Base.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index 729ed7c1..1e3ba992 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -67,11 +67,11 @@ public function testGetFiles() $file = $files['Contents'][0]; + $this->assertArrayHasKey('Key', $file); $this->assertArrayHasKey('LastModified', $file); $this->assertArrayHasKey('ETag', $file); $this->assertArrayHasKey('Size', $file); $this->assertArrayHasKey('StorageClass', $file); - $this->assertArrayHasKey('Type', $file); } public function testName() From 18d00433048a977981115f80f17487fdbbe68884 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 4 Sep 2023 16:37:06 +0300 Subject: [PATCH 05/18] Rerun tests --- tests/Storage/S3Base.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index 1e3ba992..3dbfe3e4 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -70,8 +70,8 @@ public function testGetFiles() $this->assertArrayHasKey('Key', $file); $this->assertArrayHasKey('LastModified', $file); $this->assertArrayHasKey('ETag', $file); - $this->assertArrayHasKey('Size', $file); $this->assertArrayHasKey('StorageClass', $file); + $this->assertArrayHasKey('Size', $file); } public function testName() From ec38203edf42752cbaaf8ccf0c8762ae0a8033e3 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 4 Sep 2023 17:19:07 +0300 Subject: [PATCH 06/18] Fix types --- src/Storage/Device/S3.php | 5 ++++- tests/Storage/S3Base.php | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 3582e514..0841bc8f 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -665,7 +665,10 @@ public function getPartitionTotalSpace(): float */ public function getFiles(string $dir): array { - return $this->listObjects($dir, 9999); + $data = $this->listObjects($dir); + $data['IsTruncated'] = $data['IsTruncated'] === 'true'; + $data['KeyCount'] = intval($data['KeyCount']); + return $data; } /** diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index 3dbfe3e4..b8cd1a8d 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -61,8 +61,8 @@ public function testGetFiles() { $path = $this->object->getPath('testing/'); $files = $this->object->getFiles($path); - $this->assertEquals('4', $files['KeyCount']); - $this->assertEquals('false', $files['IsTruncated']); + $this->assertEquals(4, $files['KeyCount']); + $this->assertEquals(false, $files['IsTruncated']); $this->assertIsArray($files['Contents']); $file = $files['Contents'][0]; From 76e440f1374bf23bb82f28701786af1a5a3b2d05 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 5 Sep 2023 11:27:10 +0300 Subject: [PATCH 07/18] formatting --- src/Storage/Device/S3.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 0841bc8f..caf50c38 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -668,6 +668,7 @@ public function getFiles(string $dir): array $data = $this->listObjects($dir); $data['IsTruncated'] = $data['IsTruncated'] === 'true'; $data['KeyCount'] = intval($data['KeyCount']); + return $data; } From d116ea2db84cfdd61207e416c17064889bd3a37c Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 18 Sep 2023 12:57:11 +0300 Subject: [PATCH 08/18] S3 pagination --- src/Storage/Device.php | 8 +++++--- src/Storage/Device/Local.php | 2 +- src/Storage/Device/S3.php | 19 +++++++++++++++---- tests/Storage/S3Base.php | 19 +++++++++++++++++++ 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/Storage/Device.php b/src/Storage/Device.php index 74617c9e..f71b524b 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -270,10 +270,12 @@ abstract public function getPartitionTotalSpace(): float; /** * Get all files and directories inside a directory. * - * @param string $dir Directory to scan - * @return array + * @param string $dir Directory to scan + * @param int $keys + * @param string $continuationToken + * @return array */ - abstract public function getFiles(string $dir): array; + abstract public function getFiles(string $dir, int $keys = 1000, string $continuationToken = ''): array; /** * Get the absolute path by resolving strings like ../, .., //, /\ and so on. diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index 6ef23b63..294c3ea7 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -507,7 +507,7 @@ public function getPartitionTotalSpace(): float * @param string $dir Directory to scan * @return string[] */ - public function getFiles(string $dir): array + public function getFiles(string $dir, int $keys = 1000, string $continuationToken = ''): array { if (! (\str_ends_with($dir, DIRECTORY_SEPARATOR))) { $dir .= DIRECTORY_SEPARATOR; diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index caf50c38..1f994862 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -658,24 +658,35 @@ public function getPartitionTotalSpace(): float /** * Get all files and directories inside a directory. * - * @param string $dir Directory to scan - * @return array + * @param string $dir Directory to scan + * @param int $keys + * @param string $continuationToken + * @return array * * @throws Exception */ - public function getFiles(string $dir): array + public function getFiles(string $dir, int $keys = 1000, string $continuationToken = ''): array { - $data = $this->listObjects($dir); + $data = $this->listObjects($dir, $keys, $continuationToken); + + // Set to false if all the results were returned. Set to true if more keys are available to return. $data['IsTruncated'] = $data['IsTruncated'] === 'true'; + + // KeyCount is the number of keys returned with this request. $data['KeyCount'] = intval($data['KeyCount']); + // Sets the maximum number of keys returned to the response. By default, the action returns up to 1,000 key names. + $data['MaxKeys'] = intval($data['MaxKeys']); + return $data; } /** * Get file info * + * @param string $path * @return array + * @throws Exception */ private function getInfo(string $path): array { diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index b8cd1a8d..3857eca8 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -74,6 +74,25 @@ public function testGetFiles() $this->assertArrayHasKey('Size', $file); } + public function testGetFilesPagination() + { + $path = $this->object->getPath('testing/'); + + $files = $this->object->getFiles($path, 3); + $this->assertEquals(3, $files['KeyCount']); + $this->assertEquals(1000, $files['MaxKeys']); + $this->assertEquals(true, $files['IsTruncated']); + $this->assertIsArray($files['Contents']); + $this->assertNotEmpty($files['NextContinuationToken']); + + $files = $this->object->getFiles($path, 3, $files['NextContinuationToken']); + $this->assertEquals(1, $files['KeyCount']); + $this->assertEquals(1000, $files['MaxKeys']); + $this->assertEquals(false, $files['IsTruncated']); + $this->assertIsArray($files['Contents']); + $this->assertEmpty($files['NextContinuationToken']); + } + public function testName() { $this->assertEquals($this->getAdapterName(), $this->object->getName()); From abdb8f4e9ee831c1d50aba5730dd1bbebf60794a Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 18 Sep 2023 14:37:02 +0300 Subject: [PATCH 09/18] max keys --- tests/Storage/S3Base.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index 3857eca8..4e3531b3 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -80,12 +80,12 @@ public function testGetFilesPagination() $files = $this->object->getFiles($path, 3); $this->assertEquals(3, $files['KeyCount']); - $this->assertEquals(1000, $files['MaxKeys']); + $this->assertEquals(3, $files['MaxKeys']); $this->assertEquals(true, $files['IsTruncated']); $this->assertIsArray($files['Contents']); $this->assertNotEmpty($files['NextContinuationToken']); - $files = $this->object->getFiles($path, 3, $files['NextContinuationToken']); + $files = $this->object->getFiles($path, 1000, $files['NextContinuationToken']); $this->assertEquals(1, $files['KeyCount']); $this->assertEquals(1000, $files['MaxKeys']); $this->assertEquals(false, $files['IsTruncated']); From d4a6d4658fb37871d1bc2d645e9ec139cbad5dbb Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 18 Sep 2023 14:59:00 +0300 Subject: [PATCH 10/18] NextContinuationToken --- src/Storage/Device/S3.php | 4 +++- tests/Storage/S3Base.php | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 1f994862..26522ef8 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -493,9 +493,11 @@ private function listObjects(string $prefix = '', int $maxKeys = 1000, string $c 'prefix' => $prefix, 'max-keys' => $maxKeys, ]; - if (! empty($continuationToken)) { + + if (!empty($continuationToken)) { $parameters['continuation-token'] = $continuationToken; } + $response = $this->call(self::METHOD_GET, $uri, '', $parameters); return $response->body; diff --git a/tests/Storage/S3Base.php b/tests/Storage/S3Base.php index 4e3531b3..b9228712 100644 --- a/tests/Storage/S3Base.php +++ b/tests/Storage/S3Base.php @@ -83,14 +83,14 @@ public function testGetFilesPagination() $this->assertEquals(3, $files['MaxKeys']); $this->assertEquals(true, $files['IsTruncated']); $this->assertIsArray($files['Contents']); - $this->assertNotEmpty($files['NextContinuationToken']); + $this->assertArrayHasKey('NextContinuationToken', $files); $files = $this->object->getFiles($path, 1000, $files['NextContinuationToken']); $this->assertEquals(1, $files['KeyCount']); $this->assertEquals(1000, $files['MaxKeys']); $this->assertEquals(false, $files['IsTruncated']); $this->assertIsArray($files['Contents']); - $this->assertEmpty($files['NextContinuationToken']); + $this->assertArrayNotHasKey('NextContinuationToken', $files); } public function testName() From c9e972ea76d6cb584eea761852197ae926183824 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 18 Sep 2023 16:51:23 +0300 Subject: [PATCH 11/18] $maxKeys limit --- src/Storage/Device/S3.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 26522ef8..e70b2dc1 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -483,6 +483,10 @@ public function delete(string $path, bool $recursive = false): bool */ private function listObjects(string $prefix = '', int $maxKeys = 1000, string $continuationToken = ''): array { + if($maxKeys > 1000){ + throw new Exception('max-keys limit is 1000'); + } + $uri = '/'; $prefix = ltrim($prefix, '/'); /** S3 specific requirement that prefix should never contain a leading slash */ $this->headers['content-type'] = 'text/plain'; From a4a19b9610ac4bfe634abd4b707b7cd2987cd7fe Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 18 Sep 2023 17:38:25 +0300 Subject: [PATCH 12/18] Const MAX_KEYS --- src/Storage/Device.php | 9 +++++++-- src/Storage/Device/Local.php | 2 +- src/Storage/Device/S3.php | 6 +++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/Storage/Device.php b/src/Storage/Device.php index f71b524b..2ba7b7c0 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -5,10 +5,15 @@ abstract class Device { /** - * Max chunk size while transfering file from one device to another + * Max chunk size while transferring file from one device to another */ protected int $transferChunkSize = 20000000; //20 MB + /** + * Sets the maximum number of keys returned to the response. By default, the action returns up to 1,000 key names. + */ + protected const MAX_KEYS = 1000; + /** * Set Transfer Chunk Size * @@ -275,7 +280,7 @@ abstract public function getPartitionTotalSpace(): float; * @param string $continuationToken * @return array */ - abstract public function getFiles(string $dir, int $keys = 1000, string $continuationToken = ''): array; + abstract public function getFiles(string $dir, int $keys = self::MAX_KEYS, string $continuationToken = ''): array; /** * Get the absolute path by resolving strings like ../, .., //, /\ and so on. diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index 294c3ea7..15a31b69 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -507,7 +507,7 @@ public function getPartitionTotalSpace(): float * @param string $dir Directory to scan * @return string[] */ - public function getFiles(string $dir, int $keys = 1000, string $continuationToken = ''): array + public function getFiles(string $dir, int $keys = self::MAX_KEYS, string $continuationToken = ''): array { if (! (\str_ends_with($dir, DIRECTORY_SEPARATOR))) { $dir .= DIRECTORY_SEPARATOR; diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index e70b2dc1..bf5da800 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -481,9 +481,9 @@ public function delete(string $path, bool $recursive = false): bool * * @throws Exception */ - private function listObjects(string $prefix = '', int $maxKeys = 1000, string $continuationToken = ''): array + private function listObjects(string $prefix = '', int $maxKeys = self::MAX_KEYS, string $continuationToken = ''): array { - if($maxKeys > 1000){ + if($maxKeys > self::MAX_KEYS){ throw new Exception('max-keys limit is 1000'); } @@ -671,7 +671,7 @@ public function getPartitionTotalSpace(): float * * @throws Exception */ - public function getFiles(string $dir, int $keys = 1000, string $continuationToken = ''): array + public function getFiles(string $dir, int $keys = self::MAX_KEYS, string $continuationToken = ''): array { $data = $this->listObjects($dir, $keys, $continuationToken); From f7a7b6e6f1cd21c68b83d3096380d5af9aa944eb Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 18 Sep 2023 17:39:36 +0300 Subject: [PATCH 13/18] Const MAX_KEYS --- src/Storage/Device/S3.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index bf5da800..79bc2835 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -484,7 +484,7 @@ public function delete(string $path, bool $recursive = false): bool private function listObjects(string $prefix = '', int $maxKeys = self::MAX_KEYS, string $continuationToken = ''): array { if($maxKeys > self::MAX_KEYS){ - throw new Exception('max-keys limit is 1000'); + throw new Exception('max-keys limit is ' . self::MAX_KEYS); } $uri = '/'; From 397afbb79dce7decc952bbfe7686bf6858d3babd Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 19 Sep 2023 16:23:30 +0300 Subject: [PATCH 14/18] Const MAX_KEYS --- src/Storage/Device.php | 6 +++--- src/Storage/Device/S3.php | 15 ++++++++------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/Storage/Device.php b/src/Storage/Device.php index 2ba7b7c0..ea7915ad 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -275,9 +275,9 @@ abstract public function getPartitionTotalSpace(): float; /** * Get all files and directories inside a directory. * - * @param string $dir Directory to scan - * @param int $keys - * @param string $continuationToken + * @param string $dir Directory to scan + * @param int $keys + * @param string $continuationToken * @return array */ abstract public function getFiles(string $dir, int $keys = self::MAX_KEYS, string $continuationToken = ''): array; diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index 79bc2835..d46bdbd5 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -483,8 +483,8 @@ public function delete(string $path, bool $recursive = false): bool */ private function listObjects(string $prefix = '', int $maxKeys = self::MAX_KEYS, string $continuationToken = ''): array { - if($maxKeys > self::MAX_KEYS){ - throw new Exception('max-keys limit is ' . self::MAX_KEYS); + if ($maxKeys > self::MAX_KEYS) { + throw new Exception('max-keys limit is '.self::MAX_KEYS); } $uri = '/'; @@ -498,7 +498,7 @@ private function listObjects(string $prefix = '', int $maxKeys = self::MAX_KEYS, 'max-keys' => $maxKeys, ]; - if (!empty($continuationToken)) { + if (! empty($continuationToken)) { $parameters['continuation-token'] = $continuationToken; } @@ -664,9 +664,9 @@ public function getPartitionTotalSpace(): float /** * Get all files and directories inside a directory. * - * @param string $dir Directory to scan - * @param int $keys - * @param string $continuationToken + * @param string $dir Directory to scan + * @param int $keys + * @param string $continuationToken * @return array * * @throws Exception @@ -690,8 +690,9 @@ public function getFiles(string $dir, int $keys = self::MAX_KEYS, string $contin /** * Get file info * - * @param string $path + * @param string $path * @return array + * * @throws Exception */ private function getInfo(string $path): array From aa7060a474f7363b61b72562a8dc68fedffdb816 Mon Sep 17 00:00:00 2001 From: Shmuel Fogel Date: Thu, 12 Oct 2023 12:05:38 +0300 Subject: [PATCH 15/18] Update src/Storage/Device/S3.php Co-authored-by: Jake Barnby --- src/Storage/Device/S3.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index d46bdbd5..f2243bed 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -484,7 +484,7 @@ public function delete(string $path, bool $recursive = false): bool private function listObjects(string $prefix = '', int $maxKeys = self::MAX_KEYS, string $continuationToken = ''): array { if ($maxKeys > self::MAX_KEYS) { - throw new Exception('max-keys limit is '.self::MAX_KEYS); + throw new Exception('Cannot list more than ' . self::MAX_KEYS . ' objects'); } $uri = '/'; From 0bd69e11152029e1a3bdb83264d174304d78fde7 Mon Sep 17 00:00:00 2001 From: fogelito Date: Thu, 12 Oct 2023 12:14:45 +0300 Subject: [PATCH 16/18] Address comments --- composer.lock | 70 ++++++++++++++++++------------------ src/Storage/Device.php | 6 ++-- src/Storage/Device/Local.php | 10 +++--- src/Storage/Device/S3.php | 14 ++++---- 4 files changed, 50 insertions(+), 50 deletions(-) diff --git a/composer.lock b/composer.lock index 4e63d44b..e597babb 100644 --- a/composer.lock +++ b/composer.lock @@ -605,12 +605,12 @@ "source": { "type": "git", "url": "https://github.com/doctrine/deprecations.git", - "reference": "bdaa697ed9c7f5ee2f7d3b5f9c2a6f2519523cd9" + "reference": "4f2d4f2836e7ec4e7a8625e75c6aa916004db931" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/deprecations/zipball/bdaa697ed9c7f5ee2f7d3b5f9c2a6f2519523cd9", - "reference": "bdaa697ed9c7f5ee2f7d3b5f9c2a6f2519523cd9", + "url": "https://api.github.com/repos/doctrine/deprecations/zipball/4f2d4f2836e7ec4e7a8625e75c6aa916004db931", + "reference": "4f2d4f2836e7ec4e7a8625e75c6aa916004db931", "shasum": "" }, "require": { @@ -643,9 +643,9 @@ "homepage": "https://www.doctrine-project.org/", "support": { "issues": "https://github.com/doctrine/deprecations/issues", - "source": "https://github.com/doctrine/deprecations/tree/1.1.x" + "source": "https://github.com/doctrine/deprecations/tree/1.1.2" }, - "time": "2023-07-29T16:12:19+00:00" + "time": "2023-09-27T20:04:15+00:00" }, { "name": "doctrine/instantiator", @@ -938,12 +938,12 @@ "source": { "type": "git", "url": "https://github.com/nikic/PHP-Parser.git", - "reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d" + "reference": "54103d838734be0499172026525e38cbf6af2711" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d", - "reference": "a6303e50c90c355c7eeee2c4a8b27fe8dc8fef1d", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/54103d838734be0499172026525e38cbf6af2711", + "reference": "54103d838734be0499172026525e38cbf6af2711", "shasum": "" }, "require": { @@ -954,7 +954,6 @@ "ircmaxell/php-yacc": "^0.0.7", "phpunit/phpunit": "^6.5 || ^7.0 || ^8.0 || ^9.0" }, - "default-branch": true, "bin": [ "bin/php-parse" ], @@ -985,9 +984,9 @@ ], "support": { "issues": "https://github.com/nikic/PHP-Parser/issues", - "source": "https://github.com/nikic/PHP-Parser/tree/v4.17.1" + "source": "https://github.com/nikic/PHP-Parser/tree/4.x" }, - "time": "2023-08-13T19:53:39+00:00" + "time": "2023-10-03T21:00:18+00:00" }, { "name": "openlss/lib-array2xml", @@ -1339,16 +1338,16 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "1.23.x-dev", + "version": "1.24.2", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "e540adcd37d57d1f46e2b6e172d6140d005226cb" + "reference": "bcad8d995980440892759db0c32acae7c8e79442" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/e540adcd37d57d1f46e2b6e172d6140d005226cb", - "reference": "e540adcd37d57d1f46e2b6e172d6140d005226cb", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/bcad8d995980440892759db0c32acae7c8e79442", + "reference": "bcad8d995980440892759db0c32acae7c8e79442", "shasum": "" }, "require": { @@ -1365,7 +1364,6 @@ "phpunit/phpunit": "^9.5", "symfony/process": "^5.2" }, - "default-branch": true, "type": "library", "autoload": { "psr-4": { @@ -1381,9 +1379,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/1.23.x" + "source": "https://github.com/phpstan/phpdoc-parser/tree/1.24.2" }, - "time": "2023-08-16T11:40:53+00:00" + "time": "2023-09-26T12:28:12+00:00" }, { "name": "phpunit/php-code-coverage", @@ -1391,12 +1389,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "55a161bb93e6a22a21c529515f7ddbb4d489b460" + "reference": "0b6cb84a359426d34e439ce6d1173b41e07cb672" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/55a161bb93e6a22a21c529515f7ddbb4d489b460", - "reference": "55a161bb93e6a22a21c529515f7ddbb4d489b460", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/0b6cb84a359426d34e439ce6d1173b41e07cb672", + "reference": "0b6cb84a359426d34e439ce6d1173b41e07cb672", "shasum": "" }, "require": { @@ -1461,7 +1459,7 @@ "type": "github" } ], - "time": "2023-09-04T05:23:23+00:00" + "time": "2023-10-06T09:53:04+00:00" }, { "name": "phpunit/php-file-iterator", @@ -1710,12 +1708,12 @@ "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "255265958b65d7c8ac2e301dbdc0a800d8dc7885" + "reference": "18a76cf7293527f275342720eeb063dff346c97e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/255265958b65d7c8ac2e301dbdc0a800d8dc7885", - "reference": "255265958b65d7c8ac2e301dbdc0a800d8dc7885", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/18a76cf7293527f275342720eeb063dff346c97e", + "reference": "18a76cf7293527f275342720eeb063dff346c97e", "shasum": "" }, "require": { @@ -1730,7 +1728,7 @@ "phar-io/manifest": "^2.0.3", "phar-io/version": "^3.0.2", "php": ">=7.3", - "phpunit/php-code-coverage": "^9.2.13", + "phpunit/php-code-coverage": "^9.2.28", "phpunit/php-file-iterator": "^3.0.5", "phpunit/php-invoker": "^3.1.1", "phpunit/php-text-template": "^2.0.3", @@ -1805,7 +1803,7 @@ "type": "tidelift" } ], - "time": "2023-09-04T05:22:44+00:00" + "time": "2023-10-06T09:52:37+00:00" }, { "name": "psr/container", @@ -1813,12 +1811,12 @@ "source": { "type": "git", "url": "https://github.com/php-fig/container.git", - "reference": "90db7b9ac2a2c5b849fcb69dde58f3ae182c68f5" + "reference": "707984727bd5b2b670e59559d3ed2500240cf875" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-fig/container/zipball/90db7b9ac2a2c5b849fcb69dde58f3ae182c68f5", - "reference": "90db7b9ac2a2c5b849fcb69dde58f3ae182c68f5", + "url": "https://api.github.com/repos/php-fig/container/zipball/707984727bd5b2b670e59559d3ed2500240cf875", + "reference": "707984727bd5b2b670e59559d3ed2500240cf875", "shasum": "" }, "require": { @@ -1857,9 +1855,9 @@ ], "support": { "issues": "https://github.com/php-fig/container/issues", - "source": "https://github.com/php-fig/container/tree/master" + "source": "https://github.com/php-fig/container" }, - "time": "2022-07-19T17:36:59+00:00" + "time": "2023-09-22T11:11:30+00:00" }, { "name": "psr/log", @@ -3629,12 +3627,12 @@ "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "fe9228ba417441e16f31d36ddad2b3076135f453" + "reference": "742351f7542c9b9799873e399a716cb5af6f70f2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/fe9228ba417441e16f31d36ddad2b3076135f453", - "reference": "fe9228ba417441e16f31d36ddad2b3076135f453", + "url": "https://api.github.com/repos/symfony/string/zipball/742351f7542c9b9799873e399a716cb5af6f70f2", + "reference": "742351f7542c9b9799873e399a716cb5af6f70f2", "shasum": "" }, "require": { @@ -3707,7 +3705,7 @@ "type": "tidelift" } ], - "time": "2023-07-27T06:52:43+00:00" + "time": "2023-09-18T10:40:25+00:00" }, { "name": "theseer/tokenizer", diff --git a/src/Storage/Device.php b/src/Storage/Device.php index ea7915ad..8f3207fc 100644 --- a/src/Storage/Device.php +++ b/src/Storage/Device.php @@ -12,7 +12,7 @@ abstract class Device /** * Sets the maximum number of keys returned to the response. By default, the action returns up to 1,000 key names. */ - protected const MAX_KEYS = 1000; + protected const MAX_PAGE_SIZE = PHP_INT_MAX; /** * Set Transfer Chunk Size @@ -276,11 +276,11 @@ abstract public function getPartitionTotalSpace(): float; * Get all files and directories inside a directory. * * @param string $dir Directory to scan - * @param int $keys + * @param int $max * @param string $continuationToken * @return array */ - abstract public function getFiles(string $dir, int $keys = self::MAX_KEYS, string $continuationToken = ''): array; + abstract public function getFiles(string $dir, int $max = self::MAX_PAGE_SIZE, string $continuationToken = ''): array; /** * Get the absolute path by resolving strings like ../, .., //, /\ and so on. diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index 15a31b69..1d1570ce 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -501,13 +501,13 @@ public function getPartitionTotalSpace(): float return \disk_total_space($this->getRoot()); } - /** - * Get all files and directories inside a directory. - * - * @param string $dir Directory to scan + /**s + * @param string $dir + * @param int $max + * @param string $continuationToken * @return string[] */ - public function getFiles(string $dir, int $keys = self::MAX_KEYS, string $continuationToken = ''): array + public function getFiles(string $dir, int $max = self::MAX_PAGE_SIZE, string $continuationToken = ''): array { if (! (\str_ends_with($dir, DIRECTORY_SEPARATOR))) { $dir .= DIRECTORY_SEPARATOR; diff --git a/src/Storage/Device/S3.php b/src/Storage/Device/S3.php index f2243bed..79aadeff 100644 --- a/src/Storage/Device/S3.php +++ b/src/Storage/Device/S3.php @@ -92,6 +92,8 @@ class S3 extends Device const ACL_AUTHENTICATED_READ = 'authenticated-read'; + protected const MAX_PAGE_SIZE = 1000; + /** * @var string */ @@ -481,10 +483,10 @@ public function delete(string $path, bool $recursive = false): bool * * @throws Exception */ - private function listObjects(string $prefix = '', int $maxKeys = self::MAX_KEYS, string $continuationToken = ''): array + private function listObjects(string $prefix = '', int $maxKeys = self::MAX_PAGE_SIZE, string $continuationToken = ''): array { - if ($maxKeys > self::MAX_KEYS) { - throw new Exception('Cannot list more than ' . self::MAX_KEYS . ' objects'); + if ($maxKeys > self::MAX_PAGE_SIZE) { + throw new Exception('Cannot list more than '.self::MAX_PAGE_SIZE.' objects'); } $uri = '/'; @@ -665,15 +667,15 @@ public function getPartitionTotalSpace(): float * Get all files and directories inside a directory. * * @param string $dir Directory to scan - * @param int $keys + * @param int $max * @param string $continuationToken * @return array * * @throws Exception */ - public function getFiles(string $dir, int $keys = self::MAX_KEYS, string $continuationToken = ''): array + public function getFiles(string $dir, int $max = self::MAX_PAGE_SIZE, string $continuationToken = ''): array { - $data = $this->listObjects($dir, $keys, $continuationToken); + $data = $this->listObjects($dir, $max, $continuationToken); // Set to false if all the results were returned. Set to true if more keys are available to return. $data['IsTruncated'] = $data['IsTruncated'] === 'true'; From cf00a56b278ffc2f36944419dbe96f24a5bf9686 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 18 Oct 2023 21:17:36 +1300 Subject: [PATCH 17/18] Update src/Storage/Device/Local.php --- src/Storage/Device/Local.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index 1d1570ce..c39f553f 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -501,7 +501,9 @@ public function getPartitionTotalSpace(): float return \disk_total_space($this->getRoot()); } - /**s + /** + * Get all files and directories inside a directory. + * * @param string $dir * @param int $max * @param string $continuationToken From 39a8f7b6670e46052a1a6328bdc7483b3baf7fa5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 18 Oct 2023 21:23:18 +1300 Subject: [PATCH 18/18] Fix format --- src/Storage/Device/Local.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Storage/Device/Local.php b/src/Storage/Device/Local.php index 372f1bcf..65149917 100644 --- a/src/Storage/Device/Local.php +++ b/src/Storage/Device/Local.php @@ -510,9 +510,9 @@ public function getPartitionTotalSpace(): float /** * Get all files and directories inside a directory. * - * @param string $dir - * @param int $max - * @param string $continuationToken + * @param string $dir + * @param int $max + * @param string $continuationToken * @return string[] */ public function getFiles(string $dir, int $max = self::MAX_PAGE_SIZE, string $continuationToken = ''): array