From 3a0610e12dc7f747670a536e514bfd44769913e5 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 25 Aug 2023 09:31:46 +0100 Subject: [PATCH 1/4] Fix Deployment Transfers --- playground.php | 22 +++++++++++----------- src/Migration/Cache.php | 4 ++-- src/Migration/Destinations/Local.php | 4 ++-- src/Migration/Resource.php | 10 ++++++---- src/Migration/Resources/Functions/Func.php | 18 +++++++++++++++++- src/Migration/Sources/Appwrite.php | 15 ++++++++++----- 6 files changed, 48 insertions(+), 25 deletions(-) diff --git a/playground.php b/playground.php index 91129932..4c46bc08 100644 --- a/playground.php +++ b/playground.php @@ -45,14 +45,14 @@ $_ENV['NHOST_TEST_PASSWORD'] ?? '', ); -$sourceSupabase = new Supabase( - $_ENV['SUPABASE_TEST_ENDPOINT'] ?? '', - $_ENV['SUPABASE_TEST_KEY'] ?? '', - $_ENV['SUPABASE_TEST_HOST'] ?? '', - $_ENV['SUPABASE_TEST_DATABASE'] ?? '', - $_ENV['SUPABASE_TEST_USERNAME'] ?? '', - $_ENV['SUPABASE_TEST_PASSWORD'] ?? '', -); +// $sourceSupabase = new Supabase( +// $_ENV['SUPABASE_TEST_ENDPOINT'] ?? '', +// $_ENV['SUPABASE_TEST_KEY'] ?? '', +// $_ENV['SUPABASE_TEST_HOST'] ?? '', +// $_ENV['SUPABASE_TEST_DATABASE'] ?? '', +// $_ENV['SUPABASE_TEST_USERNAME'] ?? '', +// $_ENV['SUPABASE_TEST_PASSWORD'] ?? '', +// ); /** * Initialise All Destination Adapters @@ -69,16 +69,16 @@ * Initialise Transfer Class */ $transfer = new Transfer( - $sourceFirebase, + $sourceAppwrite, $destinationLocal ); -$sourceFirebase->report(); +$sourceAppwrite->report(); // /** // * Run Transfer // */ -$transfer->run($sourceFirebase->getSupportedResources(), +$transfer->run(Appwrite::getSupportedResources(), function (array $resources) { } ); diff --git a/src/Migration/Cache.php b/src/Migration/Cache.php index b25545da..d3307c09 100644 --- a/src/Migration/Cache.php +++ b/src/Migration/Cache.php @@ -37,8 +37,8 @@ public function add($resource) $resource->setInternalId(uniqid()); } - if ($resource->getName() == Resource::TYPE_FILE || $resource->getName() == Resource::TYPE_FUNCTION) { - /** @var File|Func $resource */ + if ($resource->getName() == Resource::TYPE_FILE || $resource->getName() == Resource::TYPE_DEPLOYMENT) { + /** @var File|Deployment $resource */ $resource->setData(''); // Prevent Memory Leak } diff --git a/src/Migration/Destinations/Local.php b/src/Migration/Destinations/Local.php index 11156d89..b2201e50 100644 --- a/src/Migration/Destinations/Local.php +++ b/src/Migration/Destinations/Local.php @@ -104,7 +104,7 @@ protected function import(array $resources, callable $callback): void case Resource::TYPE_DEPLOYMENT: /** @var Deployment $resource */ if ($resource->getStart() === 0) { - $this->data[$resource->getGroup()][$resource->getName()][$resource->getInternalId()] = $resource->asArray(); + $this->data[$resource->getGroup()][$resource->getName()][] = $resource->asArray(); } file_put_contents($this->path.'deployments/'.$resource->getId().'.tar.gz', $resource->getData(), FILE_APPEND); @@ -135,7 +135,7 @@ protected function import(array $resources, callable $callback): void $resource->setData(''); break; default: - $this->data[$resource->getGroup()][$resource->getName()][$resource->getInternalId()] = $resource->asArray(); + $this->data[$resource->getGroup()][$resource->getName()][]= $resource->asArray(); break; } diff --git a/src/Migration/Resource.php b/src/Migration/Resource.php index bd294fa2..825da5c2 100644 --- a/src/Migration/Resource.php +++ b/src/Migration/Resource.php @@ -44,16 +44,18 @@ abstract class Resource public const TYPE_INDEX = 'index'; + public const TYPE_INACTIVE_DEPLOYMENTS = 'inactive-deployments'; + + public const TYPE_DEPLOYMENT = 'deployment'; + + public const TYPE_ENVVAR = 'envvar'; + // Children (Resources that are created by other resources) public const TYPE_ATTRIBUTE = 'attribute'; - public const TYPE_DEPLOYMENT = 'deployment'; - public const TYPE_HASH = 'hash'; - public const TYPE_ENVVAR = 'envvar'; - public const ALL_RESOURCES = [ self::TYPE_ATTRIBUTE, self::TYPE_BUCKET, diff --git a/src/Migration/Resources/Functions/Func.php b/src/Migration/Resources/Functions/Func.php index e6760bef..840e1ff4 100644 --- a/src/Migration/Resources/Functions/Func.php +++ b/src/Migration/Resources/Functions/Func.php @@ -21,7 +21,9 @@ class Func extends Resource protected int $timeout; - public function __construct(string $name, string $id, string $runtime, array $execute = [], bool $enabled = true, array $events = [], string $schedule = '', int $timeout = 0) + protected string $deployment; + + public function __construct(string $name, string $id, string $runtime, array $execute = [], bool $enabled = true, array $events = [], string $schedule = '', int $timeout = 0, string $deployment = '') { $this->name = $name; $this->id = $id; @@ -31,6 +33,7 @@ public function __construct(string $name, string $id, string $runtime, array $ex $this->events = $events; $this->schedule = $schedule; $this->timeout = $timeout; + $this->deployment = $deployment; } public static function getName(): string @@ -120,6 +123,18 @@ public function setTimeout(int $timeout): self return $this; } + public function getDeployment(): string + { + return $this->deployment; + } + + public function setDeployment(string $deployment): self + { + $this->deployment = $deployment; + + return $this; + } + public function asArray(): array { return [ @@ -131,6 +146,7 @@ public function asArray(): array 'events' => $this->events, 'schedule' => $this->schedule, 'timeout' => $this->timeout, + 'deployment' => $this->deployment ]; } } diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index bc4a9b0d..fa2c7e2b 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -991,7 +991,7 @@ protected function exportGroupFunctions(int $batchSize, array $resources) } if (in_array(Resource::TYPE_DEPLOYMENT, $resources)) { - $this->exportDeployments($batchSize); + $this->exportDeployments($batchSize, in_array(Resource::TYPE_INACTIVE_DEPLOYMENTS, $resources)); } } @@ -1016,7 +1016,8 @@ private function exportFunctions(int $batchSize) $function['enabled'], $function['events'], $function['schedule'], - $function['timeout'] + $function['timeout'], + $function['deployment'] ); $convertedResources[] = $convertedFunc; @@ -1033,7 +1034,7 @@ private function exportFunctions(int $batchSize) $this->callback($convertedResources); } - private function exportDeployments(int $batchSize) + private function exportDeployments(int $batchSize, bool $includeInactive = false) { $functionsClient = new Functions($this->client); $functions = $this->cache->get(Func::getName()); @@ -1061,6 +1062,10 @@ private function exportDeployments(int $batchSize) ); foreach ($response['deployments'] as $deployment) { + if (!$includeInactive && ($deployment['$id'] != $func->getDeployment())) { + continue; + } + $this->exportDeploymentData($func, $deployment); $lastDocument = $deployment['$id']; @@ -1097,13 +1102,13 @@ private function exportDeploymentData(Func $func, array $deployment) $deployment['activate'] ); - $deployment->setInternalId($deployment->getId()); + $deployment->setOriginalId($deployment->getId()); // Loop until the entire file is downloaded while ($start < $fileSize) { $chunkData = $this->call( 'GET', - "/functions/{$func->getId()}/deployments/{$deployment->getInternalId()}/download", + "/functions/{$func->getId()}/deployments/{$deployment->getOriginalId()}/download", ['range' => "bytes=$start-$end"] ); From 5ff6b2304e0a1a0b19162deacb65335836a351b4 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 25 Aug 2023 09:40:21 +0100 Subject: [PATCH 2/4] Add EnvVar Optionals --- src/Migration/Sources/Appwrite.php | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index fa2c7e2b..0f36cef9 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -987,7 +987,7 @@ private function exportFileData(File $file) protected function exportGroupFunctions(int $batchSize, array $resources) { if (in_array(Resource::TYPE_FUNCTION, $resources)) { - $this->exportFunctions($batchSize); + $this->exportFunctions($batchSize, $resources); } if (in_array(Resource::TYPE_DEPLOYMENT, $resources)) { @@ -995,7 +995,7 @@ protected function exportGroupFunctions(int $batchSize, array $resources) } } - private function exportFunctions(int $batchSize) + private function exportFunctions(int $batchSize, array $resources) { $functionsClient = new Functions($this->client); @@ -1022,12 +1022,14 @@ private function exportFunctions(int $batchSize) $convertedResources[] = $convertedFunc; - foreach ($function['vars'] as $var) { - $convertedResources[] = new EnvVar( - $convertedFunc, - $var['key'], - $var['value'], - ); + if (in_array(Resource::TYPE_ENVVAR, $resources)) { + foreach ($function['vars'] as $var) { + $convertedResources[] = new EnvVar( + $convertedFunc, + $var['key'], + $var['value'], + ); + } } } From d6bd7fa59ef5489d61dd232dbe0901e993011f40 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 25 Aug 2023 09:41:22 +0100 Subject: [PATCH 3/4] Update Appwrite.php --- src/Migration/Sources/Appwrite.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 0f36cef9..dab07ee1 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -101,6 +101,7 @@ public static function getSupportedResources(): array // Functions Resource::TYPE_FUNCTION, Resource::TYPE_DEPLOYMENT, + Resource::TYPE_INACTIVE_DEPLOYMENTS, Resource::TYPE_ENVVAR, // Settings From 1b5e1f4b4391a44b3244f900203930e427a0cec9 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Fri, 25 Aug 2023 10:34:20 +0100 Subject: [PATCH 4/4] Run Linter --- src/Migration/Cache.php | 1 - src/Migration/Destinations/Local.php | 2 +- src/Migration/Resources/Functions/Func.php | 2 +- src/Migration/Sources/Appwrite.php | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Migration/Cache.php b/src/Migration/Cache.php index d3307c09..1eb505c7 100644 --- a/src/Migration/Cache.php +++ b/src/Migration/Cache.php @@ -2,7 +2,6 @@ namespace Utopia\Migration; -use Utopia\Migration\Resources\Functions\Func; use Utopia\Migration\Resources\Storage\File; /** diff --git a/src/Migration/Destinations/Local.php b/src/Migration/Destinations/Local.php index b2201e50..457ea904 100644 --- a/src/Migration/Destinations/Local.php +++ b/src/Migration/Destinations/Local.php @@ -135,7 +135,7 @@ protected function import(array $resources, callable $callback): void $resource->setData(''); break; default: - $this->data[$resource->getGroup()][$resource->getName()][]= $resource->asArray(); + $this->data[$resource->getGroup()][$resource->getName()][] = $resource->asArray(); break; } diff --git a/src/Migration/Resources/Functions/Func.php b/src/Migration/Resources/Functions/Func.php index 840e1ff4..2a99e970 100644 --- a/src/Migration/Resources/Functions/Func.php +++ b/src/Migration/Resources/Functions/Func.php @@ -146,7 +146,7 @@ public function asArray(): array 'events' => $this->events, 'schedule' => $this->schedule, 'timeout' => $this->timeout, - 'deployment' => $this->deployment + 'deployment' => $this->deployment, ]; } } diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index dab07ee1..310abf3f 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -1065,7 +1065,7 @@ private function exportDeployments(int $batchSize, bool $includeInactive = false ); foreach ($response['deployments'] as $deployment) { - if (!$includeInactive && ($deployment['$id'] != $func->getDeployment())) { + if (! $includeInactive && ($deployment['$id'] != $func->getDeployment())) { continue; }