From 55af0e008da77b09f6a96ee0f632bd369d2f38f3 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Fri, 16 May 2025 11:16:09 +0200 Subject: [PATCH 1/2] SUPPORT-11914 add command for mass project update --- cli.php | 2 + .../Console/Command/UpdateDataRetention.php | 151 ++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 src/Keboola/Console/Command/UpdateDataRetention.php diff --git a/cli.php b/cli.php index 38b8d0d..ba98520 100644 --- a/cli.php +++ b/cli.php @@ -32,6 +32,7 @@ use Keboola\Console\Command\RemoveUserFromOrganizationProjects; use Symfony\Component\Console\Application; use Keboola\Console\Command\SetDataRetention; +use Keboola\Console\Command\UpdateDataRetention; $application = new Application(); $application->add(new MassDedup()); @@ -61,4 +62,5 @@ $application->add(new ReactivateSchedules()); $application->add(new DescribeOrganizationWorkspaces()); $application->add(new MassDeleteProjectWorkspaces()); +$application->add(new UpdateDataRetention()); $application->run(); diff --git a/src/Keboola/Console/Command/UpdateDataRetention.php b/src/Keboola/Console/Command/UpdateDataRetention.php new file mode 100644 index 0000000..92351a0 --- /dev/null +++ b/src/Keboola/Console/Command/UpdateDataRetention.php @@ -0,0 +1,151 @@ +setName('manage:update-data-retention') + ->setDescription('Update data retention time in days for all projects') + ->addArgument(self::ARG_TOKEN, InputArgument::REQUIRED, 'manage token') + ->addArgument(self::ARG_URL, InputArgument::REQUIRED, 'Stack URL') + ->addArgument(self::ARG_DATA_RETENTION, InputArgument::REQUIRED, 'Data retention time in days') + ->addOption(self::OPT_FORCE, 'f', InputOption::VALUE_NONE, 'Will actually do the work, otherwise it\'s dry run'); + } + + protected function createClient(string $host, string $token): Client + { + return new Client([ + 'url' => $host, + 'token' => $token, + ]); + } + + protected function updateProjectDataRetention( + Client $client, + OutputInterface $output, + array $projectInfo, + int $dataRetentionTimeInDays, + bool $force + ): void { + $output->writeln(sprintf('Updating data retention for project "%s" ("%s")', $projectInfo['id'], $projectInfo['name'])); + + // Disabled projects + if (isset($projectInfo["isDisabled"]) && $projectInfo["isDisabled"]) { + $output->writeln(sprintf(' - project disabled: "%s"', $projectInfo["disabled"]["reason"])); + $this->projectsDisabled++; + } else { + if (!in_array('snowflake', $projectInfo['assignedBackends'])) { + $output->writeln(' - project does not have Snowflake backend assigned. Skiping.'); + $this->projectsNoSnowflake++; + } else { + try { + if ($force) { + $client->updateProject($projectInfo['id'], ['dataRetentionTimeInDays' => $dataRetentionTimeInDays]); + $output->writeln(sprintf(' - data retention time successfully updated to %d days.', $dataRetentionTimeInDays)); + } else { + $output->writeln(sprintf(' - would update data retention time to %d days. Enable force mode with -f option', $dataRetentionTimeInDays)); + } + $this->projectsUpdated++; + } catch (ClientException $e) { + $output->writeln(sprintf(' - error updating project: "%s"', $e->getMessage())); + $this->projectsError++; + } + } + } + $output->write("\n"); + } + + protected function updateAllProjects( + Client $client, + OutputInterface $output, + int $dataRetentionTimeInDays, + bool $force + ): void { + $maintainers = $client->listMaintainers(); + $output->writeln(sprintf('Found %d maintainers', count($maintainers))); + + foreach ($maintainers as $maintainer) { + $this->maintainersChecked++; + $output->writeln(sprintf('Processing maintainer "%s" ("%s")', $maintainer['id'], $maintainer['name'])); + + $organizations = $client->listMaintainerOrganizations($maintainer['id']); + $output->writeln(sprintf('Found %d organizations for maintainer "%s"', count($organizations), $maintainer['id'])); + + foreach ($organizations as $organization) { + $this->orgsChecked++; + $output->writeln('-----'); + $output->writeln(sprintf('Processing organization "%s" ("%s")', $organization['id'], $organization['name'])); + + $projects = $client->listOrganizationProjects($organization['id']); + $output->writeln(sprintf('Found %d projects for organization "%s"', count($projects), $organization['id'])); + + foreach ($projects as $project) { + $this->updateProjectDataRetention($client, $output, $project, $dataRetentionTimeInDays, $force); + } + } + } + } + + public function execute(InputInterface $input, OutputInterface $output): ?int + { + $args = $input->getArguments(); + $force = (bool) $input->getOption(self::OPT_FORCE); + $dataRetentionTimeInDays = (int) $args[self::ARG_DATA_RETENTION]; + $client = $this->createClient($args[self::ARG_URL], $args[self::ARG_TOKEN]); + + if ($force) { + $output->writeln('Force mode enabled. Projects will be updated.'); + } else { + $output->writeln('Dry run mode. No projects will be updated. Use -f to enable force mode.'); + } + + $output->writeln(sprintf('Updating all projects with data retention time: %d days', $dataRetentionTimeInDays)); + $this->updateAllProjects($client, $output, $dataRetentionTimeInDays, $force); + + $output->writeln("\n" . 'DONE with following results:' . "\n"); + $this->printResult($output, $force); + + return 0; + } + + private function printResult(OutputInterface $output, bool $force): void + { + $output->writeln(sprintf( + 'Checked %d maintainers' . "\n" + . 'Checked %d organizations' . "\n" + . '%d projects were disabled' . "\n" + . '%d projects do not have Snowflake backend' . "\n" + . '%d ' . ($force ? 'projects updated' : 'projects would be updated in force mode') . "\n" + . '%d projects had errors during update' . "\n", + $this->maintainersChecked, + $this->orgsChecked, + $this->projectsDisabled, + $this->projectsNoSnowflake, + $this->projectsUpdated, + $this->projectsError + )); + } +} From 09482b1f48f5aa6c3a1706f6ce1552116bde8719 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Fri, 16 May 2025 14:57:36 +0200 Subject: [PATCH 2/2] note --- src/Keboola/Console/Command/UpdateDataRetention.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Keboola/Console/Command/UpdateDataRetention.php b/src/Keboola/Console/Command/UpdateDataRetention.php index 92351a0..916fddc 100644 --- a/src/Keboola/Console/Command/UpdateDataRetention.php +++ b/src/Keboola/Console/Command/UpdateDataRetention.php @@ -28,7 +28,7 @@ protected function configure(): void { $this ->setName('manage:update-data-retention') - ->setDescription('Update data retention time in days for all projects') + ->setDescription('Update data retention time in days for all projects on the whole stack.') ->addArgument(self::ARG_TOKEN, InputArgument::REQUIRED, 'manage token') ->addArgument(self::ARG_URL, InputArgument::REQUIRED, 'Stack URL') ->addArgument(self::ARG_DATA_RETENTION, InputArgument::REQUIRED, 'Data retention time in days')