From e24554429c0dcabfd66584092cb477bea2af5b34 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Tue, 26 Aug 2025 14:32:03 +0200 Subject: [PATCH 01/10] delete storage:mass-dedup because we dont have duplicity endpoint anymore --- README.md | 27 ----- src/Keboola/Console/Command/MassDedup.php | 131 ---------------------- 2 files changed, 158 deletions(-) delete mode 100644 src/Keboola/Console/Command/MassDedup.php diff --git a/README.md b/README.md index 545d36c..33239ad 100644 --- a/README.md +++ b/README.md @@ -8,33 +8,6 @@ Assorted CLI utils `docker run --rm -it keboola/cli-utils php ./cli.php` -## Mass Dedup - -Loads project ids and tables from a CSV file and runs a dedup job on each table. - -**Command** - -``` -php cli.php storage:mass-dedup MANAGETOKEN /usr/ondra/data.csv -``` - -**Dry run** - -Does not create the job or the snapshot. - -``` -php cli.php storage:mass-dedup MANAGETOKEN /usr/ondra/data.csv --dry-run -``` - -**CSV data sample** - -``` -data.csv -"project","table" -"232","out.c-rs-main.data" -"232","in.c-main.data" -``` - ## Redshift Deep Copy Performs a deep copy of a table in Redshift backend. Fails if there are any aliases based on this table. diff --git a/src/Keboola/Console/Command/MassDedup.php b/src/Keboola/Console/Command/MassDedup.php deleted file mode 100644 index 429db73..0000000 --- a/src/Keboola/Console/Command/MassDedup.php +++ /dev/null @@ -1,131 +0,0 @@ -setName('storage:mass-dedup') - ->setDescription('Migrates configuration from Storage API tables to configurations') - ->addArgument('token', InputArgument::REQUIRED, 'manage token') - ->addArgument('source-file', InputArgument::REQUIRED, 'source file') - ->addOption('dry-run', 'd', InputOption::VALUE_NONE, 'Dry run (do not save data)') - ; - } - - - protected function execute(InputInterface $input, OutputInterface $output) - { - $token = $input->getArgument('token'); - $csvFile = new CsvFile($input->getArgument('source-file')); - $projects = []; - - if ($csvFile->getHeader()[0] != 'project') { - throw new \Exception('Project not defined in file.'); - } - if ($csvFile->getHeader()[1] != 'table') { - throw new \Exception('Table not defined in file.'); - } - - $csvFile->rewind(); - $csvFile->next(); - - while ($csvFile->valid()) { - $row = $csvFile->current(); - $projects[] = $row[0]; - $csvFile->next(); - } - - $projects = array_unique($projects); - - $manageClient = new \Keboola\ManageApi\Client(["token" => $token]); - $manageClient->verifyToken(); - - foreach ($projects as $projectId) { - $output->writeln(""); - $output->writeln("Processing project " . $projectId); - try { - $projectInfo = $manageClient->getProject($projectId); - } catch (ClientException $e) { - $output->writeln($e->getMessage()); - $output->write("\n"); - continue; - } - - // Disabled projects - if (isset($projectInfo["isDisabled"]) && $projectInfo["isDisabled"]) { - $output->writeln("Project disabled: " . $projectInfo["disabled"]["reason"]); - } else { - $projectToken = $manageClient->createProjectStorageToken( - $projectId, - [ - "description" => "Dedup tables", - "canManageBuckets" => true, - "canReadAllFileUploads" => true, - "expiresIn" => 3600 - ] - ); - - $client = new Client(["token" => $projectToken["token"]]); - - $csvFile->rewind(); - $csvFile->next(); - - while ($csvFile->valid()) { - $row = $csvFile->current(); - $projects[] = $row[0]; - - if ($row[0] == $projectId) { - if (!$client->tableExists($row[1])) { - $output->writeln("Table " . $row[1] . " does not exist"); - $csvFile->next(); - continue; - } - try { - $output->writeln("Processing table " . $row[1]); - - // detect dedup - $output->write("Dedup... "); - $duplicityResponse = $client->apiGet("tables/{$row[1]}/duplicity"); - if ($duplicityResponse['maxDuplicity'] == 1) { - $output->writeln("not required"); - $csvFile->next(); - continue; - } else { - $output->writeln("required"); - } - - if (!$input->getOption("dry-run")) { - // snapshot - $output->write("Snapshot... "); - $client->createTableSnapshot($row[1], "Backup before deduplication"); - $output->writeln("created"); - - // dedup - $output->write("Dedup job... "); - $client->apiPost("tables/{$row[1]}/dedupe"); - $output->writeln("created"); - } - } catch (\Exception $e) { - print $e->getTraceAsString(); - throw $e; - } - } - $csvFile->next(); - } - } - $output->write("\n"); - } - } -} From 7fdf600b9ca733fa0981b0db06cd38856cc47176 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Tue, 26 Aug 2025 14:32:53 +0200 Subject: [PATCH 02/10] manage:mass-project-queue-migration - delete becasue we dont migrate to Queuev2 --- README.md | 26 -- .../Command/MassProjectQueueMigration.php | 282 ------------------ 2 files changed, 308 deletions(-) delete mode 100644 src/Keboola/Console/Command/MassProjectQueueMigration.php diff --git a/README.md b/README.md index 33239ad..4e291fa 100644 --- a/README.md +++ b/README.md @@ -133,32 +133,6 @@ Loads last N _(default 100)_ jobs into Marquez tool. Export has two modes: - default - jobs are identified by job IDs - with `--job-names-configurations` option - job are identified by component and configuration IDs -## Mass Project Queue Migration - -- Create a manage token. -- Prepare input file (e.g. "projects") with ids of the projects to migrate to new Queue. - ``` - 1234 - 5678 - 9012 - 3456 - ``` - -- Run the mass migration command - ``` - php cli.php manage:mass-project-queue-migration - ``` - - The command will do the following for every projectId in the source file: - - add project feature `queuev2` - - create and run configuration of `keboola.queue-migration-tool` component - - if a job was successful, it will disable legacy orchestrations in the project - - if a job ended with error, it will remove the `queuev2` feature from the project - - -- After migration delete your manage token you have created and used for migrations. - - ## Mass project enable dynamic backends Prerequisities: https://keboola.atlassian.net/wiki/spaces/KB/pages/2135982081/Enable+Dynamic+Backends#Enable-for-project diff --git a/src/Keboola/Console/Command/MassProjectQueueMigration.php b/src/Keboola/Console/Command/MassProjectQueueMigration.php deleted file mode 100644 index 1e69f82..0000000 --- a/src/Keboola/Console/Command/MassProjectQueueMigration.php +++ /dev/null @@ -1,282 +0,0 @@ -setName('manage:mass-project-queue-migration') - ->setDescription('Mass project migration to Queue v2') - ->addArgument(self::ARGUMENT_MANAGE_TOKEN, InputArgument::REQUIRED, 'Manage token') - ->addArgument(self::ARGUMENT_CONNECTION_URL, InputArgument::REQUIRED, 'Connection url') - ->addArgument(self::ARGUMENT_SOURCE_FILE, InputArgument::REQUIRED, 'Source file with project ids') - ; - } - - private function getProjectIdsForMigration( - string $sourceFile, - Client $manageClient, - OutputInterface $output - ): array { - $projectIds = []; - $output->writeln(sprintf('Fetching projects from "%s"', $sourceFile)); - $projects = $this->parseProjectIds($sourceFile); - $output->writeln(sprintf('Migrating "%s" projects', count($projects))); - - foreach ($projects as $projectId) { - try { - $projectRes = $manageClient->getProject($projectId); - if (in_array(self::FEATURE_QUEUE_V2, $projectRes['features'])) { - // don't migrate project with feature already set - $output->writeln(sprintf('Project "%s" is already on Queue v2', $projectId)); - continue; - } - - $projectIds[] = $projectId; - } catch (ManageClientException $e) { - $output->writeln(sprintf( - 'Exception occurred while accessing project %s: %s', - $projectId, - $e->getMessage() - )); - - continue; - } - } - - return array_unique($projectIds); - } - - private function createMigrationJob( - string $projectId, - string $kbcUrl, - string $manageToken, - Client $manageClient, - OutputInterface $output - ): ?array { - // set queuev2 project feature - try { - $manageClient->addProjectFeature($projectId, self::FEATURE_QUEUE_V2); - $storageToken = $this->createStorageToken($manageClient, $projectId); - } catch (ManageClientException $e) { - $output->writeln(sprintf( - 'Exception occurred while accessing project %s: %s', - $projectId, - $e->getMessage() - )); - - return null; - } - - $storageClient = new StorageClient([ - 'url' => $kbcUrl, - 'token' => $storageToken, - ]); - - $encryptedManageToken = $this->encrypt( - $storageClient->getServiceUrl('encryption'), - $manageToken - ); - - $jobData = new JobData( - self::COMPONENT_QUEUE_MIGRATION_TOOL, - '', - [ - 'parameters' => [ - '#manage_token' => $encryptedManageToken - ], - ] - ); - - $jobQueueClient = new JobQueueClient( - $storageClient->getServiceUrl('queue'), - $storageToken - ); - - try { - $jobRes = $jobQueueClient->createJob($jobData); - } catch (JobQueueClientException $e) { - $output->writeln(sprintf( - 'Exception occurred while creating migration job in project %s: %s', - $projectId, - $e->getMessage() - )); - - return null; - } - - $output->writeln(sprintf( - 'Created migration job "%s" for project "%s"', - $jobRes['id'], - $projectId - )); - - return [ - 'jobId' => $jobRes['id'], - 'projectId' => $projectId, - 'jobQueueClient' => $jobQueueClient, - 'storageToken' => $storageToken, - ]; - } - - - protected function execute(InputInterface $input, OutputInterface $output) - { - $manageToken = $input->getArgument(self::ARGUMENT_MANAGE_TOKEN); - $kbcUrl = $input->getArgument(self::ARGUMENT_CONNECTION_URL); - $sourceFile = $input->getArgument(self::ARGUMENT_SOURCE_FILE); - - $manageClient = new Client([ - 'token' => $manageToken, - 'url' => $kbcUrl, - ]); - - $projects = $this->getProjectIdsForMigration($sourceFile, $manageClient, $output); - $migrationJobs = []; - foreach ($projects as $projectId) { - $migrationJob = $this->createMigrationJob( - $projectId, - $kbcUrl, - $manageToken, - $manageClient, - $output - ); - - $migrationJobs[$migrationJob['jobId']] = $migrationJob; - } - - if (empty($migrationJobs)) { - $output->writeln('Skipping migration'); - $output->writeln(PHP_EOL); - return; - } - - $output->writeln(sprintf('Created %s migration jobs in total', count($migrationJobs))); - $output->writeln('Waiting for the jobs to finish...'); - - // wait until all migration jobs are finished - $unfinishedJobs = $migrationJobs; - - while (count($unfinishedJobs) > 0) { - foreach ($unfinishedJobs as $jobId => $data) { - /** @var JobQueueClient $jobQueueClient */ - $jobQueueClient = $data['jobQueueClient']; - $jobRes = $jobQueueClient->getJob((string) $jobId); - if (in_array($jobRes['status'], self::JOB_STATES_FINAL)) { - unset($unfinishedJobs[$jobId]); - unset($migrationJobs[$jobId]['jobQueueClient']); - $migrationJobs[$jobId]['status'] = $jobRes['status']; - } - } - sleep(2); - } - - $successJobs = array_filter($migrationJobs, fn($item) => $item['status'] === 'success'); - $errorJobs = array_filter($migrationJobs, fn($item) => $item['status'] === 'error'); - $terminatedJobs = array_filter( - $migrationJobs, - fn($item) => $item['status'] === 'terminated' || $item['status'] === 'cancelled' - ); - - $output->writeln(sprintf('%s migration jobs finished successfully', count($successJobs))); - - $output->writeln(sprintf('%s migration jobs ended with error:', count($errorJobs))); - foreach ($errorJobs as $errorJob) { - $output->writeln(sprintf( - 'Job "%s" of project "%s" ended with error', - $errorJob['jobId'], - $errorJob['projectId'] - )); - - $manageClient->removeProjectFeature($errorJob['projectId'], self::FEATURE_QUEUE_V2); - } - - $output->writeln(sprintf('%s migration jobs were terminated or cancelled:', count($terminatedJobs))); - foreach ($terminatedJobs as $terminatedJob) { - $output->writeln(sprintf( - 'Job "%s" of project "%s" ended with "%s"', - $terminatedJob['jobId'], - $terminatedJob['projectId'], - $terminatedJob['status'] - )); - } - $output->writeln(PHP_EOL); - } - - private function createStorageToken(Client $client, string $projectId): string - { - $response = $client->createProjectStorageToken($projectId, [ - 'description' => 'Cli utils - mass queue migration', - 'canManageBuckets' => true, - 'canReadAllFileUploads' => true, - 'canManageTokens' => true, - ]); - - return $response['token']; - } - - private function parseProjectIds(string $sourceFile): array - { - if (!file_exists($sourceFile)) { - throw new Exception(sprintf('Cannot open "%s"', $sourceFile)); - } - $projectsText = trim(file_get_contents($sourceFile)); - if (!$projectsText) { - return []; - } - - $projectIds = explode(PHP_EOL, $projectsText); - if (count($projectIds) > 10) { - throw new Exception(sprintf( - 'Due prevent overload of our internal API, it is only possible to ' - . 'migrate a maximum of 10 projects at once. You have %s projects waiting for migrations.', - count($projectIds) - )); - } - - return explode(PHP_EOL, $projectsText); - } - - private function encrypt( - string $encryptionApiUrl, - string $value - ): string { - $client = new GuzzleClient(); - $response = $client->post( - sprintf('%s/encrypt?componentId=%s', $encryptionApiUrl, self::COMPONENT_QUEUE_MIGRATION_TOOL), - [ - 'body' => $value, - 'headers' => [ - 'Content-Type' => 'text/plain' - ], - ] - ); - - return $response->getBody()->getContents(); - } -} From fb752ddfc85367ca161fe6d5736b0067cdfdc281 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Tue, 26 Aug 2025 14:33:42 +0200 Subject: [PATCH 03/10] storage:redshift-deep-copy - drop because we dont support RS anymore --- README.md | 8 --- .../Console/Command/RedshiftDeepCopy.php | 67 ------------------- 2 files changed, 75 deletions(-) delete mode 100644 src/Keboola/Console/Command/RedshiftDeepCopy.php diff --git a/README.md b/README.md index 4e291fa..84da99a 100644 --- a/README.md +++ b/README.md @@ -8,14 +8,6 @@ Assorted CLI utils `docker run --rm -it keboola/cli-utils php ./cli.php` -## Redshift Deep Copy - -Performs a deep copy of a table in Redshift backend. Fails if there are any aliases based on this table. - -``` -php cli.php storage:redshift-deep-copy PROJECTTOKEN in.c-main.buggy-table -``` - ## Bulk Project Add Feature Adds a project feature to multiple projects diff --git a/src/Keboola/Console/Command/RedshiftDeepCopy.php b/src/Keboola/Console/Command/RedshiftDeepCopy.php deleted file mode 100644 index 89f660f..0000000 --- a/src/Keboola/Console/Command/RedshiftDeepCopy.php +++ /dev/null @@ -1,67 +0,0 @@ -setName('storage:redshift-deep-copy') - ->setDescription('Creates a snapshot of a table and recreates the table from the snapshot') - ->addArgument('token', InputArgument::REQUIRED, 'storage api token') - ->addArgument('table', InputArgument::REQUIRED, 'table to deep copy') - ->addOption('dry-run', 'd', InputOption::VALUE_NONE, 'Dry run (do not ovewrite original table)') - ; - } - - - protected function execute(InputInterface $input, OutputInterface $output) - { - $token = $input->getArgument('token'); - $tableId = $input->getArgument('table'); - $bucketId = substr($tableId, 0, strrpos($tableId, ".")); - $tableName = substr($tableId, strrpos($tableId, ".") + 1); - - $client = new Client(["token" => $token]); - if (!$client->tableExists($tableId)) { - throw new \Exception("Table {$tableId} does not exist"); - } - - $snapshotId = $client->createTableSnapshot($tableId, "Deep Copy"); - $client->createTableFromSnapshot($bucketId, $snapshotId, $tableName . "__stg"); - $tableInfo = $client->getTable($tableId); - $newTableInfo = $client->getTable($tableId . "__stg"); - - if ($tableInfo["rowsCount"] != $newTableInfo["rowsCount"]) { - $client->dropTable($tableId . "__stg"); - throw new \Exception("Rows not equal!"); - } - if ($tableInfo["dataSizeBytes"] <= 1.5 * $newTableInfo["dataSizeBytes"]) { - $output->writeln("Deep copy of {$tableId} not required"); - } else { - $output->writeln("Deep copy of {$tableId} required"); - if (!$input->getOption("dry-run")) { - // magic! - try { - $client->dropTable($tableId); - $client->createTableFromSnapshot($bucketId, $snapshotId, $tableName); - $output->writeln("Deep copy done"); - } catch (\Keboola\StorageApi\ClientException $e) { - $client->dropTable($tableId . "__stg"); - throw $e; - } - } - } - $client->dropTable($tableId . "__stg"); - } -} From e878b2233a4d08fecb0a87cd81ead40fc4cb9c01 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Tue, 26 Aug 2025 14:34:05 +0200 Subject: [PATCH 04/10] drop RS commands --- .../Command/RedshiftOrphanedWorkspaces.php | 123 ------------------ .../Console/Command/RedshiftSchemasCount.php | 92 ------------- 2 files changed, 215 deletions(-) delete mode 100644 src/Keboola/Console/Command/RedshiftOrphanedWorkspaces.php delete mode 100644 src/Keboola/Console/Command/RedshiftSchemasCount.php diff --git a/src/Keboola/Console/Command/RedshiftOrphanedWorkspaces.php b/src/Keboola/Console/Command/RedshiftOrphanedWorkspaces.php deleted file mode 100644 index 6f099cc..0000000 --- a/src/Keboola/Console/Command/RedshiftOrphanedWorkspaces.php +++ /dev/null @@ -1,123 +0,0 @@ -setName('storage:redshift-orphaned-workspaces') - ->setDescription('Checks for orphaned Redshift workspaces in each project\'s database') - ->addArgument('token', InputArgument::REQUIRED, 'manage api token') - ->addArgument('projects', InputArgument::REQUIRED, 'single project id or range (eg 10..500)') - ->addArgument('source-file', InputArgument::REQUIRED, 'source file with deactivated workspaces, columns "id, workspaceId, tokenOwnerId"') - ; - } - - protected function execute(InputInterface $input, OutputInterface $output) - { - $token = $input->getArgument('token'); - - if (strpos($input->getArgument('projects'), '..')) { - list($start, $end) = explode('..', $input->getArgument('projects')); - $projects = range($start, $end); - } else { - $projects = [$input->getArgument('projects')]; - } - - $csvFile = new CsvFile($input->getArgument('source-file')); - - if ($csvFile->getHeader()[0] != 'id') { - throw new \Exception('Column "id" not defined in file.'); - } - if ($csvFile->getHeader()[1] != 'workspaceId') { - throw new \Exception('Column "workspaceId" not defined in file.'); - } - if ($csvFile->getHeader()[2] != 'tokenOwnerId') { - throw new \Exception('Column "tokenOwnerId" not defined in file.'); - } - - $orphans = []; - $csvFile->rewind(); - // skip header - $csvFile->next(); - while ($csvFile->current()) { - $orphans[] = 'workspace_' . $csvFile->current()[1]; - $csvFile->next(); - } - - $manageClient = new \Keboola\ManageApi\Client(["token" => $token]); - $manageClient->verifyToken(); - - $backends = $manageClient->listStorageBackend(["logins" => true]); - // var_dump($backends); - - foreach ($projects as $projectId) { - try { - $projectInfo = $manageClient->getProject($projectId); - } catch (ClientException $e) { - $output->writeln($e->getMessage()); - continue; - } - // Disabled projects - if (isset($projectInfo["isDisabled"]) && $projectInfo["isDisabled"]) { - $output->writeln("Project {$projectId} disabled."); - continue; - } - if (!isset($projectInfo["hasRedshift"]) || $projectInfo["hasRedshift"] == false) { - $output->writeln("Project {$projectId} does not have Redshift."); - continue; - } - // login and count schemas - $backendConnection = null; - foreach ($backends as $backend) { - if ($backend["id"] == $projectInfo["backends"]["redshift"]["id"]) { - $backendConnection = $backend; - } - } - if (!$backendConnection) { - $output->writeln('Backend connection for project ' . $projectId . ' not found'); - continue; - } - - $output->writeln("Detecting orphaned workspaces for project {$projectId}."); - - try { - $dsn = 'pgsql:host=' . $backendConnection["host"] . ';port=5439;dbname=sapi_' . $projectId; - $pdo = new \PDO($dsn, explode("/", $backendConnection["login"])[0], explode("/", $backendConnection["login"])[1]); - $result = $pdo->query("select nspname from pg_namespace where nspname like 'workspace_%';"); - - foreach ($result->fetchAll() as $row) { - if (in_array($row["nspname"], $orphans)) { - $output->writeln("Project {$projectId} has orphaned schema {$row["nspname"]}."); - } - } - $result->closeCursor(); - $pdo = null; - } catch (\Exception $e) { - $output->writeln($e->getMessage()); - $output->write("\n"); - } - } - } -} diff --git a/src/Keboola/Console/Command/RedshiftSchemasCount.php b/src/Keboola/Console/Command/RedshiftSchemasCount.php deleted file mode 100644 index 3806614..0000000 --- a/src/Keboola/Console/Command/RedshiftSchemasCount.php +++ /dev/null @@ -1,92 +0,0 @@ -setName('storage:redshift-schemas-count') - ->setDescription('Checks for the count of Redshift schemas in each project\'s database') - ->addArgument('token', InputArgument::REQUIRED, 'manage api token') - ->addArgument('projects', InputArgument::REQUIRED, 'single project id or range (eg 10..500)') - ; - } - - - protected function execute(InputInterface $input, OutputInterface $output) - { - $token = $input->getArgument('token'); - - if (strpos($input->getArgument('projects'), '..')) { - list($start, $end) = explode('..', $input->getArgument('projects')); - $projects = range($start, $end); - } else { - $projects = [$input->getArgument('projects')]; - } - - $manageClient = new \Keboola\ManageApi\Client(["token" => $token]); - $manageClient->verifyToken(); - - $backends = $manageClient->listStorageBackend(["logins" => true]); - // var_dump($backends); - - - foreach ($projects as $projectId) { - try { - $projectInfo = $manageClient->getProject($projectId); - } catch (ClientException $e) { - $output->writeln($e->getMessage()); - continue; - } - - // Disabled projects - if (isset($projectInfo["isDisabled"]) && $projectInfo["isDisabled"]) { - $output->writeln("Project {$projectId} disabled."); - continue; - } - if (!isset($projectInfo["hasRedshift"]) || $projectInfo["hasRedshift"] == false) { - $output->writeln("Project {$projectId} does not have Redshift."); - continue; - } - // login and count schemas - $backendConnection = null; - foreach ($backends as $backend) { - if ($backend["id"] == $projectInfo["backends"]["redshift"]["id"]) { - $backendConnection = $backend; - } - } - if (!$backendConnection) { - $output->writeln('Backend connection for project ' . $projectId . ' not found'); - } - try { - $dsn = 'pgsql:host=' . $backendConnection["host"] . ';port=5439;dbname=sapi_' . $projectId; - $pdo = new \PDO($dsn, explode("/", $backendConnection["login"])[0], explode("/", $backendConnection["login"])[1]); - $result = $pdo->query("select COUNT(*) from pg_namespace;"); - $count = $result->fetchColumn(0); - $result->closeCursor(); - if ($count >= 255) { - $output->writeln("Project {$projectId} has {$count} schemas out of 256."); - } else if ($count >= 200) { - $output->writeln("Project {$projectId} has {$count} schemas out of 256."); - } else { - $output->writeln("Project {$projectId} has {$count} schemas out of 256."); - } - $pdo = null; - } catch (\Exception $e) { - $output->writeln($e->getMessage()); - $output->write("\n"); - } - } - } -} From c7858a8ba16871c3c9a6d445b23a42b3a5e2e22f Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Tue, 26 Aug 2025 14:34:56 +0200 Subject: [PATCH 05/10] drop commands use --- cli.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cli.php b/cli.php index ba98520..e549ebe 100644 --- a/cli.php +++ b/cli.php @@ -35,12 +35,8 @@ use Keboola\Console\Command\UpdateDataRetention; $application = new Application(); -$application->add(new MassDedup()); -$application->add(new RedshiftDeepCopy()); $application->add(new ProjectsAddFeature()); $application->add(new ProjectsRemoveFeature()); -$application->add(new RedshiftSchemasCount()); -$application->add(new RedshiftOrphanedWorkspaces()); $application->add(new DeletedProjectsPurge()); $application->add(new NotifyProjects()); $application->add(new SetDataRetention()); @@ -49,7 +45,6 @@ $application->add(new MassProjectEnableDynamicBackends()); $application->add(new AddFeature()); $application->add(new AllStacksIterator()); -$application->add(new MassProjectQueueMigration()); $application->add(new LineageEventsExport()); $application->add(new QueueMassTerminateJobs()); $application->add(new DeleteOrphanedWorkspaces()); From 6096d5764bd8aec050c020424981d683928a58ea Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Tue, 26 Aug 2025 14:35:08 +0200 Subject: [PATCH 06/10] fixup --- cli.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cli.php b/cli.php index e549ebe..594b8e7 100644 --- a/cli.php +++ b/cli.php @@ -11,21 +11,16 @@ use Keboola\Console\Command\DeleteOwnerlessWorkspaces; use Keboola\Console\Command\DescribeOrganizationWorkspaces; use Keboola\Console\Command\LineageEventsExport; -use Keboola\Console\Command\MassDedup; use Keboola\Console\Command\MassDeleteProjectWorkspaces; use Keboola\Console\Command\MassProjectEnableDynamicBackends; use Keboola\Console\Command\MassProjectExtendExpiration; -use Keboola\Console\Command\MassProjectQueueMigration; use Keboola\Console\Command\MigrateFiles; use Keboola\Console\Command\OrganizationIntoMaintenanceMode; use Keboola\Console\Command\OrganizationStorageBackend; use Keboola\Console\Command\QueueMassTerminateJobs; use Keboola\Console\Command\ReactivateSchedules; -use Keboola\Console\Command\RedshiftDeepCopy; use Keboola\Console\Command\ProjectsAddFeature; use Keboola\Console\Command\ProjectsRemoveFeature; -use Keboola\Console\Command\RedshiftSchemasCount; -use Keboola\Console\Command\RedshiftOrphanedWorkspaces; use Keboola\Console\Command\DeletedProjectsPurge; use Keboola\Console\Command\DeleteProjectSandboxes; use Keboola\Console\Command\NotifyProjects; From 07d14c6c75d8bdce76d3f5adbe271deee8c604e5 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Tue, 26 Aug 2025 14:41:03 +0200 Subject: [PATCH 07/10] drop GD something... --- README.md | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/README.md b/README.md index 84da99a..bdae56b 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ cat data.csv | php cli.php storage:notify-projects MANAGETOKEN ## Mass project extend expiration: -Prepare input file "extend.txt" from looker https://keboola.looker.com/explore/keboola_internal_reporting/usage_metric_values: +Prepare input file "extend.txt" in following format: ``` 123-EU 579-US @@ -72,27 +72,6 @@ Run command: Use number of days or 0 as show to remove expiration completely. By default, it's dry-run. Override with `-f` parameter. -## Mass GD project drop: - -Prepare input file "projects.csv" with project IDs: - -``` -123abc123abc123abc123abc123abc123abc -``` - -Prepare `.env` file from `.env.dist`: -``` -GOODDATA_URL= -GOODDATA_LOGIN= -GOODDATA_PASSWORD= -``` - -Run command: - -`php cli.php gooddata:delete-projects` - -To actually drop the projects add `-f` flag. Default is dry-run. - ## Add a feature to project templates You can add a project feature to all the project templates available on the stack From 102ce632f8c4d66868f1f521531b280e211021ea Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Tue, 26 Aug 2025 14:48:30 +0200 Subject: [PATCH 08/10] .env need envs for another command --- .env.dist | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.env.dist b/.env.dist index c62d3d2..6ec1d56 100644 --- a/.env.dist +++ b/.env.dist @@ -1,3 +1,4 @@ -GOODDATA_URL= -GOODDATA_LOGIN= -GOODDATA_PASSWORD= +# for manage:mass-project-remove-expiration +KBC_MANAGE_TOKEN_US= +KBC_MANAGE_TOKEN_EU= +KBC_MANAGE_TOKEN_NE= From d0e4329193892ad538e6ba8657056a6ad0531659 Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Tue, 26 Aug 2025 14:48:43 +0200 Subject: [PATCH 09/10] Drop unused version --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7138fd8..8a38bbb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '2' services: app: build: . From c47a2f8dbc0a9dcacedd38551ec13f8eb597de5c Mon Sep 17 00:00:00 2001 From: Jiri Semmler Date: Tue, 26 Aug 2025 14:49:00 +0200 Subject: [PATCH 10/10] fix typo --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index bdae56b..64f5883 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ Assorted CLI utils `docker run --rm -it keboola/cli-utils php ./cli.php` ## Bulk Project Add Feature - Adds a project feature to multiple projects ``` @@ -145,7 +144,7 @@ It will perform a dry run unleass the `--force/-f` option is applied. - Run the command ``` - php ./cli.php storage:delete-orphaned=workspaces [--force/-f] + php ./cli.php storage:delete-orphaned-workspaces [--force/-f] ``` ## Delete Orphaned Workspaces in Organization command