diff --git a/cli.php b/cli.php index b873cb6..0889535 100644 --- a/cli.php +++ b/cli.php @@ -24,7 +24,6 @@ use Keboola\Console\Command\ProjectsAddFeature; use Keboola\Console\Command\ProjectsRemoveFeature; use Keboola\Console\Command\DeletedProjectsPurge; -use Keboola\Console\Command\DeleteProjectSandboxes; use Keboola\Console\Command\NotifyProjects; use Keboola\Console\Command\RemoveUserFromOrganizationProjects; use Symfony\Component\Console\Application; @@ -49,7 +48,6 @@ $application->add(new OrganizationStorageBackend()); $application->add(new DeleteOwnerlessWorkspaces()); $application->add(new DeleteOrganizationOwnerlessWorkspaces()); -$application->add(new DeleteProjectSandboxes()); $application->add(new RemoveUserFromOrganizationProjects()); $application->add(new ReactivateSchedules()); $application->add(new DescribeOrganizationWorkspaces()); diff --git a/composer.json b/composer.json index b77d545..897cecc 100644 --- a/composer.json +++ b/composer.json @@ -3,13 +3,15 @@ "require": { "php": "^8.3", "symfony/console": "^7.4", - "keboola/sandboxes-api-php-client": "6.32.0", + "keboola/sandboxes-service-api-client": "dev-odin/AJDA-1942", + "keboola/service-client": "^1.0", "keboola/storage-api-client": "^18.5.0", "keboola/kbc-manage-api-php-client": "^7", "symfony/event-dispatcher": "^7.4", "keboola/job-queue-api-php-client": "^5.2.0", "symfony/config": "^7.4", - "keboola/php-datatypes": "^8.0" + "keboola/php-datatypes": "^8.0", + "symfony/http-client": "^7.4" }, "require-dev": { "squizlabs/php_codesniffer": "3.*", diff --git a/composer.lock b/composer.lock index bf8b78b..e0e36ff 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d44ece76899970e99f35a12443287b84", + "content-hash": "24930258e9913cc56c614972807068d3", "packages": [ { "name": "aws/aws-crt-php", @@ -1293,39 +1293,38 @@ "time": "2025-08-27T11:12:26+00:00" }, { - "name": "keboola/sandboxes-api-php-client", - "version": "6.32.0", + "name": "keboola/sandboxes-service-api-client", + "version": "dev-odin/AJDA-1942", "source": { "type": "git", - "url": "https://github.com/keboola/sandboxes-api-php-client.git", - "reference": "2125f1121d0c877f642902b94536aa1d744e8ff2" + "url": "https://github.com/keboola/sandboxes-service-api-php-client.git", + "reference": "c66e051f46b7893e53a41a963e7234d3907bd51d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/keboola/sandboxes-api-php-client/zipball/2125f1121d0c877f642902b94536aa1d744e8ff2", - "reference": "2125f1121d0c877f642902b94536aa1d744e8ff2", + "url": "https://api.github.com/repos/keboola/sandboxes-service-api-php-client/zipball/c66e051f46b7893e53a41a963e7234d3907bd51d", + "reference": "c66e051f46b7893e53a41a963e7234d3907bd51d", "shasum": "" }, "require": { - "ext-json": "*", - "guzzlehttp/guzzle": "~6.0|~7.0", + "guzzlehttp/guzzle": "^7.8", + "monolog/monolog": "^2.0|^3.0", "php": "^8.2", - "psr/log": "^1.1|^2.0|^3.0", - "symfony/validator": "^6.0|^7.0" + "webmozart/assert": "^1.11" }, "require-dev": { + "infection/infection": "^0.27.9", "keboola/coding-standard": "^15.0", - "keboola/kbc-manage-api-php-client": "^7.1", - "keboola/storage-api-client": "^14.15", "phpstan/phpstan": "^1.10", "phpstan/phpstan-phpunit": "^1.3", - "phpunit/phpunit": "^9.6", - "vlucas/phpdotenv": "^5.6" + "phpstan/phpstan-webmozart-assert": "^1.2", + "sempro/phpunit-pretty-print": "^1.4", + "symfony/http-client": "^6.4|^7.0" }, "type": "library", "autoload": { "psr-4": { - "Keboola\\Sandboxes\\Api\\": "src/" + "Keboola\\SandboxesServiceApiClient\\": "src/" } }, "notification-url": "https://packagist.org/downloads/", @@ -1334,19 +1333,64 @@ ], "authors": [ { - "name": "Jakub Matějka", - "email": "jakub@keboola.com" + "name": "Keboola", + "email": "devel@keboola.com" } ], - "description": "PHP Client for Keboola Sandboxes API", - "keywords": [ - "keboola", - "sandboxes" + "description": "Keboola Sandboxes Service API client", + "support": { + "issues": "https://github.com/keboola/sandboxes-service-api-php-client/issues", + "source": "https://github.com/keboola/sandboxes-service-api-php-client/tree/odin/AJDA-1942" + }, + "time": "2026-04-24T21:26:08+00:00" + }, + { + "name": "keboola/service-client", + "version": "1.5.1", + "source": { + "type": "git", + "url": "https://github.com/keboola/service-client.git", + "reference": "0629d1e54b9cbd0fd66bbec01424705464794caf" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/keboola/service-client/zipball/0629d1e54b9cbd0fd66bbec01424705464794caf", + "reference": "0629d1e54b9cbd0fd66bbec01424705464794caf", + "shasum": "" + }, + "require": { + "php": ">=8.2" + }, + "require-dev": { + "infection/infection": "^0.29", + "keboola/coding-standard": "^15.0.1", + "monolog/monolog": "^3.5.0", + "phpstan/phpstan": "^2.1", + "phpunit/phpunit": "^12.1", + "symfony/dotenv": "^6.4.4|^7.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Keboola\\ServiceClient\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Keboola", + "email": "devel@keboola.com" + } ], + "description": "Service Client provides easy way to get Keboola services URLs", "support": { - "source": "https://github.com/keboola/sandboxes-api-php-client/tree/6.32.0" + "issues": "https://github.com/keboola/service-client/issues", + "source": "https://github.com/keboola/service-client/tree/1.5.1" }, - "time": "2025-11-19T16:15:43+00:00" + "time": "2026-04-14T06:59:15+00:00" }, { "name": "keboola/storage-api-client", @@ -2763,6 +2807,185 @@ ], "time": "2026-02-25T16:50:00+00:00" }, + { + "name": "symfony/http-client", + "version": "v7.4.8", + "source": { + "type": "git", + "url": "https://github.com/symfony/http-client.git", + "reference": "01933e626c3de76bea1e22641e205e78f6a34342" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/http-client/zipball/01933e626c3de76bea1e22641e205e78f6a34342", + "reference": "01933e626c3de76bea1e22641e205e78f6a34342", + "shasum": "" + }, + "require": { + "php": ">=8.2", + "psr/log": "^1|^2|^3", + "symfony/deprecation-contracts": "^2.5|^3", + "symfony/http-client-contracts": "~3.4.4|^3.5.2", + "symfony/polyfill-php83": "^1.29", + "symfony/service-contracts": "^2.5|^3" + }, + "conflict": { + "amphp/amp": "<2.5", + "amphp/socket": "<1.1", + "php-http/discovery": "<1.15", + "symfony/http-foundation": "<6.4" + }, + "provide": { + "php-http/async-client-implementation": "*", + "php-http/client-implementation": "*", + "psr/http-client-implementation": "1.0", + "symfony/http-client-implementation": "3.0" + }, + "require-dev": { + "amphp/http-client": "^4.2.1|^5.0", + "amphp/http-tunnel": "^1.0|^2.0", + "guzzlehttp/promises": "^1.4|^2.0", + "nyholm/psr7": "^1.0", + "php-http/httplug": "^1.0|^2.0", + "psr/http-client": "^1.0", + "symfony/amphp-http-client-meta": "^1.0|^2.0", + "symfony/cache": "^6.4|^7.0|^8.0", + "symfony/dependency-injection": "^6.4|^7.0|^8.0", + "symfony/http-kernel": "^6.4|^7.0|^8.0", + "symfony/messenger": "^6.4|^7.0|^8.0", + "symfony/process": "^6.4|^7.0|^8.0", + "symfony/rate-limiter": "^6.4|^7.0|^8.0", + "symfony/stopwatch": "^6.4|^7.0|^8.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Symfony\\Component\\HttpClient\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Provides powerful methods to fetch HTTP resources synchronously or asynchronously", + "homepage": "https://symfony.com", + "keywords": [ + "http" + ], + "support": { + "source": "https://github.com/symfony/http-client/tree/v7.4.8" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-03-30T12:55:43+00:00" + }, + { + "name": "symfony/http-client-contracts", + "version": "v3.6.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/http-client-contracts.git", + "reference": "75d7043853a42837e68111812f4d964b01e5101c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/75d7043853a42837e68111812f4d964b01e5101c", + "reference": "75d7043853a42837e68111812f4d964b01e5101c", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/contracts", + "name": "symfony/contracts" + }, + "branch-alias": { + "dev-main": "3.6-dev" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Contracts\\HttpClient\\": "" + }, + "exclude-from-classmap": [ + "/Test/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Generic abstractions related to HTTP clients", + "homepage": "https://symfony.com", + "keywords": [ + "abstractions", + "contracts", + "decoupling", + "interfaces", + "interoperability", + "standards" + ], + "support": { + "source": "https://github.com/symfony/http-client-contracts/tree/v3.6.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2025-04-29T11:18:49+00:00" + }, { "name": "symfony/polyfill-ctype", "version": "v1.33.0", @@ -3606,6 +3829,64 @@ } ], "time": "2026-03-06T11:10:17+00:00" + }, + { + "name": "webmozart/assert", + "version": "1.12.1", + "source": { + "type": "git", + "url": "https://github.com/webmozarts/assert.git", + "reference": "9be6926d8b485f55b9229203f962b51ed377ba68" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/webmozarts/assert/zipball/9be6926d8b485f55b9229203f962b51ed377ba68", + "reference": "9be6926d8b485f55b9229203f962b51ed377ba68", + "shasum": "" + }, + "require": { + "ext-ctype": "*", + "ext-date": "*", + "ext-filter": "*", + "php": "^7.2 || ^8.0" + }, + "suggest": { + "ext-intl": "", + "ext-simplexml": "", + "ext-spl": "" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.10-dev" + } + }, + "autoload": { + "psr-4": { + "Webmozart\\Assert\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Bernhard Schussek", + "email": "bschussek@gmail.com" + } + ], + "description": "Assertions to validate method input/output with nice error messages.", + "keywords": [ + "assert", + "check", + "validate" + ], + "support": { + "issues": "https://github.com/webmozarts/assert/issues", + "source": "https://github.com/webmozarts/assert/tree/1.12.1" + }, + "time": "2025-10-29T15:56:20+00:00" } ], "packages-dev": [ @@ -3744,7 +4025,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "keboola/sandboxes-service-api-client": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/Keboola/Console/Command/DeleteOrganizationOwnerlessWorkspaces.php b/src/Keboola/Console/Command/DeleteOrganizationOwnerlessWorkspaces.php index a559604..b77019a 100644 --- a/src/Keboola/Console/Command/DeleteOrganizationOwnerlessWorkspaces.php +++ b/src/Keboola/Console/Command/DeleteOrganizationOwnerlessWorkspaces.php @@ -2,12 +2,17 @@ namespace Keboola\Console\Command; +use Keboola\JobQueueClient\Client as JobQueueClient; +use Keboola\JobQueueClient\JobData; use Keboola\ManageApi\Client; -use Keboola\Sandboxes\Api\Client as SandboxesClient; -use Keboola\Sandboxes\Api\Sandbox; +use Keboola\SandboxesServiceApiClient\ApiClientConfiguration; +use Keboola\SandboxesServiceApiClient\Apps\AppsApiClient; +use Keboola\ServiceClient\ServiceClient; +use Keboola\StorageApi\BranchAwareClient; use Keboola\StorageApi\Client as StorageApiClient; +use Keboola\StorageApi\Components; +use Keboola\StorageApi\Options\Components\ListComponentConfigurationsOptions; use Keboola\StorageApi\Tokens; -use Keboola\StorageApi\Workspaces; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; @@ -21,7 +26,7 @@ protected function configure(): void { $this ->setName('manage:delete-organization-ownerless-workspaces') - ->setDescription('Bulk delete ownerless workspaces (sandboxes with inactive token owner) across all projects in an organization.') + ->setDescription('Bulk delete ownerless workspaces (sessions with inactive user) across all projects in an organization.') ->addOption('force', 'f', InputOption::VALUE_NONE, 'Use [--force, -f] to do it for real.') ->addOption( 'includeShared', @@ -59,9 +64,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int $organizationId = (int) $organizationId; $hostnameSuffix = $input->getArgument('hostnameSuffix'); assert(is_string($hostnameSuffix)); + assert($hostnameSuffix !== ''); $kbcUrl = sprintf('https://connection.%s', $hostnameSuffix); - $sandboxesUrl = sprintf('https://sandboxes.%s', $hostnameSuffix); + $editorUrl = sprintf('https://editor.%s', $hostnameSuffix); + $serviceClient = new ServiceClient($hostnameSuffix); $includeShared = (bool) $input->getOption('includeShared'); $force = (bool) $input->getOption('force'); @@ -78,9 +85,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int $output->writeln('This is just a dry-run, nothing will be actually deleted'); } - $totalDeletedSandboxes = 0; - $totalDeletedStorageWorkspaces = 0; - /** @var array> $summary */ + $totalDeleted = 0; + /** @var array> $summary */ $summary = []; foreach ($projects as $project) { @@ -113,90 +119,135 @@ protected function execute(InputInterface $input, OutputInterface $output): int 'backoffMaxTries' => 1, 'logger' => new ConsoleLogger($output), ]); - $workspacesClient = new Workspaces($storageClient); $tokensClient = new Tokens($storageClient); - $sandboxesClient = new SandboxesClient( - $sandboxesUrl, - $storageToken['token'], - ); + $editorClient = new EditorServiceClient($editorUrl, $storageToken['token']); - $projectDeletedSandboxes = 0; - $projectDeletedStorageWorkspaces = 0; + // Build a set of active user IDs and token IDs from project tokens + $activeUserIds = []; + $activeTokenIds = []; + foreach ($tokensClient->listTokens() as $projectToken) { + $activeTokenIds[$projectToken['id']] = true; + if (isset($projectToken['admin']['id'])) { + $activeUserIds[$projectToken['admin']['id']] = true; + } + } + + $projectDeleted = 0; $projectKey = sprintf('%s (%s)', $project['name'], $project['id']); $summary[$projectKey] = []; - $sandboxes = $sandboxesClient->list(); - $workingTokens = 0; - /** @var Sandbox $sandbox */ - foreach ($sandboxes as $sandbox) { - $tokenId = $sandbox->getTokenId(); - try { - if ($tokenId !== null) { - $tokensClient->getToken((int) $tokenId); - $workingTokens++; - if ($output->isVerbose()) { - $output->writeln('Working token ' . $tokenId); - } - continue; // token exists so no need to do anything - } - } catch (\Throwable $exception) { - if ($exception->getCode() === 403) { - $output->writeln(sprintf( - 'WARN: Access denied checking token %s for sandbox %s, skipping', - $sandbox->getTokenId(), - $sandbox->getId(), - )); - continue; - } - if ($exception->getCode() !== 404) { - throw $exception; + foreach ($editorClient->listSessions() as $session) { + if (isset($activeUserIds[$session['userId']])) { + if ($output->isVerbose()) { + $output->writeln('Active user ' . $session['userId']); } + continue; } - if (!$includeShared && $sandbox->getShared()) { + if (!$includeShared && $session['shared']) { continue; } - $physicalId = ''; - if (!in_array($sandbox->getType(), Sandbox::CONTAINER_TYPES)) { - if (empty($sandbox->getPhysicalId())) { - $output->writeln('No underlying storage workspace found for sandboxId ' . $sandbox->getId()); - } else { - $physicalId = $sandbox->getPhysicalId(); - $output->writeln('Deleting inactive storage workspace ' . $physicalId); - $projectDeletedStorageWorkspaces++; - if ($force) { - $this->deleteStorageWorkspace($workspacesClient, $physicalId, $output); - } - } - } elseif (!empty($sandbox->getStagingWorkspaceId())) { - $physicalId = $sandbox->getStagingWorkspaceId(); - $output->writeln('Deleting inactive staging storage workspace ' . $physicalId); - $projectDeletedStorageWorkspaces++; - if ($force) { - $this->deleteStorageWorkspace($workspacesClient, $physicalId, $output); + $output->writeln(sprintf( + 'Deleting configuration %s/%s (branch %s) for session %s', + $session['componentId'], + $session['configurationId'], + $session['branchId'], + $session['id'], + )); + + $summary[$projectKey][] = [ + 'sessionId' => $session['id'], + 'componentId' => $session['componentId'], + 'configurationId' => $session['configurationId'], + 'userId' => $session['userId'], + ]; + + $projectDeleted++; + if ($force) { + $branchClient = new BranchAwareClient($session['branchId'], [ + 'token' => $storageToken['token'], + 'url' => $kbcUrl, + ]); + $components = new Components($branchClient); + // First call moves the configuration to trash, second call permanently purges it. + $components->deleteConfiguration($session['componentId'], $session['configurationId']); + $components->deleteConfiguration($session['componentId'], $session['configurationId']); + } + } + + // Handle Python/R sandboxes via sandbox-service + $appsClient = new AppsApiClient(new ApiClientConfiguration( + baseUrl: $serviceClient->getSandboxesServiceUrl(), + storageToken: $storageToken['token'], + userAgent: 'Keboola CLI Utils', + )); + + $storageComponents = new Components($storageClient); + $sandboxConfigCreatorTokens = []; + foreach ($storageComponents->listComponentConfigurations( + (new ListComponentConfigurationsOptions())->setComponentId('keboola.sandboxes'), + ) as $config) { + $sandboxConfigCreatorTokens[$config['id']] = $config['creatorToken']['id'] ?? null; + } + + $queueClient = new JobQueueClient($serviceClient->getQueueUrl(), $storageToken['token']); + + foreach ($appsClient->listApps(types: ['python', 'r']) as $app) { + $creatorTokenId = $sandboxConfigCreatorTokens[$app->getConfigId()] ?? null; + if ($creatorTokenId !== null && isset($activeTokenIds[$creatorTokenId])) { + if ($output->isVerbose()) { + $output->writeln('Active token for app ' . $app->getId()); } + continue; } + $output->writeln(sprintf( + 'Deleting sandbox config keboola.sandboxes/%s (branch %s) for app %s', + $app->getConfigId(), + $app->getBranchId() ?? 'default', + $app->getId(), + )); + $summary[$projectKey][] = [ - 'sandboxId' => $sandbox->getId(), - 'physicalId' => $physicalId, - 'tokenId' => (string) ($sandbox->getTokenId() ?? ''), + 'sessionId' => $app->getId(), + 'componentId' => 'keboola.sandboxes', + 'configurationId' => $app->getConfigId(), + 'userId' => (string) ($creatorTokenId ?? ''), ]; - $projectDeletedSandboxes++; + $projectDeleted++; if ($force) { - $sandboxesClient->delete($sandbox->getId()); + try { + $queueClient->createJob(new JobData( + componentId: 'keboola.sandboxes', + configId: $app->getConfigId(), + configData: [ + 'parameters' => [ + 'task' => 'delete', + 'id' => $app->getId(), + ], + ], + branchId: $app->getBranchId(), + )); + } catch (\Throwable $e) { + $output->writeln(sprintf( + 'WARN: Job creation failed for app %s, falling back to direct deletion: %s', + $app->getId(), + $e->getMessage(), + )); + $appsClient->deleteApp($app->getId()); + // First call moves the configuration to trash, second call permanently purges it. + $storageComponents->deleteConfiguration('keboola.sandboxes', $app->getConfigId()); + $storageComponents->deleteConfiguration('keboola.sandboxes', $app->getConfigId()); + } } } - $output->writeln('Working tokens ' . $workingTokens); - $output->writeln(sprintf( - 'Project %s: %d sandboxes deleted, %d storage workspaces deleted', + 'Project %s: %d sessions/apps deleted', $project['id'], - $projectDeletedSandboxes, - $projectDeletedStorageWorkspaces, + $projectDeleted, )); try { @@ -209,50 +260,33 @@ protected function execute(InputInterface $input, OutputInterface $output): int )); } - $totalDeletedSandboxes += $projectDeletedSandboxes; - $totalDeletedStorageWorkspaces += $projectDeletedStorageWorkspaces; + $totalDeleted += $projectDeleted; } // Print summary $output->writeln(''); $output->writeln(sprintf('=== Summary for organization %s ===', $organization['name'] ?? $organizationId)); - foreach ($summary as $projectKey => $workspaces) { - if (count($workspaces) === 0) { + foreach ($summary as $projectKey => $sessions) { + if (count($sessions) === 0) { continue; } $output->writeln(sprintf(' Project: %s', $projectKey)); - foreach ($workspaces as $workspace) { + foreach ($sessions as $session) { $output->writeln(sprintf( - ' - SandboxId: %s, PhysicalId: %s, TokenId: %s', - $workspace['sandboxId'], - $workspace['physicalId'] ?: '(none)', - $workspace['tokenId'] ?: '(none)', + ' - SessionId: %s, Configuration: %s/%s, UserId: %s', + $session['sessionId'], + $session['componentId'], + $session['configurationId'], + $session['userId'] ?: '(none)', )); } } $output->writeln(''); $output->writeln(sprintf( - 'Grand total: %d sandboxes deleted and %d storage workspaces deleted', - $totalDeletedSandboxes, - $totalDeletedStorageWorkspaces, + 'Grand total: %d sessions/apps deleted', + $totalDeleted, )); return 0; } - - private function deleteStorageWorkspace( - Workspaces $workspacesClient, - string $workspaceId, - OutputInterface $output, - ): void { - try { - $workspacesClient->deleteWorkspace((int) $workspaceId); - } catch (\Throwable $clientException) { - $output->writeln(sprintf( - 'Error deleting workspace %s:%s', - $workspaceId, - $clientException->getMessage(), - )); - } - } } diff --git a/src/Keboola/Console/Command/DeleteOwnerlessWorkspaces.php b/src/Keboola/Console/Command/DeleteOwnerlessWorkspaces.php index c30c462..92d38cb 100644 --- a/src/Keboola/Console/Command/DeleteOwnerlessWorkspaces.php +++ b/src/Keboola/Console/Command/DeleteOwnerlessWorkspaces.php @@ -2,12 +2,16 @@ namespace Keboola\Console\Command; -use Keboola\Sandboxes\Api\Client as SandboxesClient; -use Keboola\Sandboxes\Api\Exception\ClientException; -use Keboola\Sandboxes\Api\Sandbox; +use Keboola\JobQueueClient\Client as JobQueueClient; +use Keboola\JobQueueClient\JobData; +use Keboola\SandboxesServiceApiClient\ApiClientConfiguration; +use Keboola\SandboxesServiceApiClient\Apps\AppsApiClient; +use Keboola\ServiceClient\ServiceClient; +use Keboola\StorageApi\BranchAwareClient; use Keboola\StorageApi\Client as StorageApiClient; +use Keboola\StorageApi\Components; +use Keboola\StorageApi\Options\Components\ListComponentConfigurationsOptions; use Keboola\StorageApi\Tokens; -use Keboola\StorageApi\Workspaces; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; @@ -46,10 +50,12 @@ protected function execute(InputInterface $input, OutputInterface $output): int { $token = $input->getArgument('storageToken'); assert(is_string($token)); + assert($token !== ''); $hostnameSuffix = $input->getArgument('hostnameSuffix'); assert(is_string($hostnameSuffix)); + assert($hostnameSuffix !== ''); $url = 'https://connection.' . $hostnameSuffix; - $sandboxesUrl = 'https://sandboxes.' . $hostnameSuffix; + $editorUrl = 'https://editor.' . $hostnameSuffix; $includeShared = (bool) $input->getOption('includeShared'); $force = (bool) $input->getOption('force'); @@ -59,88 +65,123 @@ protected function execute(InputInterface $input, OutputInterface $output): int 'backoffMaxTries' => 1, 'logger' => new ConsoleLogger($output), ]); - $workspacesClient = new Workspaces($storageClient); $tokensClient = new Tokens($storageClient); - $sandboxesClient = new SandboxesClient( - $sandboxesUrl, - $token - ); + $editorClient = new EditorServiceClient($editorUrl, $token); + if ($force) { $output->writeln('Force option is set, doing it for real'); } else { $output->writeln('This is just a dry-run, nothing will be actually deleted'); } - $totalDeletedSandboxes = 0; - $totalDeletedStorageWorkspaces = 0; - $sandboxes = $sandboxesClient->list(); - /** @var Sandbox $sandbox */ - foreach ($sandboxes as $sandbox) { - try { - $tokenId = $sandbox->getTokenId(); - if ($tokenId !== null) { - $tokensClient->getToken((int) $tokenId); - continue; // token exists so no need to do anything - } - } catch (\Throwable $exception) { - if ($exception->getCode() !== 404) { - throw $exception; - } + + // Build a set of active user IDs and token IDs from project tokens + $activeUserIds = []; + $activeTokenIds = []; + foreach ($tokensClient->listTokens() as $projectToken) { + $activeTokenIds[$projectToken['id']] = true; + if (isset($projectToken['admin']['id'])) { + $activeUserIds[$projectToken['admin']['id']] = true; } + } - // check if we should skip shared sandboxes - if (!$includeShared && $sandbox->getShared()) { - continue; + $totalDeleted = 0; + + foreach ($editorClient->listSessions() as $session) { + if (isset($activeUserIds[$session['userId']])) { + continue; // user is still active } - if (!in_array($sandbox->getType(), Sandbox::CONTAINER_TYPES)) { - // it is a database workspace - if (empty($sandbox->getPhysicalId())) { - $output->writeln('No underlying storage workspace found for sandboxId ' . $sandbox->getId()); - } else { - $output->writeln('Deleting inactive storage workspace ' . $sandbox->getPhysicalId()); - $totalDeletedStorageWorkspaces++; - if ($force) { - $this->deleteStorageWorkspace($workspacesClient, $sandbox->getPhysicalId(), $output); - } - } - } elseif (!empty($sandbox->getStagingWorkspaceId())) { - $output->writeln('Deleting inactive staging storage workspace ' . $sandbox->getStagingWorkspaceId()); - $totalDeletedStorageWorkspaces++; - if ($force) { - $this->deleteStorageWorkspace($workspacesClient, $sandbox->getStagingWorkspaceId(), $output); - } + if (!$includeShared && $session['shared']) { + continue; } - $totalDeletedSandboxes++; + $branchId = $session['branchId']; + $componentId = $session['componentId']; + $configurationId = $session['configurationId']; + $sessionId = $session['id']; + + $output->writeln(sprintf( + 'Deleting configuration %s/%s (branch %s) for session %s', + $componentId, + $configurationId, + $branchId, + $sessionId, + )); + + $totalDeleted++; if ($force) { - $sandboxesClient->delete($sandbox->getId()); + $branchClient = new BranchAwareClient($branchId, [ + 'token' => $token, + 'url' => $url, + ]); + $components = new Components($branchClient); + // First call moves the configuration to trash, second call permanently purges it. + $components->deleteConfiguration($componentId, $configurationId); + $components->deleteConfiguration($componentId, $configurationId); } } - $output->writeln(sprintf( - '%d sandboxes deleted and %d storage workspaces deleted', - $totalDeletedSandboxes, - $totalDeletedStorageWorkspaces + // Handle Python/R sandboxes via sandbox-service + $serviceClient = new ServiceClient($hostnameSuffix); + $appsClient = new AppsApiClient(new ApiClientConfiguration( + baseUrl: $serviceClient->getSandboxesServiceUrl(), + storageToken: $token, + userAgent: 'Keboola CLI Utils', )); - return 0; - } + $storageComponents = new Components($storageClient); + $sandboxConfigCreatorTokens = []; + foreach ($storageComponents->listComponentConfigurations( + (new ListComponentConfigurationsOptions())->setComponentId('keboola.sandboxes'), + ) as $config) { + $sandboxConfigCreatorTokens[$config['id']] = $config['creatorToken']['id'] ?? null; + } - private function deleteStorageWorkspace( - Workspaces $workspacesClient, - string $workspaceId, - OutputInterface $output - ): void { - try { - $workspacesClient->deleteWorkspace((int) $workspaceId); - } catch (\Throwable $clientException) { - $output->writeln( - sprintf( - 'Error deleting workspace %s:%s', - $workspaceId, - $clientException->getMessage() - ) - ); + $queueClient = new JobQueueClient($serviceClient->getQueueUrl(), $token); + + foreach ($appsClient->listApps(types: ['python', 'r']) as $app) { + $creatorTokenId = $sandboxConfigCreatorTokens[$app->getConfigId()] ?? null; + if ($creatorTokenId !== null && isset($activeTokenIds[$creatorTokenId])) { + continue; + } + + $output->writeln(sprintf( + 'Deleting sandbox config keboola.sandboxes/%s (branch %s) for app %s', + $app->getConfigId(), + $app->getBranchId() ?? 'default', + $app->getId(), + )); + + $totalDeleted++; + if ($force) { + try { + $queueClient->createJob(new JobData( + componentId: 'keboola.sandboxes', + configId: $app->getConfigId(), + configData: [ + 'parameters' => [ + 'task' => 'delete', + 'id' => $app->getId(), + ], + ], + branchId: $app->getBranchId(), + )); + } catch (\Throwable $e) { + $output->writeln(sprintf( + 'WARN: Job creation failed for app %s, falling back to direct deletion: %s', + $app->getId(), + $e->getMessage(), + )); + $appsClient->deleteApp($app->getId()); + // First call moves the configuration to trash, second call permanently purges it. + $storageComponents->deleteConfiguration('keboola.sandboxes', $app->getConfigId()); + $storageComponents->deleteConfiguration('keboola.sandboxes', $app->getConfigId()); + } + } } + + $output->writeln(sprintf('%d sessions/apps deleted', $totalDeleted)); + + return 0; } } diff --git a/src/Keboola/Console/Command/DeleteProjectSandboxes.php b/src/Keboola/Console/Command/DeleteProjectSandboxes.php deleted file mode 100644 index 73a7fa0..0000000 --- a/src/Keboola/Console/Command/DeleteProjectSandboxes.php +++ /dev/null @@ -1,123 +0,0 @@ -setName('storage:delete-project-sandboxes') - ->setDescription('Bulk delete project sandboxes.') - ->addOption('force', 'f', InputOption::VALUE_NONE, 'Use [--force, -f] to do it for real.') - ->addOption( - 'includeShared', - null, - InputOption::VALUE_NONE, - 'Use option --includeShared if you would also like to delete shared workspaces.', - ) - ->addArgument( - 'storageToken', - InputArgument::REQUIRED, - 'Keboola Storage API token to use' - ) - ->addArgument( - 'hostnameSuffix', - InputArgument::OPTIONAL, - 'Keboola Connection Hostname Suffix', - 'keboola.com' - ); - } - - protected function execute(InputInterface $input, OutputInterface $output): int - { - $token = $input->getArgument('storageToken'); - assert(is_string($token)); - $hostnameSuffix = $input->getArgument('hostnameSuffix'); - assert(is_string($hostnameSuffix)); - $url = 'https://connection.' . $hostnameSuffix; - $sandboxesUrl = 'https://sandboxes.' . $hostnameSuffix; - $includeShared = (bool) $input->getOption('includeShared'); - $force = (bool) $input->getOption('force'); - - $storageClient = new StorageApiClient([ - 'token' => $token, - 'url' => $url, - ]); - $workspacesClient = new Workspaces($storageClient); - $sandboxesClient = new SandboxesClient( - $sandboxesUrl, - $token - ); - if ($force) { - $output->writeln('Force option is set, doing it for real'); - } else { - $output->writeln('This is just a dry-run, nothing will be actually deleted'); - } - $totalDeletedSandboxes = 0; - $totalDeletedStorageWorkspaces = 0; - $sandboxes = $sandboxesClient->list(); - /** @var Sandbox $sandbox */ - foreach ($sandboxes as $sandbox) { - // check if we should skip shared sandboxes - if (!$includeShared && $sandbox->getShared()) { - continue; - } - - if ($sandbox->getDeletedTimestamp()) { - $output->writeln('Skipping already deleted sandbox ' . $sandbox->getId()); - continue; - } - - if (!in_array($sandbox->getType(), Sandbox::CONTAINER_TYPES)) { - // it is a database workspace - $output->writeln('Deleting storage workspace ' . $sandbox->getPhysicalId()); - if (!empty($sandbox->getPhysicalId())) { - if ($force) { - try { - $workspacesClient->deleteWorkspace((int) $sandbox->getPhysicalId()); - $totalDeletedStorageWorkspaces++; - } catch (Exception $exception) { - if ($exception->getCode() === 404) { - $output->writeln("Storage workspace not found"); - } - } - } - } else { - $output->writeln("No physical ID found, skipping"); - } - } elseif (!empty($sandbox->getStagingWorkspaceId())) { - $output->writeln('Deleting staging storage workspace ' . $sandbox->getPhysicalId()); - $totalDeletedStorageWorkspaces++; - if ($force) { - $workspacesClient->deleteWorkspace((int) $sandbox->getStagingWorkspaceId(), [], true); - } - } - - $totalDeletedSandboxes++; - $output->writeln('Deleting sandbox ' . $sandbox->getId()); - if ($force) { - $sandboxesClient->delete($sandbox->getId()); - } - } - - $output->writeln(sprintf( - '%d sandboxes deleted and %d storage workspaces deleted', - $totalDeletedSandboxes, - $totalDeletedStorageWorkspaces - )); - - return 0; - } -} diff --git a/src/Keboola/Console/Command/EditorServiceClient.php b/src/Keboola/Console/Command/EditorServiceClient.php new file mode 100644 index 0000000..8e08ac0 --- /dev/null +++ b/src/Keboola/Console/Command/EditorServiceClient.php @@ -0,0 +1,34 @@ +httpClient = HttpClient::createForBaseUri($url, [ + 'headers' => [ + 'X-StorageApi-Token' => $token, + ], + ]); + } + + /** + * @return list + */ + public function listSessions(): array + { + $response = $this->httpClient->request('GET', '/sql/sessions', [ + 'query' => ['listAll' => '1'], + ]); + + /** @var list $sessions */ + $sessions = $response->toArray(); + return $sessions; + } +} diff --git a/src/Keboola/Console/Command/MassDeleteProjectWorkspaces.php b/src/Keboola/Console/Command/MassDeleteProjectWorkspaces.php index d88a310..d0dd678 100644 --- a/src/Keboola/Console/Command/MassDeleteProjectWorkspaces.php +++ b/src/Keboola/Console/Command/MassDeleteProjectWorkspaces.php @@ -6,14 +6,9 @@ use InvalidArgumentException; use Keboola\Csv\CsvFile; -use Keboola\JobQueueClient\JobData; -use Keboola\Sandboxes\Api\Client as SandboxesClient; -use Keboola\JobQueueClient\Client as QueueClient; -use Keboola\Sandboxes\Api\ListOptions; -use Keboola\Sandboxes\Api\Sandbox; +use Keboola\StorageApi\BranchAwareClient; use Keboola\StorageApi\Client; -use Keboola\StorageApi\DevBranches; -use Keboola\StorageApi\Workspaces; +use Keboola\StorageApi\Components; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; @@ -42,8 +37,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $stackSuffix = $input->getArgument(self::ARGUMENT_STACK_SUFFIX); assert(is_string($stackSuffix)); $connectionUrl = 'https://connection.' . $stackSuffix; - $sandboxesUrl = 'https://sandboxes.' . $stackSuffix; - $jobsUrl = 'https://queue.' . $stackSuffix; + $editorUrl = 'https://editor.' . $stackSuffix; $sourceFile = $input->getArgument(self::ARGUMENT_SOURCE_FILE); assert(is_string($sourceFile)); $output->writeln(sprintf('Fetching projects from "%s"', $sourceFile)); @@ -79,15 +73,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int } } - // testing override -// $map = [ -// '232' => [ -// 'WORKSPACE_832798053', -// 'WORKSPACE_965913339', -// ], -// ]; - - foreach ($map as $projectId => $workspacesSchemasToDelete) { + foreach ($map as $projectId => $workspaceSchemasToDelete) { /** @var \Symfony\Component\Console\Helper\QuestionHelper $helper */ $helper = $this->getHelper('question'); $question = new Question(sprintf( @@ -97,129 +83,62 @@ protected function execute(InputInterface $input, OutputInterface $output): int $storageToken = $helper->ask($input, $output, $question); assert(is_string($storageToken)); - $storageClient = new Client([ - 'token' => $storageToken, - 'url' => $connectionUrl, - ]); - $sandboxesClient = new SandboxesClient( - $sandboxesUrl, - $storageToken - ); - $jobsClient = new QueueClient( - $jobsUrl, - $storageToken - ); - - $branchesClient = new DevBranches($storageClient); - - /** - * @var array}, sandbox: \Keboola\Sandboxes\Api\Sandbox}> $jobs - */ - $jobs = []; - foreach ($branchesClient->listBranches() as $branch) { - $output->writeln(sprintf('Checking branch "%s" for sandboxes.', $branch['id'])); - $branchId = (string) $branch['id']; - if ($branch['isDefault']) { - $branchId = null; - } - /** @var Sandbox $sandbox */ - foreach ($sandboxesClient->list((new ListOptions())->setBranchId($branchId)) as $sandbox) { - $schema = $sandbox->getWorkspaceDetails()['connection']['schema'] ?? null; - if (!in_array($schema, $workspacesSchemasToDelete, true)) { - continue; - } - $output->writeln(sprintf( - 'Sandbox "%s" with schema "%s" found.', - $sandbox->getId(), - $schema, - )); + $editorClient = new EditorServiceClient($editorUrl, $storageToken); - // remove found schema from map - unset($map[$projectId][array_search($schema, $map[$projectId], true)]); - - if ($force) { - $job = $jobsClient->createJob(new JobData( - 'keboola.sandboxes', - null, - [ - 'parameters' => [ - 'task' => 'delete', - 'id' => $sandbox->getId(), - ], - ], - )); - - /** - * @var array{id: string|int, ...} $jobArray - */ - $jobArray = (array) $job; - $jobs[] = ['job' => $jobArray, 'sandbox' => $sandbox]; - - $output->writeln(sprintf( - 'Created delete job "%s" for project "%s"', - $jobArray['id'], - $projectId - )); - } else { - $output->writeln(sprintf( - '[DRY-RUN] Created delete job "%s" for project "%s"', - '', - $projectId - )); - } - } + // index sessions by workspaceSchema for quick lookup + $sessionsBySchema = []; + foreach ($editorClient->listSessions() as $session) { + $sessionsBySchema[$session['workspaceSchema']] = $session; } - $output->writeln('Waiting for delete jobs to finish.'); - while (count($jobs) > 0) { - foreach ($jobs as $i => $jobData) { - $job = $jobData['job']; - $sandbox = $jobData['sandbox']; - /** - * @var array{id: string|int, isFinished: bool, status: string, ...} $jobRes - */ - $jobRes = (array) $jobsClient->getJob((string) $job['id']); - if ($jobRes['isFinished'] === true) { - $workspaceDetails = $sandbox->getWorkspaceDetails(); - $schema = $workspaceDetails['connection']['schema'] ?? 'unknown'; - $output->writeln(sprintf( - 'Delete job "%s" for sandbox "%s" with schema "%s" finished with status "%s"', - $job['id'], - $sandbox->getId(), - $schema, - $jobRes['status'] - )); - unset($jobs[$i]); - } + $notFound = []; + foreach ($workspaceSchemasToDelete as $schema) { + if (!isset($sessionsBySchema[$schema])) { + $notFound[] = $schema; + continue; } - sleep(2); - } - foreach ($branchesClient->listBranches() as $branch) { - $output->writeln(sprintf('Checking branch "%s" for storage workspaces.', $branch['id'])); - $workspacesClient = new Workspaces( - $storageClient->getBranchAwareClient($branch['id']) - ); - foreach ($workspacesClient->listWorkspaces() as $workspace) { - $schema = $workspace['connection']['schema'] ?? null; - if ($schema === null || !in_array($schema, $map[$projectId], true)) { - continue; - } - // remove found schema from map - unset($map[$projectId][array_search($schema, $map[$projectId], true)]); - if ($force) { - $output->writeln(sprintf('Deleting workspace "%s" with schema "%s"', $workspace['id'], $schema)); - $workspacesClient->deleteWorkspace($workspace['id'], [], true); - } else { - $output->writeln(sprintf('[DRY-RUN] Deleting workspace "%s" with schema "%s"', $workspace['id'], $workspace['connection']['schema'])); - } + $session = $sessionsBySchema[$schema]; + + $output->writeln(sprintf( + 'Session "%s" with schema "%s" found — configuration %s/%s (branch %s).', + $session['id'], + $schema, + $session['componentId'], + $session['configurationId'], + $session['branchId'], + )); + + if ($force) { + $branchClient = new BranchAwareClient($session['branchId'], [ + 'token' => $storageToken, + 'url' => $connectionUrl, + ]); + $components = new Components($branchClient); + // First call moves the configuration to trash, second call permanently purges it. + $components->deleteConfiguration($session['componentId'], $session['configurationId']); + $components->deleteConfiguration($session['componentId'], $session['configurationId']); + + $output->writeln(sprintf( + 'Deleted configuration %s/%s for schema "%s".', + $session['componentId'], + $session['configurationId'], + $schema, + )); + } else { + $output->writeln(sprintf( + '[DRY-RUN] Would delete configuration %s/%s for schema "%s".', + $session['componentId'], + $session['configurationId'], + $schema, + )); } } - if (count($map[$projectId]) !== 0) { + if (count($notFound) !== 0) { $output->writeln([ - 'Following schemas were not found (are deleted or needs to be deleted manually):', - implode(', ', $map[$projectId]), + 'Following schemas were not found (are deleted or need to be deleted manually):', + implode(', ', $notFound), ]); } }