From 97af6ccbc3cba2a296e152f556b2d380766699be Mon Sep 17 00:00:00 2001 From: Alexander Piskun <13381981+bigcat88@users.noreply.github.com> Date: Wed, 17 Apr 2024 11:48:38 +0300 Subject: [PATCH 1/4] extra image pulling for selected "compute device" Signed-off-by: Alexander Piskun --- CHANGELOG.md | 1 + docs/tech_details/InstallationFlow.rst | 22 +++++++++++- lib/DeployActions/DockerActions.php | 46 +++++++++++++++++++------- 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7b1761d..699307ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Different compute device configuration for Daemon (NVIDIA, AMD, CPU). #267 - Ability to add optional parameters when registering a daemon, for example *OVERRIDE_APP_HOST*. #269 - Correct support of the Docker `HEALTHCHECK` instruction. #273 +- Support of pulling "custom" images for the selected compute device. #274 ### Fixed diff --git a/docs/tech_details/InstallationFlow.rst b/docs/tech_details/InstallationFlow.rst index 13276ce1..1c0810f8 100644 --- a/docs/tech_details/InstallationFlow.rst +++ b/docs/tech_details/InstallationFlow.rst @@ -3,6 +3,25 @@ App Installation Flow ===================== +Image Pulling(Docker) +--------------------- + +AppAPI **2.5.0+** will always first try to pull a docker image with a ``suffix`` equal to value of *computeDevice*. + +Let us remind you that ``computeDevice`` can take the following values: ``cpu``, ``cuda``, ``rocm`` + +The suffix will be added as follows: + +.. code:: + + return $imageParams['image_src'] . '/' . + $imageParams['image_name'] . '-' . $daemonConfig['computeDevice']['id'] . ':' . $imageParams['image_tag']; + +For ``cpu`` AppAPI will first try to get the image from ``ghcr.io/cloud-py-api/skeleton-cpu:latest``. +In case the image is not found, ``ghcr.io/cloud-py-api/skeleton:latest`` will be pulled. + +If you as an application developer want to have a custom images for any of these values, you can push that extended images to registry in addition to the based one. + Heartbeat --------- @@ -12,7 +31,8 @@ In the case of ``Docker``, this is: #. 1. performing an image pull #. 2. creating container from the docker image -#. 3. waiting until the “/heartbeat” endpoint becomes available with a ``GET`` request. +#. 3. if the container supports `healthcheck` - AppAPI waits for the `healthy` status +#. 4. waiting until the “/heartbeat” endpoint becomes available with a ``GET`` request The application, in response to the request "/heartbeat", should return json: ``{"status": "ok"}``. diff --git a/lib/DeployActions/DockerActions.php b/lib/DeployActions/DockerActions.php index 33bbcf32..7dfcc8a0 100644 --- a/lib/DeployActions/DockerActions.php +++ b/lib/DeployActions/DockerActions.php @@ -47,18 +47,16 @@ public function deployExApp(ExApp $exApp, DaemonConfig $daemonConfig, array $par if (!isset($params['image_params'])) { return 'Missing image_params.'; } - $imageParams = $params['image_params']; - if (!isset($params['container_params'])) { return 'Missing container_params.'; } - $containerParams = $params['container_params']; $dockerUrl = $this->buildDockerUrl($daemonConfig); $this->initGuzzleClient($daemonConfig); $this->exAppService->setAppDeployProgress($exApp, 0); - $result = $this->pullImage($dockerUrl, $imageParams, $exApp, 0, 94); + $imageId = ''; + $result = $this->pullImage($dockerUrl, $params['image_params'], $exApp, 0, 94, $daemonConfig, $imageId); if ($result) { return $result; } @@ -72,7 +70,7 @@ public function deployExApp(ExApp $exApp, DaemonConfig $daemonConfig, array $par } } $this->exAppService->setAppDeployProgress($exApp, 96); - $result = $this->createContainer($dockerUrl, $imageParams, $containerParams); + $result = $this->createContainer($dockerUrl, $imageId, $params['container_params']); if (isset($result['error'])) { return $result['error']; } @@ -93,18 +91,27 @@ public function buildApiUrl(string $dockerUrl, string $route): string { return sprintf('%s/%s/%s', $dockerUrl, self::DOCKER_API_VERSION, $route); } - public function buildImageName(array $imageParams): string { - return $imageParams['image_src'] . '/' . $imageParams['image_name'] . ':' . $imageParams['image_tag']; + public function buildBaseImageName(array $imageParams): string { + return $imageParams['image_src'] . '/' . + $imageParams['image_name'] . ':' . $imageParams['image_tag']; } - public function createContainer(string $dockerUrl, array $imageParams, array $params = []): array { + public function buildExtendedImageName(array $imageParams, DaemonConfig $daemonConfig): ?string { + if (empty($daemonConfig['computeDevice']['id'])) { + return null; + } + return $imageParams['image_src'] . '/' . + $imageParams['image_name'] . '-' . $daemonConfig['computeDevice']['id'] . ':' . $imageParams['image_tag']; + } + + public function createContainer(string $dockerUrl, string $imageId, array $params = []): array { $createVolumeResult = $this->createVolume($dockerUrl, $this->buildExAppVolumeName($params['name'])); if (isset($createVolumeResult['error'])) { return $createVolumeResult; } $containerParams = [ - 'Image' => $this->buildImageName($imageParams), + 'Image' => $imageId, 'Hostname' => $params['hostname'], 'HostConfig' => [ 'NetworkMode' => $params['net'], @@ -200,20 +207,35 @@ public function removeContainer(string $dockerUrl, string $containerId): string return sprintf('Failed to remove container: %s', $containerId); } - public function pullImage(string $dockerUrl, array $params, ExApp $exApp, int $startPercent, int $maxPercent): string { + public function pullImage( + string $dockerUrl, array $params, ExApp $exApp, int $startPercent, int $maxPercent, DaemonConfig $daemonConfig, string &$imageId + ): string { # docs: https://github.com/docker/compose/blob/main/pkg/compose/pull.go $layerInProgress = ['preparing', 'waiting', 'pulling fs layer', 'download', 'extracting', 'verifying checksum']; $layerFinished = ['already exists', 'pull complete']; $disableProgressTracking = false; - $imageId = $this->buildImageName($params); + $imageId = $this->buildExtendedImageName($params, $daemonConfig); + if ($imageId === null) { + $imageId = $this->buildBaseImageName($params); + } $url = $this->buildApiUrl($dockerUrl, sprintf('images/create?fromImage=%s', urlencode($imageId))); - $this->logger->info(sprintf('Pulling ExApp Image: %s', $imageId)); try { + $this->logger->info(sprintf('Pulling ExApp Image: %s', $imageId)); if ($this->useSocket) { $response = $this->guzzleClient->post($url); } else { $response = $this->guzzleClient->post($url, ['stream' => true]); } + if (($response->getStatusCode() === 404) && ($imageId !== $this->buildBaseImageName($params))) { + $imageId = $this->buildBaseImageName($params); + $url = $this->buildApiUrl($dockerUrl, sprintf('images/create?fromImage=%s', urlencode($imageId))); + $this->logger->info(sprintf('Pulling ExApp Image: %s', $imageId)); + if ($this->useSocket) { + $response = $this->guzzleClient->post($url); + } else { + $response = $this->guzzleClient->post($url, ['stream' => true]); + } + } if ($response->getStatusCode() !== 200) { return sprintf('Pulling ExApp Image: %s return status code: %d', $imageId, $response->getStatusCode()); } From 9d26fd1dc7e99cad71f2550f994f98a503e9cb1d Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Wed, 17 Apr 2024 13:36:00 +0300 Subject: [PATCH 2/4] Docker can return "500" error when image is not present Signed-off-by: Alexander Piskun --- lib/DeployActions/DockerActions.php | 163 +++++++++++++++------------- 1 file changed, 86 insertions(+), 77 deletions(-) diff --git a/lib/DeployActions/DockerActions.php b/lib/DeployActions/DockerActions.php index 7dfcc8a0..de299b75 100644 --- a/lib/DeployActions/DockerActions.php +++ b/lib/DeployActions/DockerActions.php @@ -97,11 +97,11 @@ public function buildBaseImageName(array $imageParams): string { } public function buildExtendedImageName(array $imageParams, DaemonConfig $daemonConfig): ?string { - if (empty($daemonConfig['computeDevice']['id'])) { + if (empty($daemonConfig->getDeployConfig()['computeDevice']['id'])) { return null; } return $imageParams['image_src'] . '/' . - $imageParams['image_name'] . '-' . $daemonConfig['computeDevice']['id'] . ':' . $imageParams['image_tag']; + $imageParams['image_name'] . '-' . $daemonConfig->getDeployConfig()['computeDevice']['id'] . ':' . $imageParams['image_tag']; } public function createContainer(string $dockerUrl, string $imageId, array $params = []): array { @@ -210,95 +210,104 @@ public function removeContainer(string $dockerUrl, string $containerId): string public function pullImage( string $dockerUrl, array $params, ExApp $exApp, int $startPercent, int $maxPercent, DaemonConfig $daemonConfig, string &$imageId ): string { - # docs: https://github.com/docker/compose/blob/main/pkg/compose/pull.go - $layerInProgress = ['preparing', 'waiting', 'pulling fs layer', 'download', 'extracting', 'verifying checksum']; - $layerFinished = ['already exists', 'pull complete']; - $disableProgressTracking = false; $imageId = $this->buildExtendedImageName($params, $daemonConfig); if ($imageId === null) { $imageId = $this->buildBaseImageName($params); + $this->logger->info(sprintf('Pulling "base" image: %s', $imageId)); } - $url = $this->buildApiUrl($dockerUrl, sprintf('images/create?fromImage=%s', urlencode($imageId))); try { - $this->logger->info(sprintf('Pulling ExApp Image: %s', $imageId)); - if ($this->useSocket) { - $response = $this->guzzleClient->post($url); - } else { - $response = $this->guzzleClient->post($url, ['stream' => true]); - } - if (($response->getStatusCode() === 404) && ($imageId !== $this->buildBaseImageName($params))) { - $imageId = $this->buildBaseImageName($params); - $url = $this->buildApiUrl($dockerUrl, sprintf('images/create?fromImage=%s', urlencode($imageId))); - $this->logger->info(sprintf('Pulling ExApp Image: %s', $imageId)); - if ($this->useSocket) { - $response = $this->guzzleClient->post($url); - } else { - $response = $this->guzzleClient->post($url, ['stream' => true]); - } - } - if ($response->getStatusCode() !== 200) { - return sprintf('Pulling ExApp Image: %s return status code: %d', $imageId, $response->getStatusCode()); - } - if ($this->useSocket) { - return ''; - } - $lastPercent = $startPercent; - $layers = []; - $buffer = ''; - $responseBody = $response->getBody(); - while (!$responseBody->eof()) { - $buffer .= $responseBody->read(1024); - try { - while (($newlinePos = strpos($buffer, "\n")) !== false) { - $line = substr($buffer, 0, $newlinePos); - $buffer = substr($buffer, $newlinePos + 1); - $jsonLine = json_decode(trim($line)); - if ($jsonLine) { - if (isset($jsonLine->id) && isset($jsonLine->status)) { - $layerId = $jsonLine->id; - $status = strtolower($jsonLine->status); - foreach ($layerInProgress as $substring) { - if (str_contains($status, $substring)) { - $layers[$layerId] = false; - break; - } + $r = $this->pullImageInternal($dockerUrl, $exApp, $startPercent, $maxPercent, $imageId); + } catch (GuzzleException $e) { + $r = sprintf('Failed to pull image, GuzzleException occur: %s', $e->getMessage()); + } + if (($r === '') || ($imageId === $this->buildBaseImageName($params))) { + return $r; + } + $this->logger->info(sprintf('Pulling "base" image: %s', $imageId)); + $imageId = $this->buildBaseImageName($params); + try { + $r = $this->pullImageInternal($dockerUrl, $exApp, $startPercent, $maxPercent, $imageId); + } catch (GuzzleException $e) { + $r = sprintf('Failed to pull image, GuzzleException occur: %s', $e->getMessage()); + } + return $r; + } + + /** + * @throws GuzzleException + */ + public function pullImageInternal( + string $dockerUrl, ExApp $exApp, int $startPercent, int $maxPercent, string $imageId + ): string { + # docs: https://github.com/docker/compose/blob/main/pkg/compose/pull.go + $layerInProgress = ['preparing', 'waiting', 'pulling fs layer', 'download', 'extracting', 'verifying checksum']; + $layerFinished = ['already exists', 'pull complete']; + $disableProgressTracking = false; + $url = $this->buildApiUrl($dockerUrl, sprintf('images/create?fromImage=%s', urlencode($imageId))); + $this->logger->info(sprintf('Pulling ExApp Image: %s', $imageId)); + if ($this->useSocket) { + $response = $this->guzzleClient->post($url); + } else { + $response = $this->guzzleClient->post($url, ['stream' => true]); + } + if ($response->getStatusCode() !== 200) { + return sprintf('Pulling ExApp Image: %s return status code: %d', $imageId, $response->getStatusCode()); + } + if ($this->useSocket) { + return ''; + } + $lastPercent = $startPercent; + $layers = []; + $buffer = ''; + $responseBody = $response->getBody(); + while (!$responseBody->eof()) { + $buffer .= $responseBody->read(1024); + try { + while (($newlinePos = strpos($buffer, "\n")) !== false) { + $line = substr($buffer, 0, $newlinePos); + $buffer = substr($buffer, $newlinePos + 1); + $jsonLine = json_decode(trim($line)); + if ($jsonLine) { + if (isset($jsonLine->id) && isset($jsonLine->status)) { + $layerId = $jsonLine->id; + $status = strtolower($jsonLine->status); + foreach ($layerInProgress as $substring) { + if (str_contains($status, $substring)) { + $layers[$layerId] = false; + break; } - foreach ($layerFinished as $substring) { - if (str_contains($status, $substring)) { - $layers[$layerId] = true; - break; - } + } + foreach ($layerFinished as $substring) { + if (str_contains($status, $substring)) { + $layers[$layerId] = true; + break; } } - } else { - $this->logger->warning( - sprintf("Progress tracking of image pulling(%s) disabled, error: %d, data: %s", $exApp->getAppid(), json_last_error(), $line) - ); - $disableProgressTracking = true; } + } else { + $this->logger->warning( + sprintf("Progress tracking of image pulling(%s) disabled, error: %d, data: %s", $exApp->getAppid(), json_last_error(), $line) + ); + $disableProgressTracking = true; } - } catch (Exception $e) { - $this->logger->warning( - sprintf("Progress tracking of image pulling(%s) disabled, exception: %s", $exApp->getAppid(), $e->getMessage()), ['exception' => $e] - ); - $disableProgressTracking = true; } - if (!$disableProgressTracking) { - $completedLayers = count(array_filter($layers)); - $totalLayers = count($layers); - $newLastPercent = intval($totalLayers > 0 ? ($completedLayers / $totalLayers) * ($maxPercent - $startPercent) : 0); - if ($lastPercent != $newLastPercent) { - $this->exAppService->setAppDeployProgress($exApp, $newLastPercent); - $lastPercent = $newLastPercent; - } + } catch (Exception $e) { + $this->logger->warning( + sprintf("Progress tracking of image pulling(%s) disabled, exception: %s", $exApp->getAppid(), $e->getMessage()), ['exception' => $e] + ); + $disableProgressTracking = true; + } + if (!$disableProgressTracking) { + $completedLayers = count(array_filter($layers)); + $totalLayers = count($layers); + $newLastPercent = intval($totalLayers > 0 ? ($completedLayers / $totalLayers) * ($maxPercent - $startPercent) : 0); + if ($lastPercent != $newLastPercent) { + $this->exAppService->setAppDeployProgress($exApp, $newLastPercent); + $lastPercent = $newLastPercent; } } - return ''; - } catch (GuzzleException $e) { - $this->logger->error('Failed to pull image', ['exception' => $e]); - error_log($e->getMessage()); - return 'Failed to pull image, GuzzleException occur.'; } + return ''; } public function inspectContainer(string $dockerUrl, string $containerId): array { From 02261ac8decacbc3f0fd9b2f2a01cbfe0cf8a859 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Wed, 17 Apr 2024 13:45:59 +0300 Subject: [PATCH 3/4] added "info" logging Signed-off-by: Alexander Piskun --- lib/DeployActions/DockerActions.php | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/DeployActions/DockerActions.php b/lib/DeployActions/DockerActions.php index de299b75..c5b25fa2 100644 --- a/lib/DeployActions/DockerActions.php +++ b/lib/DeployActions/DockerActions.php @@ -223,6 +223,7 @@ public function pullImage( if (($r === '') || ($imageId === $this->buildBaseImageName($params))) { return $r; } + $this->logger->info(sprintf('Failed to pull "extended" image for %s: %s', $imageId, $r)); $this->logger->info(sprintf('Pulling "base" image: %s', $imageId)); $imageId = $this->buildBaseImageName($params); try { From a671198c280fc0dddff8285eba97e712b202d373 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Wed, 17 Apr 2024 13:53:28 +0300 Subject: [PATCH 4/4] removed duplicated log record Signed-off-by: Alexander Piskun --- lib/DeployActions/DockerActions.php | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/DeployActions/DockerActions.php b/lib/DeployActions/DockerActions.php index c5b25fa2..7a17ccde 100644 --- a/lib/DeployActions/DockerActions.php +++ b/lib/DeployActions/DockerActions.php @@ -245,7 +245,6 @@ public function pullImageInternal( $layerFinished = ['already exists', 'pull complete']; $disableProgressTracking = false; $url = $this->buildApiUrl($dockerUrl, sprintf('images/create?fromImage=%s', urlencode($imageId))); - $this->logger->info(sprintf('Pulling ExApp Image: %s', $imageId)); if ($this->useSocket) { $response = $this->guzzleClient->post($url); } else {