diff --git a/VERSION b/VERSION index 4a76bfdc14..133872a354 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.32.5 \ No newline at end of file +1.33.0-B1 \ No newline at end of file diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthService.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthService.groovy index 9e1c01082f..5e689ebb16 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthService.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthService.groovy @@ -58,6 +58,16 @@ interface RegistryAuthService { */ String getAuthorization(String image, RegistryAuth auth, RegistryCredentials creds) throws RegistryUnauthorizedAccessException + /** + * Get the authorization header for push operations (pull,push scope). + * + * @param image The image name for which the authorisation is needed + * @param auth The {@link RegistryAuth} information modelling the target registry + * @param creds The user credentials + * @return The authorization header including the 'Basic' or 'Bearer' prefix + */ + String getAuthorizationForPush(String image, RegistryAuth auth, RegistryCredentials creds) throws RegistryUnauthorizedAccessException + /** * Invalidate a cached authorization token * diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy index 89a6261799..756602230a 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy @@ -110,6 +110,15 @@ class RegistryAuthServiceImpl implements RegistryAuthService { private LoadingCache cacheTokens + private CacheLoader pushLoader = new CacheLoader() { + @Override + String load(CacheKey key) throws Exception { + return getPushToken(key) + } + } + + private LoadingCache cachePushTokens + @Inject private RegistryLookupService lookupService @@ -124,6 +133,13 @@ class RegistryAuthServiceImpl implements RegistryAuthService { .expireAfterAccess(_1_HOUR.toMillis(), TimeUnit.MILLISECONDS) .executor(ioExecutor) .build(loader) + + cachePushTokens = Caffeine + .newBuilder() + .maximumSize(1_000) + .expireAfterAccess(_1_HOUR.toMillis(), TimeUnit.MILLISECONDS) + .executor(ioExecutor) + .build(pushLoader) } /** @@ -215,23 +231,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService { */ @Override String getAuthorization(String image, RegistryAuth auth, RegistryCredentials creds) throws RegistryUnauthorizedAccessException { - if( !auth ) - throw new RegistryUnauthorizedAccessException("Missing authentication credentials") - - if( !auth.type ) - return null - - if( auth.type == RegistryAuth.Type.Bearer ) { - final token = getAuthToken(image, auth, creds) - return "Bearer $token" - } - - if( auth.type == RegistryAuth.Type.Basic ) { - final String basic = creds ? "$creds.username:$creds.password".bytes.encodeBase64() : null - return basic ? "Basic $basic" : null - } - - throw new RegistryUnauthorizedAccessException("Unknown authentication type: $auth.type") + return getAuthorization0(image, auth, creds, false) } /** @@ -242,9 +242,12 @@ class RegistryAuthServiceImpl implements RegistryAuthService { * @return The resulting bearer token to authorise a pull request */ protected String getToken0(CacheKey key) { + return fetchToken(buildLoginUrl(key.auth.realm, key.image, key.auth.service), key.creds) + } + + private String fetchToken(String login, RegistryCredentials creds) { final httpClient = HttpClientFactory.followRedirectsHttpClient() - final login = buildLoginUrl(key.auth.realm, key.image, key.auth.service) - final req = makeRequest(login, key.creds) + final req = makeRequest(login, creds) log.trace "Token request=$req" // retry strategy @@ -272,18 +275,30 @@ class RegistryAuthServiceImpl implements RegistryAuthService { throw new RegistryForwardException("Unexpected response acquiring token for '$login' [${response.statusCode()}]", response) } - String buildLoginUrl(URI realm, String image, String service){ - String result = "${realm}?scope=repository:${image}:pull" - if(service) { + String buildLoginUrl(URI realm, String image, String service, String scope='pull') { + String result = "${realm}?scope=repository:${image}:${scope}" + if( service ) { result += "&service=$service" } return result } protected String getAuthToken(String image, RegistryAuth auth, RegistryCredentials creds) { + return getCachedToken(cacheTokens, image, auth, creds) + } + + protected String getPushToken(CacheKey key) { + return fetchToken(buildLoginUrl(key.auth.realm, key.image, key.auth.service, 'pull,push'), key.creds) + } + + protected String getPushAuthToken(String image, RegistryAuth auth, RegistryCredentials creds) { + return getCachedToken(cachePushTokens, image, auth, creds) + } + + private String getCachedToken(LoadingCache cache, String image, RegistryAuth auth, RegistryCredentials creds) { final key = new CacheKey(image, auth, creds) try { - return cacheTokens.get(key) + return cache.get(key) } catch (CompletionException e) { // this catches the exception thrown in the cache loader lookup @@ -292,6 +307,31 @@ class RegistryAuthServiceImpl implements RegistryAuthService { } } + @Override + String getAuthorizationForPush(String image, RegistryAuth auth, RegistryCredentials creds) throws RegistryUnauthorizedAccessException { + return getAuthorization0(image, auth, creds, true) + } + + private String getAuthorization0(String image, RegistryAuth auth, RegistryCredentials creds, boolean push) { + if( !auth ) + throw new RegistryUnauthorizedAccessException("Missing authentication credentials") + + if( !auth.type ) + return null + + if( auth.type == RegistryAuth.Type.Bearer ) { + final token = push ? getPushAuthToken(image, auth, creds) : getAuthToken(image, auth, creds) + return "Bearer $token" + } + + if( auth.type == RegistryAuth.Type.Basic ) { + final String basic = creds ? "$creds.username:$creds.password".bytes.encodeBase64() : null + return basic ? "Basic $basic" : null + } + + throw new RegistryUnauthorizedAccessException("Unknown authentication type: $auth.type") + } + /** * Invalidate a cached authorization token * @@ -302,6 +342,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService { void invalidateAuthorization(String image, RegistryAuth auth, RegistryCredentials creds) { final key = new CacheKey(image, auth, creds) cacheTokens.invalidate(key) + cachePushTokens.invalidate(key) tokenStore.remove(getStableKey(key)) } diff --git a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy index 80c12b2ae0..4be2cb112f 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy @@ -152,6 +152,9 @@ class BuildConfig { @Value('${wave.build.logs.maxLength:100000}') long maxLength + @Value('${wave.build.skip-cache:false}') + boolean skipCache + @PostConstruct private void init() { log.info("Builder config: " + diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index 9958a79c5f..70a8e76cef 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -63,6 +63,7 @@ import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildTrack import io.seqera.wave.service.builder.ContainerBuildService import io.seqera.wave.service.builder.FreezeService +import io.seqera.wave.service.builder.MultiPlatformBuildService import io.seqera.wave.service.inclusion.ContainerInclusionService import io.seqera.wave.service.inspect.ContainerInspectService import io.seqera.wave.service.mirror.ContainerMirrorService @@ -76,6 +77,7 @@ import io.seqera.wave.service.request.ContainerRequestService import io.seqera.wave.service.request.ContainerStatusService import io.seqera.wave.service.request.TokenData import io.seqera.wave.service.scan.ContainerScanService +import io.seqera.wave.core.ChildRefs import io.seqera.wave.service.validation.ValidationService import io.seqera.wave.service.validation.ValidationServiceImpl import io.seqera.wave.tower.PlatformId @@ -92,6 +94,7 @@ import static io.seqera.wave.util.ContainerHelper.condaFileFromRequest import static io.seqera.wave.util.ContainerHelper.containerFileFromRequest import static io.seqera.wave.util.ContainerHelper.decodeBase64OrFail import static io.seqera.wave.util.ContainerHelper.makeContainerId +import static io.seqera.wave.util.ContainerHelper.makeMultiPlatformContainerId import static io.seqera.wave.util.ContainerHelper.makeResponseV1 import static io.seqera.wave.util.ContainerHelper.makeResponseV2 import static io.seqera.wave.util.ContainerHelper.makeTargetImage @@ -178,6 +181,10 @@ class ContainerController { @Nullable private ContainerScanService scanService + @Inject + @Nullable + private MultiPlatformBuildService multiPlatformBuildService + @PostConstruct private void init() { log.info "Wave server url: $serverUrl; allowAnonymous: $allowAnonymous; tower-endpoint-url: $towerEndpointUrl; default-build-repo: ${buildConfig?.defaultBuildRepository}; default-cache-repo: ${buildConfig?.defaultCacheRepository}; default-public-repo: ${buildConfig?.defaultPublicRepository}" @@ -258,10 +265,23 @@ class ContainerController { final generated = containerFileFromRequest(req) req = req.copyWith(containerFile: generated.bytes.encodeBase64().toString()) } - // make sure container platform is defined + // make sure container platform is defined if( !req.containerPlatform ) req.containerPlatform = ContainerPlatform.DEFAULT.toString() + // multi-platform validation + final parsedPlatform = ContainerPlatform.of(req.containerPlatform) + if( parsedPlatform.isMultiArch() ) { + if( parsedPlatform != ContainerPlatform.MULTI_PLATFORM ) + throw new BadRequestException("Only linux/amd64,linux/arm64 multi-platform combination is currently supported") + if( !req.containerFile && !req.packages ) + throw new BadRequestException("Multi-platform builds require either 'containerFile' or 'packages' attribute") + if( req.formatSingularity() ) + throw new BadRequestException("Multi-platform builds are not supported for Singularity format") + if( !multiPlatformBuildService ) + throw new UnsupportedBuildServiceException() + } + final ip = addressResolver.resolve(httpRequest) // check the rate limit before continuing if( rateLimiterService ) @@ -388,6 +408,21 @@ class ContainerController { ) } + protected ChildRefs makeChildScanIds(BuildRequest build, SubmitContainerTokenRequest req) { + if( !scanService || !build.platform.isMultiArch() ) + return null + final multiPlatform = build.platform + final scanMode = req.scanMode!=null ? req.scanMode : ScanMode.async + final scanIdByPlatform = new LinkedHashMap() + for( ContainerPlatform.Platform p : multiPlatform.platforms ) { + final platform = p.toString() + final id = scanService.getScanId("${build.targetImage}#${platform}", null, scanMode, req.format) + if( id ) + scanIdByPlatform.put(id, platform) + } + return ChildRefs.of(scanIdByPlatform) + } + protected BuildTrack checkBuild(BuildRequest build, boolean dryRun) { final digest = registryProxyService.getImageDigest(build) // check for dry-run execution @@ -408,6 +443,37 @@ class ContainerController { } } + protected BuildTrack checkMultiPlatformBuild(BuildRequest templateBuild, SubmitContainerTokenRequest req, PlatformId identity, boolean dryRun) { + final containerSpec = templateBuild.containerFile + final condaContent = templateBuild.condaFile + final buildRepository = ContainerCoordinates.parse(templateBuild.targetImage).repository + + // compute multi-platform container ID and target image + final containerId = makeMultiPlatformContainerId(containerSpec, condaContent, buildRepository, req.buildContext, req.freeze ? req.containerConfig : null) + final targetImage = makeTargetImage(templateBuild.format, buildRepository, containerId, condaContent, req.nameStrategy) + + // check for dry-run execution + if( dryRun ) { + log.debug "== Dry-run multi-platform build request for $targetImage" + final dryId = containerId + BuildRequest.SEP + '0' + final digest = registryProxyService.getImageDigest(targetImage, identity) + return new BuildTrack(dryId, targetImage, digest!=null, true) + } + + // check if the multi-platform image already exists + if( !buildConfig.skipCache ) { + final digest = registryProxyService.getImageDigest(targetImage, identity) + if( digest ) { + log.debug "== Found cached multi-platform build for $targetImage" + final cache = persistenceService.loadBuildSucceed(targetImage, digest) + return new BuildTrack(cache?.buildId, targetImage, true, true) + } + } + + // delegate to multi-platform build service + return multiPlatformBuildService.buildMultiPlatformImage(templateBuild, containerId, targetImage, identity) + } + protected String getContainerDigest(String containerImage, PlatformId identity) { containerImage ? registryProxyService.getImageDigest(containerImage, identity) @@ -446,8 +512,26 @@ class ContainerController { String buildId boolean buildNew String scanId + ChildRefs scanChildIds = null Boolean succeeded - if( req.containerFile ) { + if( req.containerFile && ContainerPlatform.of(req.containerPlatform).isMultiArch() ) { + if( !buildService ) throw new UnsupportedBuildServiceException() + final build0 = makeBuildRequest(req, identity, ip) + // create per-platform scan IDs for multi-arch builds + final childScans = makeChildScanIds(build0, req) + final build = childScans ? build0.withChildScanIds(childScans) : build0 + final track = checkMultiPlatformBuild(build, req, identity, req.dryRun) + targetImage = track.targetImage + targetContent = build.containerFile + condaContent = build.condaFile + buildId = track.id + buildNew = !track.cached + scanId = build.scanId + scanChildIds = build.scanChildIds + succeeded = track.succeeded + type = ContainerRequest.Type.Build + } + else if( req.containerFile ) { if( !buildService ) throw new UnsupportedBuildServiceException() final build = makeBuildRequest(req, identity, ip) final track = checkBuild(build, req.dryRun) @@ -501,6 +585,7 @@ class ContainerController { buildNew, req.freeze, scanId, + scanChildIds, req.scanMode, req.scanLevels, req.dryRun, diff --git a/src/main/groovy/io/seqera/wave/controller/InspectController.groovy b/src/main/groovy/io/seqera/wave/controller/InspectController.groovy index 506b0c6c98..e99957f863 100644 --- a/src/main/groovy/io/seqera/wave/controller/InspectController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/InspectController.groovy @@ -32,6 +32,7 @@ import io.micronaut.scheduling.TaskExecutors import io.micronaut.scheduling.annotation.ExecuteOn import io.seqera.wave.api.ContainerInspectRequest import io.seqera.wave.api.ContainerInspectResponse +import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.exception.BadRequestException import io.seqera.wave.service.UserService import io.seqera.wave.service.inspect.ContainerInspectService @@ -74,6 +75,9 @@ class InspectController { if( !req.containerImage ) throw new BadRequestException("Missing 'containerImage' attribute") + // multi-platform values are not allowed for inspect requests + ContainerPlatform.validateSinglePlatform(platform) + // this is needed for backward compatibility with old clients if( !req.towerEndpoint ) { req.towerEndpoint = towerEndpointUrl diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy index 6317f9088b..5ea4bf581e 100644 --- a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy @@ -56,6 +56,7 @@ import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveScanRecord import io.seqera.wave.service.scan.ContainerScanService import io.seqera.wave.service.scan.ScanEntry +import io.seqera.wave.core.ChildRefs import io.seqera.wave.service.scan.ScanType import io.seqera.wave.service.scan.ScanVulnerability import io.seqera.wave.util.JacksonHelper @@ -137,8 +138,7 @@ class ViewController { binding.mirror_digest = result.digest ?: '-' binding.mirror_user = result.userName ?: '-' binding.put('server_url', serverUrl) - binding.scan_url = result.scanId && result.succeeded() ? "$serverUrl/view/scans/${result.scanId}" : null - binding.scan_id = result.scanId + ChildRefs.populateScanBinding(binding, result.scanId, null, result.succeeded(), serverUrl) return binding } @@ -254,8 +254,8 @@ class ViewController { binding.build_condafile = result.condaFile binding.build_digest = result.digest ?: '-' binding.put('server_url', serverUrl) - binding.scan_url = result.scanId && result.succeeded() ? "$serverUrl/view/scans/${result.scanId}" : null - binding.scan_id = result.scanId + ChildRefs.populateScanBinding(binding, result.scanId, result.scanChildIds, result.succeeded(), serverUrl) + ChildRefs.populateBuildBinding(binding, result.buildChildIds, serverUrl) // inspect uri binding.inspect_url = result.succeeded() ? "$serverUrl/view/inspect?image=${result.targetImage}&platform=${result.platform}" : null // configure build logs when available @@ -314,8 +314,7 @@ class ViewController { binding.build_url = data.buildId ? "$serverUrl/view/builds/${data.buildId}" : null binding.fusion_version = data.fusionVersion ?: '-' - binding.scan_id = data.scanId - binding.scan_url = data.scanId ? "$serverUrl/view/scans/${data.scanId}" : null + ChildRefs.populateScanBinding(binding, data.scanId, data.scanChildIds, true, serverUrl) binding.mirror_id = data.mirror ? data.buildId : null binding.mirror_url = data.mirror ? "$serverUrl/view/mirrors/${data.buildId}" : null diff --git a/src/main/groovy/io/seqera/wave/core/ChildRefs.groovy b/src/main/groovy/io/seqera/wave/core/ChildRefs.groovy new file mode 100644 index 0000000000..eb3c065038 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/core/ChildRefs.groovy @@ -0,0 +1,143 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.core + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode + +/** + * A list of per-platform child IDs (builds or scans). + * + * Serialises to/from JSON via Moshi as: {"refs":[{"id":"sc-abc_1","value":"linux/amd64"}, ...]} + * + * @author Paolo Di Tommaso + */ +@CompileStatic +@EqualsAndHashCode(includes = 'refs') +class ChildRefs implements Iterable { + + @CompileStatic + @EqualsAndHashCode + static class Ref { + String id + String value + + Ref() {} + + Ref(String id, String value) { + this.id = id + this.value = value + } + + @Override + String toString() { + return "${id}:${value}" + } + } + + List refs + + ChildRefs() { + this.refs = [] + } + + ChildRefs(List refs) { + this.refs = refs != null ? new ArrayList<>(refs) : [] + } + + /** + * Create from a map of id-by-platform. + * @param idByPlatform Map where keys are IDs and values are platform strings + * @return A new ChildRefs, or null if the map is null/empty + */ + static ChildRefs of(Map idByPlatform) { + if( !idByPlatform ) + return null + final result = new ChildRefs() + for( Map.Entry it : idByPlatform.entrySet() ) { + result.add(new Ref(it.key, it.value)) + } + return result + } + + // -- delegate methods -- + + int size() { refs.size() } + + boolean isEmpty() { refs.isEmpty() } + + Ref getAt(int index) { refs[index] } + + void add(Ref entry) { refs.add(entry) } + + boolean asBoolean() { refs != null && !refs.isEmpty() } + + @Override + Iterator iterator() { refs.iterator() } + + def List collect(Closure closure) { refs.collect(closure) } + + /** + * Get the first/primary ID + */ + String primary() { + return refs ? refs[0].id : null + } + + /** + * Get all IDs + */ + List allIds() { + return refs.collect { it.id } + } + + @Override + String toString() { + return refs.collect { it.toString() }.toString() + } + + // -- template binding helpers -- + + static void populateScanBinding(Map binding, String scanId, ChildRefs scanChildIds, boolean succeeded, String serverUrl) { + if( scanChildIds && succeeded ) { + binding.scan_entries = scanChildIds.collect { Ref entry -> + [scan_id: entry.id, scan_platform: entry.value, scan_url: "${serverUrl}/view/scans/${entry.id}"] as Map + } + binding.scan_url = null + binding.scan_id = scanChildIds.primary() + } + else { + binding.scan_entries = null + binding.scan_url = scanId && succeeded ? "${serverUrl}/view/scans/${scanId}" : null + binding.scan_id = scanId + } + } + + static void populateBuildBinding(Map binding, ChildRefs buildChildIds, String serverUrl) { + if( buildChildIds ) { + binding.build_entries = buildChildIds.collect { Ref entry -> + [build_id: entry.id, build_platform: entry.value, build_url: "${serverUrl}/view/builds/${entry.id}"] as Map + } + } + else { + binding.build_entries = null + } + } + +} diff --git a/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy b/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy index 3b73782add..22b9225df6 100644 --- a/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy +++ b/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy @@ -39,6 +39,7 @@ import io.seqera.wave.model.ContainerOrIndexSpec import io.seqera.wave.proxy.ProxyClient import io.seqera.wave.storage.Storage import io.seqera.wave.util.JacksonHelper +import io.seqera.wave.service.builder.MultiPlatformBuildService import io.seqera.wave.util.RegHelper import static io.seqera.wave.model.ContentType.DOCKER_IMAGE_CONFIG_V1 import static io.seqera.wave.model.ContentType.DOCKER_IMAGE_INDEX_V2 @@ -66,6 +67,16 @@ class ContainerAugmenter { final Boolean oci final ManifestSpec manifestSpec } + + /** + * Holds the digest and size of an augmented manifest, used to update + * entries in a manifest index after per-platform augmentation. + */ + @Canonical + static class AugmentedManifest { + final String digest + final int size + } private ProxyClient client private ContainerConfig containerConfig @@ -110,8 +121,14 @@ class ContainerAugmenter { this.platform = route.request?.platform ?: ContainerPlatform.DEFAULT // note: do not propagate container config when "freeze" mode is enabled, because it has been already // applied during the container build phase, and therefore it should be ignored by the augmenter - if( route.request?.containerConfig && !route.request.freeze ) - this.containerConfig = route.request.containerConfig + if( route.request?.containerConfig && !route.request.freeze ) { + log.debug "Augmenter resolve: platform=${this.platform}; layers=${route.request.containerConfig?.layers?.collect { it.location }}" + // for single-arch: filters fusion layers to keep only the matching arch + // for multi-arch: this is a no-op (returns all layers); per-platform filtering + // happens later in augmentManifest() when the actual image architecture is known + this.containerConfig = MultiPlatformBuildService.filterLayersForPlatform(route.request.containerConfig, this.platform) + log.debug "Augmenter resolve: filtered layers=${this.containerConfig?.layers?.collect { it.location }}" + } return resolve(route.image, route.reference, headers) } @@ -185,46 +202,211 @@ class ContainerAugmenter { return new ContainerDigestPair(digest, v1Digest) } - final manifestResult = findImageManifestAndDigest(manifestsList, imageName, tag, headers) + // Parse the manifest body to inspect its mediaType field. + // This determines whether we're dealing with a manifest index (multi-platform list) + // or a single platform manifest. + final parsedManifest = new JsonSlurper().parseText(manifestsList) as Map + final media = parsedManifest.mediaType as String + + // Manifest index (Docker v2 or OCI): + // Contains a list of platform-specific manifests. Delegate to resolveImageIndex + // which iterates over matching platform entries, augments each one separately + // with the correct arch-specific fusion layers, and updates the index. + // This handles both single-arch (one match) and multi-arch (multiple matches). + if( media==DOCKER_IMAGE_INDEX_V2 || media==OCI_IMAGE_INDEX_V1 ) { + return resolveImageIndex(imageName, tag, headers, manifestsList, digest, parsedManifest, media) + } - // fetch the image config - final resp5 = client.getString("/v2/$imageName/blobs/$manifestResult.configDigest", headers) - checkResponseCode(resp5, client.route, true) - final imageConfig = resp5.body() + // Single platform manifest (Docker v2 or OCI): + // No index wrapping — augment the manifest directly with fusion layers. + if( media==DOCKER_MANIFEST_V2_TYPE || media==OCI_IMAGE_MANIFEST_V1 ) { + final ref = ManifestSpec.of(parsedManifest) + // targetDigest is null because there's no parent index entry to update + final manifestInfo = new ManifestInfo(manifestsList, ref.config.digest, null, media==OCI_IMAGE_MANIFEST_V1, ref) + final result = augmentManifest(imageName, tag, headers, manifestInfo) + return new ContainerDigestPair(digest, result.digest) + } + + throw new IllegalArgumentException("Unexpected media type for request '$imageName:$tag' - offending value: $media") + } + + /** + * Resolve an image index (manifest list) by augmenting each matching platform manifest. + * + * An image index is a JSON document listing platform-specific manifests (e.g. linux/amd64, + * linux/arm64). This method iterates over those entries, finds the ones matching our target + * platform(s), and augments each one separately with the correct arch-specific fusion layers. + * + * For single-arch requests (e.g. platform=linux/amd64): only one entry matches, so we + * augment one manifest and update one index entry. + * + * For multi-arch requests (e.g. platform=linux/amd64,linux/arm64): multiple entries match, + * each gets augmented with its own filtered fusion layers (amd64 gets fusion-amd64.tar.gz, + * arm64 gets fusion-arm64.tar.gz), and all index entries are updated. + * + * @param imageName the repository name (e.g. "library/ubuntu") + * @param tag the image tag or reference + * @param headers HTTP headers for upstream registry requests + * @param manifestsList the raw JSON string of the manifest index + * @param originalDigest the original digest of the unmodified index + * @param indexJson the already-parsed index JSON (avoids double parsing) + * @param indexMedia the mediaType of the index (OCI or Docker) + * @return a pair of (original digest, new augmented index digest) + */ + protected ContainerDigestPair resolveImageIndex(String imageName, String tag, Map> headers, String manifestsList, String originalDigest, Map indexJson, String indexMedia) { + final oci = indexMedia == OCI_IMAGE_INDEX_V1 + // each entry in this list represents a platform-specific manifest (os/arch/variant) + final manifests = indexJson.manifests as List + // save the original container config (with ALL fusion layers for all platforms) + // because augmentManifest mutates this.containerConfig via filterLayersForPlatform, + // and we need to reset it before processing each platform + final originalConfig = this.containerConfig + // collect old digest → (new digest, new size) for each augmented platform + final digestUpdates = new LinkedHashMap() + + try { + for( Map indexEntry : manifests ) { + // each index entry has a 'platform' object with os/architecture/variant fields + final entryPlatform = indexEntry.platform as Map + if( !entryPlatform ) + continue + + // verify the entry's mediaType matches what we expect for this index type: + // OCI indexes reference OCI manifests, Docker indexes reference Docker v2 manifests + final entryMedia = indexEntry.mediaType as String + final expectedMedia = oci ? OCI_IMAGE_MANIFEST_V1 : DOCKER_MANIFEST_V2_TYPE + if( entryMedia != expectedMedia ) + continue + + // check if this entry's platform (e.g. linux/amd64) matches any of our + // target platforms — for single-arch this matches one, for multi-arch it + // matches each platform in turn + if( !matchesPlatformEntry(entryPlatform) ) + continue + + final targetDigest = indexEntry.digest as String + if( !targetDigest ) + continue + + // fetch the actual platform-specific manifest (not the index — the real manifest + // that contains the layer list and config reference for this architecture) + final resp = client.getString("/v2/$imageName/manifests/$targetDigest", headers) + checkResponseCode(resp, client.route, false) + final platformManifest = resp.body() as String + // parse the manifest to extract config digest, layer info, and mediaType + final manifestInfo = parseManifest(platformManifest, targetDigest) + if( !manifestInfo ) + continue + + // reset to the original unfiltered container config before each platform, + // because augmentManifest will filter it down to only the layers matching + // this specific architecture (e.g. keep fusion-amd64, discard fusion-arm64) + this.containerConfig = originalConfig + // augment this platform's manifest: fetch config, filter layers, inject layers + final result = augmentManifest(imageName, tag, headers, manifestInfo) + // record the mapping from old digest to new digest+size for the index update + digestUpdates.put(targetDigest, result) + } + } + finally { + // always restore the original container config, even if an exception occurs, + // to avoid leaving the augmenter in a partially-filtered state + this.containerConfig = originalConfig + } + + if( digestUpdates.isEmpty() ) { + throw new BadRequestException("Cannot find matching platform '${platform}' in the image index for '$imageName:$tag'") + } + + // rewrite the image index: replace old digests with new augmented digests, + // recompute the index's own digest, and store it in the cache + final newListDigest = updateImageIndex(imageName, manifestsList, digestUpdates, oci) + if( log.isTraceEnabled() ) + log.trace "resolveImageIndex: new index digest: $newListDigest (updated ${digestUpdates.size()} platform entries)" + // return: original unmodified index digest + new augmented index digest + return new ContainerDigestPair(originalDigest, newListDigest) + } + + /** + * Check if a manifest index entry's platform matches any of our target platforms. + * For single-arch (e.g. linux/amd64), the platforms list has one entry. + * For multi-arch (e.g. linux/amd64,linux/arm64), it has multiple entries. + * Delegates to Platform.matches() which handles arch aliases (x86_64→amd64) + * and variant normalization (arm64/v8→arm64). + */ + protected boolean matchesPlatformEntry(Map entryPlatform) { + return platform.platforms.any { it.matches(entryPlatform) } + } + + /** + * Augment a single platform manifest by injecting fusion layers and config changes. + * + * This is the core augmentation logic shared by both single-manifest and index-based + * resolution. It performs three steps: + * 1. Fetch the image config blob to determine the actual architecture + * 2. Filter fusion layers to keep only those matching the resolved architecture + * (e.g. for amd64: keep fusion-amd64.tar.gz, discard fusion-arm64.tar.gz) + * 3. Inject the filtered layers into both the image config and manifest + * + * NOTE: This method mutates this.containerConfig via filterLayersForPlatform. + * When called from resolveImageIndex, the caller must reset this.containerConfig + * before each invocation to ensure correct filtering per platform. + * + * @return a Tuple2 of (newManifestDigest, newManifestSize) for updating the parent index + */ + protected AugmentedManifest augmentManifest(String imageName, String tag, Map> headers, ManifestInfo manifestInfo) { + // fetch the image config blob — this JSON contains the architecture field, + // rootfs layer digests, and container config (env, entrypoint, etc.) + final resp = client.getString("/v2/$imageName/blobs/$manifestInfo.configDigest", headers) + checkResponseCode(resp, client.route, true) + final imageConfig = resp.body() as String if( log.isTraceEnabled() ) - log.trace "Resolve (5): image $imageName:$tag => image config=\n${JsonOutput.prettyPrint(imageConfig)}" + log.trace "Augment: image $imageName:$tag => image config=\n${JsonOutput.prettyPrint(imageConfig)}" + + // read the architecture from the image config to determine which fusion layers to keep. + // this is the authoritative source of the image's architecture — more reliable than + // the platform field in the manifest index, which may be missing or incorrect. + final configJson = new JsonSlurper().parseText(imageConfig) as Map + final resolvedArch = configJson.architecture as String + if( resolvedArch ) { + // filter this.containerConfig.layers to only include: + // - non-fusion layers (always kept, e.g. the wave launcher data layer) + // - fusion layers matching this architecture (e.g. fusion-amd64.tar.gz for amd64) + // fusion layers for other architectures are discarded + final resolvedPlatform = ContainerPlatform.of("linux/${resolvedArch}") + this.containerConfig = MultiPlatformBuildService.filterLayersForPlatform(this.containerConfig, resolvedPlatform) + log.debug "Augment: filtered layers for arch=${resolvedArch}; remaining=${this.containerConfig?.layers?.size()}" + } - // update the image config adding the new layer - final newConfigResult = updateImageConfig(imageName, imageConfig, manifestResult.oci) + // update the image config: add layer tar digests to rootfs.diff_ids, + // apply container config changes (env, entrypoint, cmd, workingDir), + // then store the new config blob and return its digest + final newConfigResult = updateImageConfig(imageName, imageConfig, manifestInfo.oci) final newConfigDigest = newConfigResult[0] final newConfigJson = newConfigResult[1] if( log.isTraceEnabled() ) - log.trace "Resolve (6) ==> new config digest: $newConfigDigest => new config=\n${JsonOutput.prettyPrint(newConfigJson)} " + log.trace "Augment: new config digest: $newConfigDigest" - // update the image manifest adding a new layer - // returns the updated image manifest digest - final newManifestResult = updateImageManifest(imageName, manifestResult.imageManifest, newConfigDigest, newConfigJson.size(), manifestResult.oci) - final newManifestDigest = newManifestResult.v1 - final newManifestSize = newManifestResult.v2 - if( log.isTraceEnabled() ) - log.trace "Resolve (7) ==> new image digest: $newManifestDigest" + // update the image manifest: append new layer blobs to the layers array, + // update the config reference to point to the new config digest, + // then store the new manifest and return its digest + size + final newManifestResult = updateImageManifest(imageName, manifestInfo.imageManifest, newConfigDigest, newConfigJson.size(), manifestInfo.oci) + return new AugmentedManifest(newManifestResult.v1, newManifestResult.v2) + } - if( !manifestResult.targetDigest ) { - return new ContainerDigestPair(digest, newManifestDigest) - } - else { - // update the manifests list with the new digest - // returns the manifests list digest - final newListDigest = updateImageIndex(imageName, manifestsList, manifestResult.targetDigest, newManifestDigest, newManifestSize, manifestResult.oci) - if( log.isTraceEnabled() ) - log.trace "Resolve (8) ==> new list digest: $newListDigest" - return new ContainerDigestPair(digest, newListDigest) - } + protected ManifestInfo parseManifest(String manifest, String targetDigest) { + final json = new JsonSlurper().parseText(manifest) as Map + final media = json.mediaType as String + return parseManifest(media, manifest, json, targetDigest) } protected ManifestInfo parseManifest(String media, String manifest, String targetDigest) { + final json = new JsonSlurper().parseText(manifest) as Map + return parseManifest(media, manifest, json, targetDigest) + } + + protected ManifestInfo parseManifest(String media, String manifest, Map json, String targetDigest) { if( media==DOCKER_MANIFEST_V2_TYPE || media==OCI_IMAGE_MANIFEST_V1 ) { - final json = new JsonSlurper().parseText(manifest) as Map final ref = ManifestSpec.of(json) return new ManifestInfo(manifest, ref.config.digest, targetDigest, media==OCI_IMAGE_MANIFEST_V1, ref) } @@ -264,26 +446,31 @@ class ContainerAugmenter { } - protected String updateImageIndex(String imageName, String manifestsList, String targetDigest, String newDigest, Integer newSize, boolean oci) { + /** + * Rewrite the image index by replacing old platform manifest digests with new augmented ones. + * Handles one or more digest replacements in a single pass, then stores the updated index. + */ + protected String updateImageIndex(String imageName, String manifestsList, Map digestUpdates, boolean oci) { + // parse the original index JSON final json = new JsonSlurper().parseText(manifestsList) as Map final list = json.manifests as List - final entry = list.find( it -> it.digest==targetDigest ) - if( !entry ) - throw new IllegalStateException("Missing manifest entry for digest: $targetDigest") - // update the target entry digest and size - entry.digest = newDigest - entry.size = newSize - // serialize to json again + + // replace each old digest with the new augmented digest and size + for( Map.Entry update : digestUpdates.entrySet() ) { + final entry = list.find( it -> it.digest == update.key ) + if( !entry ) + throw new IllegalStateException("Missing manifest entry for digest: ${update.key}") + entry.digest = update.value.digest + entry.size = update.value.size + } + final updated = JsonOutput.toJson(json) final result = RegHelper.digest(updated) final type = oci ? OCI_IMAGE_INDEX_V1 : DOCKER_IMAGE_INDEX_V2 - // make sure the manifest was updated - if( manifestsList==updated ) - throw new IllegalArgumentException("Unable to find target digest '$targetDigest' into image index") - // store in the cache + if( manifestsList == updated ) + throw new IllegalArgumentException("Unable to update image index - no entries were modified") final target = "$client.registry.name/v2/$imageName/manifests/$result" storage.saveManifest(target, updated, type, result) - // return the updated manifests list digest return result } @@ -420,11 +607,11 @@ class ContainerAugmenter { } protected boolean matchesDockerManifest(Map record) { - return record.mediaType == DOCKER_MANIFEST_V2_TYPE && platform.matches(record.platform as Map) + return record.mediaType == DOCKER_MANIFEST_V2_TYPE && matchesPlatformEntry(record.platform as Map) } protected boolean matchesOciManifest(Map record) { - return record.mediaType == OCI_IMAGE_MANIFEST_V1 && platform.matches(record.platform as Map) + return record.mediaType == OCI_IMAGE_MANIFEST_V1 && matchesPlatformEntry(record.platform as Map) } protected void rewriteHistoryV1( List history ){ diff --git a/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy b/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy index 6b90d16346..a6938f9280 100644 --- a/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy +++ b/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy @@ -18,19 +18,19 @@ package io.seqera.wave.core -import groovy.transform.Canonical +import com.fasterxml.jackson.annotation.JsonCreator +import com.fasterxml.jackson.annotation.JsonValue import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode import io.seqera.wave.exception.BadRequestException /** * Model a container platform * @author Paolo Di Tommaso */ -@Canonical +@EqualsAndHashCode(includes = 'platforms') @CompileStatic class ContainerPlatform { - public static final ContainerPlatform DEFAULT = new ContainerPlatform(DEFAULT_OS, DEFAULT_ARCH) - public static final List ARM64 = ['arm64', 'aarch64'] private static final List V8 = ['8','v8'] public static final List AMD64 = ['amd64', 'x86_64', 'x86-64'] @@ -38,101 +38,183 @@ class ContainerPlatform { public static final String DEFAULT_ARCH = 'amd64' public static final String DEFAULT_OS = 'linux' - final String os - final String arch - final String variant + @EqualsAndHashCode + @CompileStatic + static class Platform implements Serializable { + final String os + final String arch + final String variant + + Platform(String os, String arch, String variant=null) { + this.os = os + this.arch = arch + this.variant = variant + } - String toString() { - def result = os + "/" + arch - if( variant ) - result += "/" + variant - return result - } + static Platform of(String value) { + if( !value ) + throw new BadRequestException("Missing container platform attribute") - boolean matches(Map record) { - return sameOs(record) && sameArch(record) && sameVariant(record) - } + final items = value.tokenize('/') + if( items.size()==1 ) + items.add(0, DEFAULT_OS) - private boolean sameOs(Map record) { - this.os == record.os - } + if( items.size()==2 || items.size()==3 ) { + final os = os0(items[0]) + final arch = arch0(items[1]) + // variant v8 for amd64 is normalised to empty + // see https://github.com/containerd/containerd/blob/v1.4.3/platforms/database.go#L96 + final variant = variant0(arch, items[2]) + return new Platform(os, arch, variant) + } - private boolean sameArch(Map record) { - if( this.arch==record.architecture ) - return true - if( this.arch=='amd64' && record.architecture in AMD64 ) - return true - if( this.arch=='arm64' && record.architecture in ARM64 ) - return true - else - return false - } + throw new BadRequestException("Invalid container platform: $value -- offending value: $value") + } + + /** + * Check if this platform matches a manifest index entry's platform record. + * Handles architecture aliases (x86_64→amd64, aarch64→arm64) and + * variant normalization (arm64/v8 matches arm64 with no variant). + * + * @param record a map with 'os', 'architecture', and optionally 'variant' keys + * as found in a manifest index entry's platform field + */ + boolean matches(Map record) { + return sameOs(record) && sameArch(record) && sameVariant(record) + } - private boolean sameVariant(Map record) { - if( this.variant == record.variant ) - return true - if( this.arch=='arm64' ) { - if( !this.variant && (!record.variant || record.variant in V8)) + private boolean sameOs(Map record) { + this.os == record.os + } + + private boolean sameArch(Map record) { + if( this.arch==record.architecture ) + return true + if( this.arch=='amd64' && record.architecture in AMD64 ) return true - if( this.variant && record.variant==this.variant ) + if( this.arch=='arm64' && record.architecture in ARM64 ) return true + else + return false + } + + private boolean sameVariant(Map record) { + if( this.variant == record.variant ) + return true + if( this.arch=='arm64' ) { + if( !this.variant && (!record.variant || record.variant in V8)) + return true + if( this.variant && record.variant==this.variant ) + return true + } + return false + } + + @Override + String toString() { + def result = os + "/" + arch + if( variant ) + result += "/" + variant + return result + } + + static private String arch0(String value) { + if( !value ) + return DEFAULT_ARCH + if( value !in ALLOWED_ARCH) throw new BadRequestException("Unsupported container platform: $value") + // see + // https://github.com/containerd/containerd/blob/v1.4.3/platforms/database.go#L89 + if( value in AMD64 ) + return AMD64.get(0) + if( value in ARM64 ) + return ARM64.get(0) + return value + } + + static private String os0(String value) { + return value ?: DEFAULT_OS + } + + static private String variant0(String arch, String variant) { + if( arch in ARM64 && variant in V8 ) { + // this also address this issue + // https://github.com/GoogleContainerTools/kaniko/issues/1995#issuecomment-1327706161 + return null + } + if( arch == 'arm' ) { + // arm defaults to variant v7 + // https://github.com/containerd/containerd/blob/v1.4.3/platforms/database.go#L89 + if( (!variant || variant=='7') ) return 'v7' + if( variant in ['5','6','8']) return 'v'+variant + } + return variant } - return false } - static ContainerPlatform parseOrDefault(String value, ContainerPlatform defaultPlatform=DEFAULT) { - return value ? of(value) : defaultPlatform + public static final ContainerPlatform DEFAULT = new ContainerPlatform([new Platform(DEFAULT_OS, DEFAULT_ARCH)]) + + public static final ContainerPlatform MULTI_PLATFORM = new ContainerPlatform([ + new Platform('linux', 'amd64'), + new Platform('linux', 'arm64') + ]) + + final List platforms + + ContainerPlatform(String os, String arch, String variant=null) { + this.platforms = List.of(new Platform(os, arch, variant)) } - static ContainerPlatform of(String value) { - if( !value ) - throw new BadRequestException("Missing container platform attribute") + private ContainerPlatform(List platforms) { + assert platforms.size() >= 1, "Platform list must not be empty" + this.platforms = List.copyOf(platforms) + } - final items= value.tokenize('/') - if( items.size()==1 ) - items.add(0, DEFAULT_OS) - - if( items.size()==2 || items.size()==3 ) { - final os = os0(items[0]) - final arch = arch0(items[1]) - // variant v8 for amd64 is normalised to empty - // see https://github.com/containerd/containerd/blob/v1.4.3/platforms/database.go#L96 - final variant = variant0(arch,items[2]) - return new ContainerPlatform(os, arch, variant) - } + @Deprecated + String getOs() { requireSinglePlatform(); platforms[0].os } + + @Deprecated + String getArch() { requireSinglePlatform(); platforms[0].arch } - throw new BadRequestException("Invalid container platform: $value -- offending value: $value") + @Deprecated + String getVariant() { requireSinglePlatform(); platforms[0].variant } + + private void requireSinglePlatform() { + if( isMultiArch() ) + throw new IllegalStateException("Cannot access single-platform property on multi-arch ContainerPlatform: ${toString()}") } - static private String arch0(String value) { - if( !value ) - return DEFAULT_ARCH - if( value !in ALLOWED_ARCH) throw new BadRequestException("Unsupported container platform: $value") - // see - // https://github.com/containerd/containerd/blob/v1.4.3/platforms/database.go#L89 - if( value in AMD64 ) - return AMD64.get(0) - if( value in ARM64 ) - return ARM64.get(0) - return value + boolean isMultiArch() { + return platforms.size() > 1 + } + + @JsonValue + String toString() { + return platforms.collect { it.toString() }.join(',') } - static private String os0(String value) { - return value ?: DEFAULT_OS + + static ContainerPlatform parseOrDefault(String value, ContainerPlatform defaultPlatform=DEFAULT) { + return value ? of(value) : defaultPlatform } - static private String variant0(String arch, String variant) { - if( arch in ARM64 && variant in V8 ) { - // this also address this issue - // https://github.com/GoogleContainerTools/kaniko/issues/1995#issuecomment-1327706161 - return null - } - if( arch == 'arm' ) { - // arm defaults to variant v7 - // https://github.com/containerd/containerd/blob/v1.4.3/platforms/database.go#L89 - if( (!variant || variant=='7') ) return 'v7' - if( variant in ['5','6','8']) return 'v'+variant - } - return variant + /** + * Validate that the given platform string is a single platform value (not multi-arch). + * Throws {@link BadRequestException} if the value contains comma-separated platforms. + * + * @param value The platform string to validate + */ + static void validateSinglePlatform(String value) { + if( value && value.contains(',') ) + throw new BadRequestException("Container multi-platform architecture not allowed - offending value: $value") + } + + @JsonCreator + static ContainerPlatform of(String value) { + if( !value ) + throw new BadRequestException("Missing container platform attribute") + + return new ContainerPlatform(value + .tokenize(',') + .collect(it-> Platform.of(it.trim()))) } } diff --git a/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy b/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy index 4a7541cccc..d6c2b8895a 100644 --- a/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy +++ b/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy @@ -83,7 +83,7 @@ class PullMetricsRequestsFilter implements HttpServerFilter { final contentType = response.headers.get(HttpHeaders.CONTENT_TYPE) if( contentType && contentType in MANIFEST_TYPES ) { final route = routeHelper.parse(request.path) - final arch = route.request.platform.arch + final arch = route.request.platform?.platforms?.collect(it->it.arch)?.join(',') CompletableFuture.runAsync(() -> metricsService.incrementPullsCounter(route.identity, arch), executor) final version = route.request?.containerConfig?.fusionVersion() if (version) { diff --git a/src/main/groovy/io/seqera/wave/proxy/ProxyClient.groovy b/src/main/groovy/io/seqera/wave/proxy/ProxyClient.groovy index 8027aa3869..c03f042547 100644 --- a/src/main/groovy/io/seqera/wave/proxy/ProxyClient.groovy +++ b/src/main/groovy/io/seqera/wave/proxy/ProxyClient.groovy @@ -61,6 +61,7 @@ class ProxyClient { private ContainerPath route private HttpClientConfig httpConfig private List retryableHttpErrors = HTTP_RETRYABLE_ERRORS + private boolean pushMode ProxyClient(HttpClient httpClient, HttpClientConfig httpConfig) { if( httpClient.followRedirects()!= HttpClient.Redirect.NEVER ) @@ -104,6 +105,11 @@ class ProxyClient { return this } + ProxyClient withPushMode(boolean value) { + this.pushMode = value + return this + } + URI makeUri(String path) { assert path.startsWith('/'), "Request past should start with a slash character — offending path: $path" return URI.create(registry.host.toString() + path) @@ -225,7 +231,7 @@ class ProxyClient { copyHeaders(headers, builder) if( authorize ) { // add authorisation header - final header = loginService.getAuthorization(image, registry.auth, credentials) + final header = getAuthHeader() if( header ) builder.setHeader("Authorization", header) } @@ -279,10 +285,10 @@ class ProxyClient { // https://zetcode.com/java/httpclient/ final builder = HttpRequest.newBuilder(uri) .method("HEAD", HttpRequest.BodyPublishers.noBody()) - // copy headers + // copy headers copyHeaders(headers, builder) // add authorisation header - final header = loginService.getAuthorization(image, registry.auth, credentials) + final header = getAuthHeader() if( header ) builder.setHeader("Authorization", header) // build the request @@ -293,6 +299,59 @@ class ProxyClient { return response } + private String getAuthHeader() { + return pushMode + ? loginService.getAuthorizationForPush(image, registry.auth, credentials) + : loginService.getAuthorization(image, registry.auth, credentials) + } + + HttpResponse put(String path, byte[] body, String contentType, Map> headers=null) { + final uri = makeUri(path) + final retryable = Retryable + .>of(httpConfig) + .retryIf((resp) -> resp.statusCode() in retryableHttpErrors) + .onRetry((event) -> "Failure on PUT request: $uri - event: $event") + return retryable.apply(()-> put0(uri, body, contentType, headers)) + } + + private HttpResponse put0(URI uri, byte[] body, String contentType, Map> headers) { + int authRetry = 0 + while( true ) { + final result = put1(uri, body, contentType, headers) + if( result.statusCode()==401 && (authRetry++)<2 && registry.auth.isRefreshable() ) { + loginService.invalidateAuthorization(image, registry.auth, credentials) + continue + } + return result + } + } + + private HttpResponse put1(URI uri, byte[] body, String contentType, Map> headers) { + try { + final builder = HttpRequest.newBuilder(uri) + .PUT(HttpRequest.BodyPublishers.ofByteArray(body)) + if( contentType ) + builder.setHeader("Content-Type", contentType) + copyHeaders(headers, builder) + final header = getAuthHeader() + if( header ) + builder.setHeader("Authorization", header) + final request = builder.build() + final response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()) + traceResponse(response) + return response + } + catch (IOException e) { + throw e + } + catch (RegistryForwardException | RegistryUnauthorizedAccessException e) { + throw e + } + catch (Exception e) { + throw new InternalServerException("Unexpected error on HTTP PUT request '$uri'", e) + } + } + private void traceResponse(HttpResponse resp) { // dump response if( !log.isTraceEnabled() || !resp ) @@ -316,7 +375,7 @@ class ProxyClient { // copy headers copyHeaders(headers, request) // add authorisation header - final header = loginService.getAuthorization(image, registry.auth, credentials) + final header = getAuthHeader() if( header ) request.header("Authorization", header) @@ -336,7 +395,7 @@ class ProxyClient { result.add("${entry.key}: $entry.value") } // add authorisation header - final header = loginService.getAuthorization(image, registry.auth, credentials) + final header = getAuthHeader() if( header ) { result.add('-H') result.add("Authorization: $header") diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy index 61e1428308..df28bb4bd2 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy @@ -29,6 +29,7 @@ import groovy.transform.EqualsAndHashCode import io.seqera.wave.api.BuildCompression import io.seqera.wave.api.BuildContext import io.seqera.wave.api.ContainerConfig +import io.seqera.wave.core.ChildRefs import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.tower.PlatformId import static io.seqera.wave.service.builder.BuildFormat.DOCKER @@ -149,6 +150,21 @@ class BuildRequest { */ final String buildTemplate + /** + * When {@code true}, email notifications should not be sent for this build + */ + final boolean noEmail + + /** + * Child build IDs for multi-platform builds + */ + final ChildRefs buildChildIds + + /** + * Child scan IDs for multi-platform builds + */ + final ChildRefs scanChildIds + BuildRequest( String containerId, String containerFile, @@ -167,7 +183,8 @@ class BuildRequest { BuildFormat format, Duration maxDuration, BuildCompression compression, - String buildTemplate + String buildTemplate, + boolean noEmail = false ) { this.containerId = containerId @@ -189,6 +206,7 @@ class BuildRequest { this.maxDuration = maxDuration this.compression = compression this.buildTemplate = buildTemplate + this.noEmail = noEmail // NOTE: this is meant to be updated - automatically - when the request is submitted this.buildId = computeBuildId(containerId) } @@ -217,12 +235,43 @@ class BuildRequest { this.compression = opts.compression as BuildCompression this.buildId = opts.buildId ?: computeBuildId(containerId) this.buildTemplate = opts.buildTemplate + this.noEmail = opts.noEmail as boolean + this.buildChildIds = opts.buildChildIds as ChildRefs + this.scanChildIds = opts.scanChildIds as ChildRefs } static BuildRequest of(Map opts) { new BuildRequest(opts) } + BuildRequest withChildScanIds(ChildRefs scanChildIds) { + return BuildRequest.of( + containerId: this.containerId, + containerFile: this.containerFile, + condaFile: this.condaFile, + workspace: this.workspace, + targetImage: this.targetImage, + identity: this.identity, + platform: this.platform, + cacheRepository: this.cacheRepository, + startTime: this.startTime, + ip: this.ip, + configJson: this.configJson, + offsetId: this.offsetId, + containerConfig: this.containerConfig, + scanId: this.scanId, + buildContext: this.buildContext, + format: this.format, + maxDuration: this.maxDuration, + compression: this.compression, + buildId: this.buildId, + buildTemplate: this.buildTemplate, + noEmail: this.noEmail, + buildChildIds: this.buildChildIds, + scanChildIds: scanChildIds + ) + } + @Override String toString() { return "BuildRequest[containerId=$containerId; targetImage=$targetImage; identity=$identity; dockerFile=${trunc(containerFile)}; condaFile=${trunc(condaFile)}; buildId=$buildId, maxDuration=$maxDuration]" diff --git a/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy b/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy new file mode 100644 index 0000000000..2eff6a74fe --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy @@ -0,0 +1,153 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.seqera.wave.auth.RegistryAuthService +import io.seqera.wave.auth.RegistryCredentialsProvider +import io.seqera.wave.auth.RegistryLookupService +import io.seqera.wave.configuration.HttpClientConfig +import io.seqera.wave.util.RegHelper +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.core.RegistryProxyService +import io.seqera.wave.core.RoutePath +import io.seqera.wave.http.HttpClientFactory +import io.seqera.wave.model.ContainerCoordinates +import io.seqera.wave.proxy.ProxyClient +import io.seqera.wave.tower.PlatformId +import jakarta.inject.Inject +import jakarta.inject.Singleton +import static io.seqera.wave.WaveDefault.ACCEPT_HEADERS +import static io.seqera.wave.model.ContentType.OCI_IMAGE_INDEX_V1 +import static io.seqera.wave.model.ContentType.OCI_IMAGE_MANIFEST_V1 +/** + * Assembles an OCI Image Index (manifest list) from individual + * platform-specific manifests and pushes it to the registry. + * + * @author Paolo Di Tommaso + */ +@Slf4j +@Singleton +@CompileStatic +class ManifestAssembler { + + @Inject + private RegistryLookupService registryLookup + + @Inject + private RegistryCredentialsProvider credentialsProvider + + @Inject + private RegistryAuthService loginService + + @Inject + private HttpClientConfig httpConfig + + /** + * Create and push an OCI Image Index (manifest list) to the registry. + * + * @param targetImage The final tag for the manifest list (e.g. repo:tag) + * @param platformImages List of platform-specific images [amd64-image, arm64-image] + * @param identity The platform identity for credentials lookup + */ + void createAndPushManifestList(String targetImage, List platformEntries, PlatformId identity) { + log.debug "Creating manifest list for targetImage=$targetImage from platformEntries=$platformEntries" + + // 1. Fetch each platform manifest + final manifests = platformEntries.collect { Map entry -> fetchPlatformManifest(entry.image as String, entry.platform as ContainerPlatform, identity) } + + // 2. Build the OCI Image Index JSON + final indexJson = buildImageIndex(manifests) + log.debug "OCI Image Index JSON: $indexJson" + + // 3. Push the index to the registry under the target tag + pushManifest(targetImage, indexJson, identity) + log.info "Successfully pushed manifest list for targetImage=$targetImage" + } + + protected Map fetchPlatformManifest(String image, ContainerPlatform platform, PlatformId identity) { + final coords = ContainerCoordinates.parse(image) + final route = RoutePath.v2manifestPath(coords, identity) + final client = createClient(route, identity, false) + + final resp = client.getString(route.path, ACCEPT_HEADERS) + if( resp.statusCode() != 200 ) + throw new IllegalStateException("Failed to GET manifest for '$image' — status: ${resp.statusCode()}, body: ${resp.body()}") + + final body = resp.body() + final bodyBytes = body.bytes + final contentType = resp.headers().firstValue('content-type').orElse(OCI_IMAGE_MANIFEST_V1) + final digest = resp.headers().firstValue('docker-content-digest').orElse(RegHelper.digest(bodyBytes)) + final size = bodyBytes.length + + return [ + mediaType: contentType, + digest: digest, + size: size, + platform: [architecture: platform.arch, os: platform.os] + ] + } + + static String buildImageIndex(List manifests) { + final index = [ + schemaVersion: 2, + mediaType: OCI_IMAGE_INDEX_V1, + manifests: manifests.collect { Map m -> + [ + mediaType: m.mediaType, + digest: m.digest, + size: m.size, + platform: m.platform + ] + } + ] + return JsonOutput.toJson(index) + } + + protected void pushManifest(String targetImage, String indexJson, PlatformId identity) { + final coords = ContainerCoordinates.parse(targetImage) + final route = RoutePath.v2manifestPath(coords, identity) + final client = createClient(route, identity, true) + + final body = indexJson.bytes + final path = "/v2/${coords.image}/manifests/${coords.reference}" + final resp = client.put(path, body, OCI_IMAGE_INDEX_V1) + + if( resp.statusCode() != 201 && resp.statusCode() != 200 ) { + throw new IllegalStateException("Failed to PUT manifest list for '$targetImage' — status: ${resp.statusCode()}, body: ${resp.body()}") + } + log.debug "PUT manifest list for '$targetImage' — status: ${resp.statusCode()}" + } + + private ProxyClient createClient(RoutePath route, PlatformId identity, boolean pushMode) { + final httpClient = HttpClientFactory.neverRedirectsHttpClient() + final registry = registryLookup.lookup(route.registry) + final creds = credentialsProvider.getCredentials(route, identity) + new ProxyClient(httpClient, httpConfig) + .withRoute(route) + .withImage(route.image) + .withRegistry(registry) + .withCredentials(creds) + .withLoginService(loginService) + .withPushMode(pushMode) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/builder/MultiBuildEntry.groovy b/src/main/groovy/io/seqera/wave/service/builder/MultiBuildEntry.groovy new file mode 100644 index 0000000000..01e3bdc5de --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/builder/MultiBuildEntry.groovy @@ -0,0 +1,78 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString +import io.seqera.wave.service.job.JobEntry +import io.seqera.data.store.state.RequestIdAware +import io.seqera.data.store.state.StateEntry +/** + * Model a multi-platform build entry object + * + * @author Paolo Di Tommaso + */ +@ToString(includeNames = true, includePackage = false) +@EqualsAndHashCode +@CompileStatic +class MultiBuildEntry implements StateEntry, JobEntry, RequestIdAware { + + final MultiBuildRequest request + + final BuildResult result + + protected MultiBuildEntry() {} + + MultiBuildEntry(MultiBuildRequest request, BuildResult result) { + this.request = request + this.result = result + } + + MultiBuildRequest getRequest() { request } + + BuildResult getResult() { result } + + @Override + String getKey() { + return request.targetImage + } + + @Override + String getRequestId() { + return request.multiBuildId + } + + @Override + boolean done() { + result?.done() + } + + Boolean succeeded() { + return done() ? result.succeeded() : null + } + + MultiBuildEntry withResult(BuildResult result) { + new MultiBuildEntry(this.request, result) + } + + static MultiBuildEntry of(MultiBuildRequest request) { + new MultiBuildEntry(request, BuildResult.create(request.buildId)) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/builder/MultiBuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/MultiBuildRequest.groovy new file mode 100644 index 0000000000..fc8b954000 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/builder/MultiBuildRequest.groovy @@ -0,0 +1,101 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import java.time.Duration +import java.time.Instant + +import groovy.transform.Canonical +import groovy.transform.CompileStatic +import groovy.transform.ToString +import io.seqera.random.LongRndKey +import io.seqera.wave.tower.PlatformId +/** + * Model a multi-platform build request + * + * @author Paolo Di Tommaso + */ +@ToString(includeNames = true, includePackage = false) +@Canonical +@CompileStatic +class MultiBuildRequest { + + static final String ID_PREFIX = 'mb-' + + final String multiBuildId + final String targetImage + final String containerId + final String buildId + final String amd64TargetImage + final String arm64TargetImage + final boolean amd64Cached + final boolean arm64Cached + final PlatformId identity + final Instant creationTime + final Duration maxDuration + + static MultiBuildRequest create( + String containerId, + String targetImage, + String buildId, + String amd64TargetImage, + String arm64TargetImage, + boolean amd64Cached, + boolean arm64Cached, + PlatformId identity, + Duration maxDuration + ) { + assert containerId, "Argument 'containerId' cannot be empty" + assert targetImage, "Argument 'targetImage' cannot be empty" + assert buildId, "Argument 'buildId' cannot be empty" + assert amd64TargetImage, "Argument 'amd64TargetImage' cannot be empty" + assert arm64TargetImage, "Argument 'arm64TargetImage' cannot be empty" + + final multiBuildId = ID_PREFIX + LongRndKey.rndHex() + return new MultiBuildRequest( + multiBuildId, + targetImage, + containerId, + buildId, + amd64TargetImage, + arm64TargetImage, + amd64Cached, + arm64Cached, + identity, + Instant.now(), + maxDuration + ) + } + + static MultiBuildRequest of(Map opts) { + new MultiBuildRequest( + opts.multiBuildId as String, + opts.targetImage as String, + opts.containerId as String, + opts.buildId as String, + opts.amd64TargetImage as String, + opts.arm64TargetImage as String, + opts.amd64Cached as boolean, + opts.arm64Cached as boolean, + opts.identity as PlatformId, + opts.creationTime as Instant, + opts.maxDuration as Duration + ) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/builder/MultiBuildStateStore.groovy b/src/main/groovy/io/seqera/wave/service/builder/MultiBuildStateStore.groovy new file mode 100644 index 0000000000..635f1e6140 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/builder/MultiBuildStateStore.groovy @@ -0,0 +1,58 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import java.time.Duration + +import groovy.transform.CompileStatic +import io.micronaut.context.annotation.Requires +import io.seqera.wave.configuration.BuildConfig +import io.seqera.wave.configuration.BuildEnabled +import io.seqera.serde.moshi.MoshiEncodeStrategy +import io.seqera.data.store.state.AbstractStateStore +import io.seqera.data.store.state.impl.StateProvider +import jakarta.inject.Inject +import jakarta.inject.Singleton +/** + * Implement a {@link io.seqera.data.store.state.StateStore} for {@link MultiBuildEntry} objects + * + * @author Paolo Di Tommaso + */ +@Singleton +@Requires(bean = BuildEnabled) +@CompileStatic +class MultiBuildStateStore extends AbstractStateStore { + + @Inject + private BuildConfig config + + MultiBuildStateStore(StateProvider provider) { + super(provider, new MoshiEncodeStrategy() {}) + } + + @Override + protected String getPrefix() { + return 'wave-multibuild/v1' + } + + @Override + protected Duration getDuration() { + return config.statusDuration + } +} diff --git a/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy new file mode 100644 index 0000000000..c79efdeadd --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy @@ -0,0 +1,313 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import java.time.Instant +import java.util.regex.Pattern + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.context.event.ApplicationEventPublisher + +import io.seqera.wave.api.ContainerConfig +import io.seqera.wave.api.ContainerLayer +import io.seqera.wave.core.ChildRefs +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.model.ContainerCoordinates +import io.seqera.wave.service.job.JobHandler +import io.seqera.wave.service.job.JobService +import io.seqera.wave.service.job.JobSpec +import io.seqera.wave.service.job.JobState +import io.seqera.wave.service.persistence.PersistenceService +import io.seqera.wave.service.persistence.WaveBuildRecord +import io.seqera.wave.service.scan.ContainerScanService +import io.seqera.wave.tower.PlatformId +import jakarta.annotation.Nullable +import jakarta.inject.Inject +import jakarta.inject.Named +import jakarta.inject.Singleton +import static io.seqera.wave.util.ContainerHelper.makeContainerId +import static io.seqera.wave.util.ContainerHelper.makeTargetImage +/** + * Orchestrates multi-platform container builds by fanning out + * two single-platform builds (linux/amd64 + linux/arm64) and + * assembling the results into an OCI Image Index. + * + * @author Paolo Di Tommaso + */ +@Slf4j +@Singleton +@Named('MultiBuild') +@CompileStatic +class MultiPlatformBuildService implements JobHandler { + + @Inject + ContainerBuildService buildService + + @Inject + BuildStateStore buildStore + + @Inject + MultiBuildStateStore multiBuildStore + + @Inject + ManifestAssembler manifestAssembler + + @Inject + JobService jobService + + @Inject + ApplicationEventPublisher eventPublisher + + @Inject + PersistenceService persistenceService + + @Inject @Nullable + ContainerScanService scanService + + static final ContainerPlatform PLATFORM_AMD64 = ContainerPlatform.of('linux/amd64') + static final ContainerPlatform PLATFORM_ARM64 = ContainerPlatform.of('linux/arm64') + + /** + * Build a multi-platform image by orchestrating two single-platform builds + * and assembling the result into a manifest list. + * + * @param templateRequest The original build request used as template + * @param finalContainerId The container ID for the multi-platform image + * @param finalTargetImage The final target image tag for the manifest list + * @param identity The platform identity + * @return A BuildTrack with the final target image (succeeded=null means in-progress) + */ + BuildTrack buildMultiPlatformImage(BuildRequest templateRequest, String finalContainerId, String finalTargetImage, PlatformId identity) { + log.debug "Starting multi-platform build: finalTargetImage=$finalTargetImage, finalContainerId=$finalContainerId" + + // Create platform-specific build requests + final amd64Request = createPlatformRequest(templateRequest, PLATFORM_AMD64, '-linux-amd64') + final arm64Request = createPlatformRequest(templateRequest, PLATFORM_ARM64, '-linux-arm64') + + // Submit both platform builds + final amd64Track = buildService.buildImage(amd64Request) + final arm64Track = buildService.buildImage(arm64Request) + + log.debug "Submitted platform builds: amd64=${amd64Track.targetImage} (cached=${amd64Track.cached}), arm64=${arm64Track.targetImage} (cached=${arm64Track.cached})" + + final buildId = BuildRequest.ID_PREFIX + finalContainerId + BuildRequest.SEP + '0' + final startTime = Instant.now() + + // Encode child build IDs for the parent build view + final buildChildIds = new ChildRefs([ + new ChildRefs.Ref(amd64Track.id, PLATFORM_AMD64.toString()), + new ChildRefs.Ref(arm64Track.id, PLATFORM_ARM64.toString()) + ]) + + // Store an in-progress build entry so status polling can find it + final syntheticRequest = BuildRequest.of( + buildId: buildId, + containerId: finalContainerId, + targetImage: finalTargetImage, + startTime: startTime, + identity: templateRequest.identity, + containerFile: templateRequest.containerFile, + condaFile: templateRequest.condaFile, + workspace: templateRequest.workspace, + platform: ContainerPlatform.MULTI_PLATFORM, + configJson: templateRequest.configJson, + ip: templateRequest.ip, + offsetId: templateRequest.offsetId, + scanId: templateRequest.scanId, + format: templateRequest.format, + compression: templateRequest.compression, + buildTemplate: templateRequest.buildTemplate, + buildChildIds: buildChildIds, + scanChildIds: templateRequest.scanChildIds + ) + final initialEntry = BuildEntry.create(syntheticRequest) + final stored = buildStore.storeIfAbsent(finalTargetImage, initialEntry) + if( !stored ) { + // another request already started a build for this image — return the existing build track + log.debug "Multi-platform build already in progress for $finalTargetImage" + return new BuildTrack(buildId, finalTargetImage, false, null) + } + + // Create the multi-build request and entry, then launch via job queue + final multiBuildRequest = MultiBuildRequest.create( + finalContainerId, + finalTargetImage, + buildId, + amd64Track.targetImage, + arm64Track.targetImage, + amd64Track.cached, + arm64Track.cached, + identity, + templateRequest.maxDuration + ) + final multiBuildEntry = MultiBuildEntry.of(multiBuildRequest) + multiBuildStore.put(finalTargetImage, multiBuildEntry) + + // Launch the multi-build job — goes directly to processing queue + jobService.launchMultiBuild(multiBuildRequest) + + // Return immediately — build is in progress (succeeded=null) + return new BuildTrack(buildId, finalTargetImage, false, null) + } + + // ************************************************************** + // ** JobHandler implementation + // ************************************************************** + + @Override + MultiBuildEntry getJobEntry(JobSpec job) { + multiBuildStore.get(job.entryKey) + } + + @Override + JobSpec launchJob(JobSpec job, MultiBuildEntry entry) { + // no-op: multi-build jobs don't launch K8s/Docker processes + return job.withLaunchTime(Instant.now()) + } + + @Override + void onJobCompletion(JobSpec job, MultiBuildEntry entry, JobState state) { + final request = entry.request + final buildId = request.buildId + final startTime = request.creationTime + + if( state.succeeded() ) { + try { + final List platformEntries = [ + [image: request.amd64TargetImage, platform: PLATFORM_AMD64], + [image: request.arm64TargetImage, platform: PLATFORM_ARM64] + ] + manifestAssembler.createAndPushManifestList(request.targetImage, platformEntries, request.identity) + log.info "Multi-platform manifest list assembled for ${request.targetImage}" + + final completedResult = BuildResult.completed(buildId, 0, 'Multi-platform build completed', startTime, null) + updateStores(entry, completedResult) + } + catch (Exception e) { + log.error "Multi-platform manifest assembly failed for ${request.targetImage}", e + final failedResult = BuildResult.failed(buildId, "Manifest assembly failed: ${e.message}", startTime) + updateStores(entry, failedResult) + } + } + else { + final failedResult = BuildResult.failed(buildId, state.stdout ?: "Multi-platform build failed", startTime) + updateStores(entry, failedResult) + } + } + + @Override + void onJobException(JobSpec job, MultiBuildEntry entry, Throwable error) { + final request = entry.request + final result = BuildResult.failed(request.buildId, error.message, request.creationTime) + updateStores(entry, result) + log.error("Multi-platform build exception for '${request.targetImage}' - operation=${job.operationName}; cause=${error.message}", error) + } + + @Override + void onJobTimeout(JobSpec job, MultiBuildEntry entry) { + final request = entry.request + final result = BuildResult.failed(request.buildId, "Multi-platform build timed out '${request.targetImage}'", request.creationTime) + updateStores(entry, result) + log.warn "Multi-platform build timed out '${request.targetImage}'; operation=${job.operationName}; duration=${result.duration}" + } + + // ************************************************************** + // ** helper methods + // ************************************************************** + + private void updateStores(MultiBuildEntry entry, BuildResult result) { + final targetImage = entry.request.targetImage + // update the multi-build state store + multiBuildStore.put(targetImage, entry.withResult(result)) + // update the build state store for status polling and publish event + final existing = buildStore.getBuild(targetImage) + if( existing ) { + final updatedEntry = existing.withResult(result) + buildStore.storeBuild(targetImage, updatedEntry) + // trigger scan on the composite image before persisting the record + // (scan record must exist before build record to avoid status polling race condition) + scanService?.scanOnBuild(updatedEntry) + // persist the build record and publish event (triggers email notification) + final event = new BuildEvent(updatedEntry.request, result) + persistenceService.saveBuildAsync(WaveBuildRecord.fromEvent(event)) + eventPublisher.publishEvent(event) + } + else { + log.warn "Multi-platform build entry not found in build store for $targetImage" + } + } + + static final private Pattern FUSION_LAYER_ARCH = Pattern.compile('.*/fusion-(\\w+)\\.tar\\.gz$') + + /** + * Filter container config layers to only include fusion layers matching the target platform arch. + * Non-fusion layers are kept as-is. + */ + static ContainerConfig filterLayersForPlatform(ContainerConfig config, ContainerPlatform platform) { + if( !config?.layers || !platform || platform.isMultiArch() ) + return config + final filtered = config.layers.findAll { ContainerLayer layer -> + final matcher = layer.location ? FUSION_LAYER_ARCH.matcher(layer.location) : null + final keep = !matcher?.matches() || matcher.group(1) == platform.arch + if( matcher?.matches() ) { + log.debug "Layer filter: platform=${platform}; layer=${layer.location}; arch=${matcher.group(1)}; keep=${keep}" + } + return keep + } + log.debug "Layer filter: platform=${platform}; input=${config.layers.size()} layers; output=${filtered.size()} layers" + return new ContainerConfig(config.entrypoint, config.cmd, config.env, config.workingDir, filtered) + } + + protected BuildRequest createPlatformRequest(BuildRequest template, ContainerPlatform platform, String suffix) { + final repo = ContainerCoordinates.parse(template.targetImage).repository + final containerConfig = filterLayersForPlatform(template.containerConfig, platform) + final platformId = makeContainerId( + template.containerFile, + template.condaFile, + platform, + repo, + template.buildContext, + containerConfig + ) + final platformImage = makeTargetImage(template.format, repo, platformId, template.condaFile, null) + + return new BuildRequest( + platformId, + template.containerFile, + template.condaFile, + template.workspace, + platformImage, + template.identity, + platform, + template.cacheRepository, + template.ip, + template.configJson, + template.offsetId, + containerConfig, + null, // scanId - suppress per-platform scans; scan runs on the composite image + template.buildContext, + template.format, + template.maxDuration, + template.compression, + template.buildTemplate, + true // noEmail - suppress individual sub-build notifications + ) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/cleanup/CleanupServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/cleanup/CleanupServiceImpl.groovy index ee168237b7..1af56be2d7 100644 --- a/src/main/groovy/io/seqera/wave/service/cleanup/CleanupServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/cleanup/CleanupServiceImpl.groovy @@ -150,8 +150,10 @@ class CleanupServiceImpl implements Runnable, CleanupService { ? config.succeededDuration : config.failedDuration final expirationSecs = Instant.now().plus(ttl).epochSecond - // schedule the job deletion - store.add(JOB_PREFIX + job.operationName, expirationSecs) + // schedule the job deletion (skip for MultiBuild — no K8s/Docker job to delete) + if( job.type != JobSpec.Type.MultiBuild ) { + store.add(JOB_PREFIX + job.operationName, expirationSecs) + } // schedule work dir path deletion if( job.workDir ) { store.add(DIR_PREFIX + job.workDir, expirationSecs) diff --git a/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy b/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy index efd1acdf66..afec2716f4 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy @@ -28,6 +28,7 @@ import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.configuration.WaveLite import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.builder.MultiBuildRequest import io.seqera.wave.configuration.MirrorConfig import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.scan.ScanRequest @@ -95,6 +96,16 @@ class JobFactory { ) } + JobSpec multiBuild(MultiBuildRequest request) { + assert request.multiBuildId.startsWith(MultiBuildRequest.ID_PREFIX) + JobSpec.multiBuild( + request.targetImage, + request.multiBuildId.replace('_', '-'), + request.creationTime, + request.maxDuration + ) + } + static private String generate(String type, String id, Instant creationTime) { final prefix = type.toLowerCase() return prefix + '-' + Hashing diff --git a/src/main/groovy/io/seqera/wave/service/job/JobService.groovy b/src/main/groovy/io/seqera/wave/service/job/JobService.groovy index ff8d62dfe0..61b5bc634a 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobService.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobService.groovy @@ -20,6 +20,7 @@ package io.seqera.wave.service.job import io.seqera.wave.service.blob.BlobEntry import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.builder.MultiBuildRequest import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.scan.ScanRequest @@ -38,6 +39,8 @@ interface JobService { JobSpec launchMirror(MirrorRequest request) + JobSpec launchMultiBuild(MultiBuildRequest request) + JobState status(JobSpec jobSpec) void cleanup(JobSpec jobSpec, Integer exitStatus) diff --git a/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy index bd502997a6..3c322fdb73 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy @@ -27,6 +27,10 @@ import io.seqera.wave.configuration.WaveLite import io.seqera.wave.service.blob.BlobEntry import io.seqera.wave.service.blob.TransferStrategy import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.builder.BuildStateStore +import io.seqera.wave.service.builder.MultiBuildEntry +import io.seqera.wave.service.builder.MultiBuildRequest +import io.seqera.wave.service.builder.MultiBuildStateStore import io.seqera.wave.service.cleanup.CleanupService import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.scan.ScanRequest @@ -62,6 +66,14 @@ class JobServiceImpl implements JobService { @Nullable private TransferStrategy transferStrategy + @Inject + @Nullable + private BuildStateStore buildStateStore + + @Inject + @Nullable + private MultiBuildStateStore multiBuildStateStore + @Override JobSpec launchTransfer(BlobEntry blob, List command) { if( !transferStrategy ) @@ -107,8 +119,20 @@ class JobServiceImpl implements JobService { return job } + @Override + JobSpec launchMultiBuild(MultiBuildRequest request) { + // create the unique job id for the multi-platform build + final job = jobFactory.multiBuild(request) + // multi-build jobs go directly to processing queue (no K8s resources needed) + processingQueue.offer(job.withLaunchTime(java.time.Instant.now())) + return job + } + @Override JobState status(JobSpec job) { + if( job.type == JobSpec.Type.MultiBuild ) { + return multiBuildStatus(job) + } try { return operations.status(job) } @@ -122,4 +146,47 @@ class JobServiceImpl implements JobService { void cleanup(JobSpec job, Integer exitStatus) { cleanupService.cleanupJob(job, exitStatus) } + + protected JobState multiBuildStatus(JobSpec job) { + try { + final entry = multiBuildStateStore?.get(job.entryKey) as MultiBuildEntry + if( !entry ) + return new JobState(JobState.Status.UNKNOWN, null, null) + + // check if both sub-builds are done (skip store read for cached platforms) + final amd64Entry = entry.request.amd64Cached ? null : buildStateStore?.getBuild(entry.request.amd64TargetImage) + final arm64Entry = entry.request.arm64Cached ? null : buildStateStore?.getBuild(entry.request.arm64TargetImage) + + // fail if a non-cached sub-build entry has been evicted from the store + if( !entry.request.amd64Cached && amd64Entry == null ) { + return JobState.failed(null, "amd64 sub-build entry not found in store") + } + if( !entry.request.arm64Cached && arm64Entry == null ) { + return JobState.failed(null, "arm64 sub-build entry not found in store") + } + + final amd64Done = entry.request.amd64Cached || amd64Entry?.done() + final arm64Done = entry.request.arm64Cached || arm64Entry?.done() + + if( !amd64Done || !arm64Done ) { + return JobState.running() + } + + // both done — check if both succeeded + final amd64Ok = entry.request.amd64Cached || amd64Entry?.result?.succeeded() + final arm64Ok = entry.request.arm64Cached || arm64Entry?.result?.succeeded() + + if( amd64Ok && arm64Ok ) { + return JobState.succeeded(null) + } + else { + final msg = "Multi-platform sub-build failed — amd64=${amd64Ok}, arm64=${arm64Ok}" + return JobState.failed(null, msg) + } + } + catch (Throwable t) { + log.warn "Unable to obtain multi-build status for job=${job.operationName} - cause: ${t.message}", t + return new JobState(JobState.Status.UNKNOWN, null, t.message) + } + } } diff --git a/src/main/groovy/io/seqera/wave/service/job/JobSpec.groovy b/src/main/groovy/io/seqera/wave/service/job/JobSpec.groovy index 96f72a41c8..af21ebd94f 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobSpec.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobSpec.groovy @@ -37,7 +37,7 @@ import io.seqera.random.LongRndKey @CompileStatic class JobSpec { - enum Type { Transfer, Build, Scan, Mirror } + enum Type { Transfer, Build, Scan, Mirror, MultiBuild } /** * The job unique identifier @@ -144,6 +144,19 @@ class JobSpec { ) } + static JobSpec multiBuild(String recordId, String operationName, Instant creationTime, Duration maxDuration) { + new JobSpec( + LongRndKey.rndHex(), + Type.MultiBuild, + recordId, + operationName, + creationTime, + null, + maxDuration, + null + ) + } + JobSpec withLaunchTime(Instant instant) { new JobSpec( this.id, diff --git a/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy index 24f66bffe8..7eb3c47313 100644 --- a/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy @@ -59,6 +59,8 @@ class MailServiceImpl implements MailService { @EventListener void onBuildEvent(BuildEvent event) { + if( event.request.noEmail ) + return try { sendCompletionEmail(event.request, event.result) } @@ -101,8 +103,6 @@ class MailServiceImpl implements MailService { binding.build_condafile = req.condaFile binding.build_digest = result.digest ?: '-' binding.build_url = "$serverUrl/view/builds/${result.buildId}" - binding.scan_url = req.scanId && result.succeeded() ? "$serverUrl/view/scans/${req.scanId}" : null - binding.scan_id = req.scanId binding.put('server_url', serverUrl) // result the main object Mail mail = new Mail() diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy index 415439c6fe..9d05426618 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy @@ -26,6 +26,7 @@ import groovy.transform.EqualsAndHashCode import groovy.transform.ToString import io.seqera.wave.api.BuildCompression import io.seqera.wave.api.BuildStatusResponse +import io.seqera.wave.core.ChildRefs import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildFormat @@ -61,6 +62,8 @@ class WaveBuildRecord { String digest BuildCompression compression String buildTemplate + ChildRefs buildChildIds + ChildRefs scanChildIds Boolean succeeded() { return duration != null ? (exitStatus==0) : null @@ -91,7 +94,7 @@ class WaveBuildRecord { userId: request.identity.user?.id, requestIp: request.ip, startTime: request.startTime, - platform: request.platform, + platform: request.platform?.toString(), offsetId: request.offsetId, scanId: request.scanId, format: request.format, @@ -99,7 +102,9 @@ class WaveBuildRecord { exitStatus: result?.exitStatus, digest: result?.digest, compression: request?.compression, - buildTemplate: request?.buildTemplate + buildTemplate: request?.buildTemplate, + buildChildIds: request?.buildChildIds, + scanChildIds: request?.scanChildIds ) } diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy index 7261020adc..9d5504ec26 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy @@ -28,6 +28,7 @@ import groovy.transform.ToString import groovy.util.logging.Slf4j import io.seqera.wave.api.ContainerConfig import io.seqera.wave.api.SubmitContainerTokenRequest +import io.seqera.wave.core.ChildRefs import io.seqera.wave.service.request.ContainerRequest import io.seqera.wave.tower.User import io.seqera.wave.util.FusionVersionStringDeserializer @@ -176,6 +177,11 @@ class WaveContainerRecord { */ final String scanId + /** + * Child scan IDs for multi-platform builds + */ + final ChildRefs scanChildIds + WaveContainerRecord(SubmitContainerTokenRequest request, ContainerRequest data, String waveImage, String addr, Instant expiration) { this.id = data.requestId this.user = data.identity.user @@ -202,6 +208,7 @@ class WaveContainerRecord { this.fusionVersion = request?.containerConfig?.fusionVersion()?.number this.mirror = data.mirror this.scanId = data.scanId + this.scanChildIds = data.scanChildIds } WaveContainerRecord(WaveContainerRecord that, String sourceDigest, String waveDigest) { @@ -226,8 +233,9 @@ class WaveContainerRecord { this.buildNew = that.buildNew this.freeze = that.freeze this.fusionVersion = that.fusionVersion - this.mirror == that.mirror + this.mirror = that.mirror this.scanId = that.scanId + this.scanChildIds = that.scanChildIds // -- digest part this.sourceDigest = sourceDigest this.waveDigest = waveDigest diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy index cbdff2e9ec..d74fe847ec 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy @@ -26,7 +26,6 @@ import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString import groovy.util.logging.Slf4j -import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.scan.ScanEntry import io.seqera.wave.service.scan.ScanVulnerability /** @@ -46,7 +45,7 @@ class WaveScanRecord implements Cloneable { String mirrorId String requestId String containerImage - ContainerPlatform platform + String platform Instant startTime Duration duration String status @@ -64,7 +63,7 @@ class WaveScanRecord implements Cloneable { String mirrorId, String requestId, String containerImage, - ContainerPlatform platform, + String platform, Instant startTime, Duration duration, String status, @@ -97,7 +96,7 @@ class WaveScanRecord implements Cloneable { this.mirrorId = scan.mirrorId this.requestId = scan.requestId this.containerImage = scan.containerImage - this.platform = scan.platform + this.platform = scan.platform?.toString() this.startTime = scan.startTime this.duration = scan.duration this.status = scan.status diff --git a/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy index c59898b4bd..1420ff36bd 100644 --- a/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy @@ -25,6 +25,7 @@ import groovy.transform.CompileStatic import io.seqera.wave.api.ContainerConfig import io.seqera.wave.api.ScanLevel import io.seqera.wave.api.ScanMode +import io.seqera.wave.core.ChildRefs import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.model.ContainerCoordinates import io.seqera.wave.tower.PlatformId @@ -53,6 +54,7 @@ class ContainerRequest { Boolean buildNew Boolean freeze String scanId + ChildRefs scanChildIds ScanMode scanMode List scanLevels boolean dryRun @@ -88,6 +90,7 @@ class ContainerRequest { "buildNew=${buildNew}; " + "freeze=${freeze}; " + "scanId=${scanId}; " + + "scanChildIds=${scanChildIds}; " + "scanMode=${scanMode}; " + "scanLevels=${scanLevels}; " + "dryRun=${dryRun}; " + @@ -144,6 +147,10 @@ class ContainerRequest { return scanId } + ChildRefs getScanChildIds() { + return scanChildIds + } + ScanMode getScanMode() { return scanMode } @@ -176,6 +183,7 @@ class ContainerRequest { Boolean buildNew, Boolean freeze, String scanId, + ChildRefs scanChildIds, ScanMode scanMode, List scanLevels, boolean dryRun, @@ -196,6 +204,7 @@ class ContainerRequest { buildNew, freeze, scanId, + scanChildIds, scanMode, scanLevels, dryRun, @@ -222,6 +231,7 @@ class ContainerRequest { (Boolean) data.buildNew, (Boolean) data.freeze, data.scanId as String, + data.scanChildIds as ChildRefs, data.scanMode as ScanMode, data.scanLevels as List, data.dryRun as boolean, diff --git a/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy index c098d07f4b..9edb5e87a6 100644 --- a/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy @@ -18,6 +18,7 @@ package io.seqera.wave.service.request +import io.seqera.wave.core.ChildRefs import io.seqera.wave.exception.UnsupportedBuildServiceException import io.seqera.wave.exception.UnsupportedMirrorServiceException import io.seqera.wave.exception.UnsupportedScanServiceException @@ -104,6 +105,9 @@ class ContainerStatusServiceImpl implements ContainerStatusService { } if( request.scanId && request.scanMode == ScanMode.required && scanService ) { + if( request.scanChildIds ) { + return handleMultiScanStatus(request, state) + } final scan = getScanState(request.scanId) if ( !scan ) throw new NotFoundException("Missing container scan record with id: ${request.scanId}") @@ -177,6 +181,42 @@ class ContainerStatusServiceImpl implements ContainerStatusService { ) } + protected ContainerStatusResponse handleMultiScanStatus(ContainerRequest request, ContainerState state) { + final List scans = new ArrayList<>(request.scanChildIds.size()) + boolean allDone = true + boolean allSucceeded = true + Duration maxScanDuration = Duration.ZERO + for( ChildRefs.Ref pair : request.scanChildIds ) { + final scan = getScanState(pair.id) + if( !scan ) + throw new NotFoundException("Missing container scan record with id: ${pair.id}") + scans.add(scan) + if( !scan.duration ) { + allDone = false + } + else { + if( scan.duration > maxScanDuration ) + maxScanDuration = scan.duration + if( !scan.succeeded() ) + allSucceeded = false + } + } + + if( !allDone ) { + return createResponse0(SCANNING, request, new ContainerState(state.startTime)) + } + + // all scans completed — pick the first failure or the last success for the response + final combinedScan = allSucceeded + ? scans.last() + : scans.find { !it.succeeded() } + // use max duration since per-arch scans run in parallel + final newState = state + ? new ContainerState(state.startTime, state.duration + maxScanDuration, allSucceeded) + : new ContainerState(combinedScan.startTime, maxScanDuration, allSucceeded) + return createScanResponse(request, newState, combinedScan) + } + protected StageResult buildResult(ContainerRequest request, ContainerState state) { if( state.succeeded ) return new StageResult(true) @@ -196,12 +236,13 @@ class ContainerStatusServiceImpl implements ContainerStatusService { } protected StageResult scanResult(ContainerRequest request, ScanEntry scan) { + final scanDetailId = scan.scanId // scan was not successful if (!scan.succeeded()) { return new StageResult( false, "Container security scan did not complete successfully", - "${serverUrl}/view/scans/${request.scanId}" + "${serverUrl}/view/scans/${scanDetailId}" ) } @@ -219,7 +260,7 @@ class ContainerStatusServiceImpl implements ContainerStatusService { return new StageResult( false, "Container security scan operation found one or more vulnerabilities with severity: ${foundLevels.join(',')}", - "${serverUrl}/view/scans/${request.scanId}" + "${serverUrl}/view/scans/${scanDetailId}" ) } @@ -227,7 +268,7 @@ class ContainerStatusServiceImpl implements ContainerStatusService { return new StageResult( true, "Container security scan operation found one or more vulnerabilities that are compatible with requested security levels: ${allowedLevels.join(',')}", - "${serverUrl}/view/scans/${request.scanId}" + "${serverUrl}/view/scans/${scanDetailId}" ) } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index 53541f8396..29833c090c 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -27,6 +27,7 @@ import java.util.concurrent.ExecutorService import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import io.seqera.wave.core.ChildRefs import io.micronaut.context.annotation.Requires import io.micronaut.core.annotation.Nullable import io.micronaut.http.server.types.files.StreamedFile @@ -36,6 +37,7 @@ import io.micronaut.objectstorage.request.UploadRequest import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.api.ScanMode import io.seqera.wave.configuration.ScanConfig +import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.configuration.ScanEnabled import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildRequest @@ -126,8 +128,16 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler, JobEntry { request.creationTime, null, PENDING, - List.of()) + List.of(), + null, + null) } ScanEntry success(List vulnerabilities){ @@ -159,7 +161,8 @@ class ScanEntry implements StateEntry, JobEntry { Duration.between(this.startTime, Instant.now()), SUCCEEDED, vulnerabilities, - 0 ) + 0, + null) } ScanEntry failure(Integer exitCode, String logs){ @@ -193,7 +196,9 @@ class ScanEntry implements StateEntry, JobEntry { request.creationTime, Duration.between(request.creationTime, Instant.now()), FAILED, - List.of()) + List.of(), + null, + null) } @@ -234,7 +239,7 @@ class ScanEntry implements StateEntry, JobEntry { record.mirrorId, record.requestId, record.containerImage, - record.platform, + record.platform ? ContainerPlatform.of(record.platform) : null, record.workDir, null, record.startTime, diff --git a/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy b/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy index ed0b55609e..ba745e1d46 100644 --- a/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy +++ b/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy @@ -298,6 +298,10 @@ class ContainerHelper { return RegHelper.sipHash(attrs) } + static String makeMultiPlatformContainerId(String containerFile, String condaFile, String repository, BuildContext buildContext, ContainerConfig containerConfig) { + return makeContainerId(containerFile, condaFile, ContainerPlatform.MULTI_PLATFORM, repository, buildContext, containerConfig) + } + static void checkContainerSpec(String file) { if( !file ) return diff --git a/src/main/resources/io/seqera/wave/build-notification.html b/src/main/resources/io/seqera/wave/build-notification.html index 5e8295dba0..d9f4455b3a 100644 --- a/src/main/resources/io/seqera/wave/build-notification.html +++ b/src/main/resources/io/seqera/wave/build-notification.html @@ -344,12 +344,6 @@

Build Summary

${build_exit_status} - <% if (scan_url) { %> - - Security Scan - ${scan_id} - - <% } %> diff --git a/src/main/resources/io/seqera/wave/build-view.hbs b/src/main/resources/io/seqera/wave/build-view.hbs index e93feb3786..d4f2e2f161 100644 --- a/src/main/resources/io/seqera/wave/build-view.hbs +++ b/src/main/resources/io/seqera/wave/build-view.hbs @@ -96,6 +96,16 @@ Platform {{build_platform}} + {{#if build_entries}} + {{#each build_entries}} + + Build ({{build_platform}}) + + {{build_id}} + + + {{/each}} + {{/if}} Build template {{build_template}} @@ -112,7 +122,16 @@ Exit status {{build_exit_status}} - {{#if scan_url}} + {{#if scan_entries}} + {{#each scan_entries}} + + Security scan ({{scan_platform}}) + + {{scan_id}} + + + {{/each}} + {{else if scan_url}} Security scan diff --git a/src/main/resources/io/seqera/wave/container-view.hbs b/src/main/resources/io/seqera/wave/container-view.hbs index 50131951b6..848b10c973 100644 --- a/src/main/resources/io/seqera/wave/container-view.hbs +++ b/src/main/resources/io/seqera/wave/container-view.hbs @@ -244,7 +244,14 @@ {{mirror_cached}} {{/if}} - {{#if scan_id}} + {{#if scan_entries}} + {{#each scan_entries}} + + Security scan ({{scan_platform}}) + {{scan_id}} + + {{/each}} + {{else if scan_id}} Security scan {{scan_id}} diff --git a/src/main/resources/io/seqera/wave/mirror-view.hbs b/src/main/resources/io/seqera/wave/mirror-view.hbs index 52b5d40a84..b9695ef2c5 100644 --- a/src/main/resources/io/seqera/wave/mirror-view.hbs +++ b/src/main/resources/io/seqera/wave/mirror-view.hbs @@ -114,7 +114,14 @@ {{mirror_exitcode}} {{/if}} - {{#if scan_url}} + {{#if scan_entries}} + {{#each scan_entries}} + + Security scan ({{scan_platform}}) + {{scan_id}} + + {{/each}} + {{else if scan_url}} Security scan {{scan_id}} diff --git a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy index 004a3ee813..54a29d0848 100644 --- a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy @@ -371,7 +371,7 @@ class ViewControllerTest extends Specification { 'mr-12345', 'cr-12345', 'docker.io/some:image', - ContainerPlatform.DEFAULT, + ContainerPlatform.DEFAULT.toString(), Instant.now(), Duration.ofMinutes(1), ScanEntry.SUCCEEDED, @@ -410,7 +410,7 @@ class ViewControllerTest extends Specification { 'mr-12345', 'cr-12345', 'docker.io/some:image', - ContainerPlatform.DEFAULT, + ContainerPlatform.DEFAULT.toString(), Instant.now(), Duration.ofMinutes(1), ScanEntry.SUCCEEDED, @@ -684,7 +684,7 @@ class ViewControllerTest extends Specification { def service = Mock(ContainerScanService) def controller = new ViewController(scanService: service) def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/arm64') + def PLATFORM = 'linux/arm64' def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') @@ -730,7 +730,7 @@ class ViewControllerTest extends Specification { def 'should find all scans' () { given: def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/arm64') + def PLATFORM = 'linux/arm64' def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') diff --git a/src/test/groovy/io/seqera/wave/core/ChildRefsTest.groovy b/src/test/groovy/io/seqera/wave/core/ChildRefsTest.groovy new file mode 100644 index 0000000000..c9941a1ff8 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/core/ChildRefsTest.groovy @@ -0,0 +1,271 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.core + +import com.squareup.moshi.Moshi +import spock.lang.Specification + +/** + * Tests for {@link ChildRefs} + * + * @author Paolo Di Tommaso + */ +class ChildRefsTest extends Specification { + + def 'should create from map with single entry'() { + when: + def result = ChildRefs.of(['sc-abc_1': 'linux/amd64']) + then: + result.size() == 1 + result[0].id == 'sc-abc_1' + result[0].value == 'linux/amd64' + } + + def 'should create from map with multiple entries'() { + given: + def map = new LinkedHashMap() + map.put('sc-abc_1', 'linux/amd64') + map.put('sc-def_2', 'linux/arm64') + + when: + def result = ChildRefs.of(map) + then: + result.size() == 2 + result[0].id == 'sc-abc_1' + result[0].value == 'linux/amd64' + result[1].id == 'sc-def_2' + result[1].value == 'linux/arm64' + } + + def 'should return null from null or empty map'() { + expect: + ChildRefs.of(null) == null + ChildRefs.of([:]) == null + } + + def 'should create from list of entries'() { + when: + def result = new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') + ]) + then: + result.size() == 2 + result[0].id == 'sc-abc_1' + result[0].value == 'linux/amd64' + result[1].id == 'sc-def_2' + result[1].value == 'linux/arm64' + } + + def 'should get primary id'() { + expect: + new ChildRefs([new ChildRefs.Ref('sc-abc_1', null)]).primary() == 'sc-abc_1' + and: + new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') + ]).primary() == 'sc-abc_1' + and: + new ChildRefs([]).primary() == null + } + + def 'should get all ids'() { + expect: + new ChildRefs([new ChildRefs.Ref('sc-abc_1', null)]).allIds() == ['sc-abc_1'] + and: + new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') + ]).allIds() == ['sc-abc_1', 'sc-def_2'] + } + + def 'should roundtrip via of'() { + given: + def map = new LinkedHashMap() + map.put('sc-abc_1', 'linux/amd64') + map.put('sc-def_2', 'linux/arm64') + + when: + def result = ChildRefs.of(map) + + then: + result.size() == 2 + result[0].id == 'sc-abc_1' + result[0].value == 'linux/amd64' + result[1].id == 'sc-def_2' + result[1].value == 'linux/arm64' + } + + def 'should support groovy truth'() { + expect: + new ChildRefs([new ChildRefs.Ref('sc-abc_1', null)]) as boolean == true + new ChildRefs(null) as boolean == false + new ChildRefs([]) as boolean == false + } + + def 'should support equality'() { + expect: + new ChildRefs([new ChildRefs.Ref('sc-abc_1', 'linux/amd64')]) == new ChildRefs([new ChildRefs.Ref('sc-abc_1', 'linux/amd64')]) + new ChildRefs([new ChildRefs.Ref('sc-abc_1', null)]) != new ChildRefs([new ChildRefs.Ref('sc-def_2', null)]) + } + + // -- template binding tests -- + + def 'should populate scan binding for single scanId'() { + given: + def binding = new HashMap() + + when: + ChildRefs.populateScanBinding(binding, 'sc-abc_1', null, true, 'https://wave.io') + then: + binding.scan_id == 'sc-abc_1' + binding.scan_url == 'https://wave.io/view/scans/sc-abc_1' + binding.scan_entries == null + } + + def 'should populate scan binding for multi-platform scanChildIds'() { + given: + def binding = new HashMap() + def scanChildIds = new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') + ]) + + when: + ChildRefs.populateScanBinding(binding, null, scanChildIds, true, 'https://wave.io') + then: + binding.scan_id == 'sc-abc_1' + binding.scan_url == null + binding.scan_entries.size() == 2 + binding.scan_entries[0].scan_id == 'sc-abc_1' + binding.scan_entries[0].scan_platform == 'linux/amd64' + binding.scan_entries[0].scan_url == 'https://wave.io/view/scans/sc-abc_1' + binding.scan_entries[1].scan_id == 'sc-def_2' + binding.scan_entries[1].scan_platform == 'linux/arm64' + } + + def 'should populate scan binding when not succeeded'() { + given: + def binding = new HashMap() + def scanChildIds = new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') + ]) + + when: + ChildRefs.populateScanBinding(binding, null, scanChildIds, false, 'https://wave.io') + then: + binding.scan_id == null + binding.scan_url == null + binding.scan_entries == null + } + + def 'should populate scan binding when scanId is null'() { + given: + def binding = new HashMap() + + when: + ChildRefs.populateScanBinding(binding, null, null, true, 'https://wave.io') + then: + binding.scan_id == null + binding.scan_url == null + } + + def 'should populate build binding for child builds'() { + given: + def binding = new HashMap() + def buildChildIds = new ChildRefs([ + new ChildRefs.Ref('bd-abc_0', 'linux/amd64'), + new ChildRefs.Ref('bd-def_0', 'linux/arm64') + ]) + + when: + ChildRefs.populateBuildBinding(binding, buildChildIds, 'https://wave.io') + then: + binding.build_entries.size() == 2 + binding.build_entries[0].build_id == 'bd-abc_0' + binding.build_entries[0].build_platform == 'linux/amd64' + binding.build_entries[0].build_url == 'https://wave.io/view/builds/bd-abc_0' + binding.build_entries[1].build_id == 'bd-def_0' + binding.build_entries[1].build_platform == 'linux/arm64' + binding.build_entries[1].build_url == 'https://wave.io/view/builds/bd-def_0' + } + + def 'should roundtrip through Moshi serialization'() { + given: + def moshi = new Moshi.Builder().build() + def adapter = moshi.adapter(ChildRefs) + def original = new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') + ]) + + when: + def json = adapter.toJson(original) + def restored = adapter.fromJson(json) + + then: + restored == original + restored.size() == 2 + restored[0].id == 'sc-abc_1' + restored[0].value == 'linux/amd64' + restored[1].id == 'sc-def_2' + restored[1].value == 'linux/arm64' + } + + def 'should deserialise from Moshi json string'() { + given: + def moshi = new Moshi.Builder().build() + def adapter = moshi.adapter(ChildRefs) + + expect: + adapter.fromJson(JSON) == EXPECTED + + where: + JSON | EXPECTED + '{"refs":[]}' | new ChildRefs([]) + '{"refs":[{"id":"sc-1","value":"linux/amd64"}]}' | new ChildRefs([new ChildRefs.Ref('sc-1', 'linux/amd64')]) + '{"refs":[{"id":"sc-1","value":"linux/amd64"},{"id":"sc-2","value":"linux/arm64"}]}' | new ChildRefs([new ChildRefs.Ref('sc-1', 'linux/amd64'), new ChildRefs.Ref('sc-2', 'linux/arm64')]) + } + + def 'should serialise to Moshi json string'() { + given: + def moshi = new Moshi.Builder().build() + def adapter = moshi.adapter(ChildRefs) + + expect: + adapter.toJson(INPUT) == EXPECTED + + where: + INPUT | EXPECTED + new ChildRefs([]) | '{"refs":[]}' + new ChildRefs([new ChildRefs.Ref('sc-1', 'linux/amd64')]) | '{"refs":[{"id":"sc-1","value":"linux/amd64"}]}' + new ChildRefs([new ChildRefs.Ref('sc-1', 'linux/amd64'), new ChildRefs.Ref('sc-2', 'linux/arm64')]) | '{"refs":[{"id":"sc-1","value":"linux/amd64"},{"id":"sc-2","value":"linux/arm64"}]}' + } + + def 'should populate build binding when null'() { + given: + def binding = new HashMap() + + when: + ChildRefs.populateBuildBinding(binding, null, 'https://wave.io') + then: + binding.build_entries == null + } +} diff --git a/src/test/groovy/io/seqera/wave/core/ContainerAugmenterTest.groovy b/src/test/groovy/io/seqera/wave/core/ContainerAugmenterTest.groovy index 6dfbe04e53..f1abdb9301 100644 --- a/src/test/groovy/io/seqera/wave/core/ContainerAugmenterTest.groovy +++ b/src/test/groovy/io/seqera/wave/core/ContainerAugmenterTest.groovy @@ -301,7 +301,7 @@ class ContainerAugmenterTest extends Specification { .withContainerConfig(config) when: - def digest = scanner.updateImageIndex(IMAGE, MANIFEST, DIGEST, NEW_DIGEST, NEW_SIZE, false) + def digest = scanner.updateImageIndex(IMAGE, MANIFEST, [(DIGEST): new ContainerAugmenter.AugmentedManifest(NEW_DIGEST, NEW_SIZE)], false) then: def entry = storage.getManifest("$REGISTRY/v2/$IMAGE/manifests/$digest").get() diff --git a/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy b/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy index eb27efc00e..da82d44c47 100644 --- a/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy +++ b/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy @@ -83,7 +83,7 @@ class ContainerPlatformTest extends Specification { @Unroll def 'should match' () { expect: - ContainerPlatform.of(PLATFORM).matches(RECORD) + ContainerPlatform.of(PLATFORM).platforms[0].matches(RECORD) where: PLATFORM | RECORD 'amd64' | [os:'linux', architecture:'amd64'] @@ -106,7 +106,7 @@ class ContainerPlatformTest extends Specification { @Unroll def 'should not match' () { expect: - !ContainerPlatform.of(PLATFORM).matches(RECORD) + !ContainerPlatform.of(PLATFORM).platforms[0].matches(RECORD) where: PLATFORM | RECORD 'amd64' | [os:'linux', architecture:'arm64'] @@ -117,4 +117,104 @@ class ContainerPlatformTest extends Specification { 'linux/arm64/v8'| [os:'linux', architecture:'arm64', variant: 'v9'] } + + def 'should parse multi-arch platform' () { + when: + def platform = ContainerPlatform.of('linux/amd64,linux/arm64') + then: + platform.platforms[0].os == 'linux' + platform.platforms[0].arch == 'amd64' + platform.platforms == [new ContainerPlatform.Platform('linux','amd64'), new ContainerPlatform.Platform('linux','arm64')] + platform.isMultiArch() + platform.toString() == 'linux/amd64,linux/arm64' + } + + def 'should detect single-arch platform' () { + when: + def platform = ContainerPlatform.of('linux/amd64') + then: + platform.os == 'linux' + platform.arch == 'amd64' + platform.platforms == [new ContainerPlatform.Platform('linux','amd64')] + !platform.isMultiArch() + platform.toString() == 'linux/amd64' + } + + def 'should round-trip multi-arch through toString and of' () { + when: + def original = ContainerPlatform.of('linux/amd64,linux/arm64') + def roundTripped = ContainerPlatform.of(original.toString()) + then: + roundTripped == original + roundTripped.isMultiArch() + roundTripped.platforms == [new ContainerPlatform.Platform('linux','amd64'), new ContainerPlatform.Platform('linux','arm64')] + } + + def 'should have MULTI_PLATFORM constant' () { + expect: + ContainerPlatform.MULTI_PLATFORM.isMultiArch() + ContainerPlatform.MULTI_PLATFORM.platforms[0].os == 'linux' + ContainerPlatform.MULTI_PLATFORM.platforms[0].arch == 'amd64' + ContainerPlatform.MULTI_PLATFORM.platforms == [new ContainerPlatform.Platform('linux','amd64'), new ContainerPlatform.Platform('linux','arm64')] + ContainerPlatform.MULTI_PLATFORM.toString() == 'linux/amd64,linux/arm64' + } + + def 'should test equality for multi-arch' () { + expect: + ContainerPlatform.of('linux/amd64,linux/arm64') == ContainerPlatform.MULTI_PLATFORM + ContainerPlatform.of('linux/amd64,linux/arm64') == ContainerPlatform.of('linux/amd64,linux/arm64') + ContainerPlatform.of('linux/amd64') != ContainerPlatform.MULTI_PLATFORM + } + + def 'should validate single platform allows valid values' () { + when: + ContainerPlatform.validateSinglePlatform(VALUE) + then: + noExceptionThrown() + + where: + VALUE | _ + null | _ + '' | _ + 'linux/amd64' | _ + 'linux/arm64' | _ + 'amd64' | _ + } + + def 'should validate single platform rejects multi-platform values' () { + when: + ContainerPlatform.validateSinglePlatform('linux/amd64,linux/arm64') + then: + def e = thrown(BadRequestException) + e.message.contains('Container multi-platform architecture not allowed') + } + + def 'should throw when accessing os/arch/variant on multi-arch platform' () { + given: + def platform = ContainerPlatform.of('linux/amd64,linux/arm64') + + when: + platform.os + then: + thrown(IllegalStateException) + + when: + platform.arch + then: + thrown(IllegalStateException) + + when: + platform.variant + then: + thrown(IllegalStateException) + } + + def 'should allow os/arch/variant on single-arch platform' () { + given: + def platform = ContainerPlatform.of('linux/amd64') + expect: + platform.os == 'linux' + platform.arch == 'amd64' + platform.variant == null + } } diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy index e72ae904d9..19a4b2872e 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy @@ -22,15 +22,19 @@ import spock.lang.Specification import java.nio.file.Path import java.time.Duration +import java.time.Instant import java.time.OffsetDateTime import io.seqera.wave.api.BuildCompression import io.seqera.wave.api.BuildContext import io.seqera.wave.api.ContainerConfig + import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User +import io.seqera.wave.core.ChildRefs import io.seqera.wave.util.ContainerHelper +import io.seqera.serde.moshi.MoshiEncodeStrategy /** * * @author Paolo Di Tommaso @@ -270,6 +274,76 @@ class BuildRequestTest extends Specification { req7.offsetId == 'UTC+2' } + def 'should serialise and deserialise via Moshi'() { + given: + def buildChildIds = new ChildRefs([ + new ChildRefs.Ref('bd-abc_0', 'linux/amd64'), + new ChildRefs.Ref('bd-def_0', 'linux/arm64') + ]) + def scanChildIds = new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') + ]) + def request = BuildRequest.of( + containerId: 'abc123', + containerFile: 'FROM ubuntu', + condaFile: 'samtools=1.0', + workspace: Path.of('/some/workspace'), + targetImage: 'docker.io/wave:abc123', + identity: new PlatformId(new User(id: 1, email: 'foo@user.com')), + platform: ContainerPlatform.of('linux/amd64'), + cacheRepository: 'docker.io/cache', + startTime: Instant.parse('2024-01-15T10:30:00Z'), + ip: '10.20.30.40', + configJson: '{"config":"json"}', + offsetId: '+02:00', + scanId: 'sc-main', + format: BuildFormat.DOCKER, + maxDuration: Duration.ofMinutes(10), + compression: BuildCompression.gzip, + buildId: 'bd-abc123_0', + buildTemplate: 'some-template', + noEmail: true, + buildChildIds: buildChildIds, + scanChildIds: scanChildIds + ) + // use the same encode strategy as BuildStateStoreImpl + def encoder = new MoshiEncodeStrategy() {} + def entry = BuildEntry.create(request) + + when: + def json = encoder.encode(entry) + def restored = encoder.decode(json) + + then: + restored.request.containerId == 'abc123' + restored.request.containerFile == 'FROM ubuntu' + restored.request.condaFile == 'samtools=1.0' + restored.request.targetImage == 'docker.io/wave:abc123' + restored.request.platform == ContainerPlatform.of('linux/amd64') + restored.request.cacheRepository == 'docker.io/cache' + restored.request.ip == '10.20.30.40' + restored.request.configJson == '{"config":"json"}' + restored.request.offsetId == '+02:00' + restored.request.scanId == 'sc-main' + restored.request.format == BuildFormat.DOCKER + restored.request.buildId == 'bd-abc123_0' + restored.request.buildTemplate == 'some-template' + restored.request.noEmail == true + and: + restored.request.buildChildIds.size() == 2 + restored.request.buildChildIds[0].id == 'bd-abc_0' + restored.request.buildChildIds[0].value == 'linux/amd64' + restored.request.buildChildIds[1].id == 'bd-def_0' + restored.request.buildChildIds[1].value == 'linux/arm64' + and: + restored.request.scanChildIds.size() == 2 + restored.request.scanChildIds[0].id == 'sc-abc_1' + restored.request.scanChildIds[0].value == 'linux/amd64' + restored.request.scanChildIds[1].id == 'sc-def_2' + restored.request.scanChildIds[1].value == 'linux/arm64' + } + def 'should parse legacy id' () { expect: BuildRequest.legacyBuildId(BUILD_ID) == EXPECTED diff --git a/src/test/groovy/io/seqera/wave/service/builder/ManifestAssemblerTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/ManifestAssemblerTest.groovy new file mode 100644 index 0000000000..08f33eb2e9 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/builder/ManifestAssemblerTest.groovy @@ -0,0 +1,91 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import spock.lang.Specification + +import groovy.json.JsonSlurper + +import static io.seqera.wave.model.ContentType.OCI_IMAGE_INDEX_V1 +import static io.seqera.wave.model.ContentType.OCI_IMAGE_MANIFEST_V1 + +class ManifestAssemblerTest extends Specification { + + def 'should build OCI image index JSON'() { + given: + def manifests = [ + [ + mediaType: OCI_IMAGE_MANIFEST_V1, + digest: 'sha256:aaa111', + size: 1234, + platform: [architecture: 'amd64', os: 'linux'] + ], + [ + mediaType: OCI_IMAGE_MANIFEST_V1, + digest: 'sha256:bbb222', + size: 5678, + platform: [architecture: 'arm64', os: 'linux'] + ] + ] + + when: + def json = ManifestAssembler.buildImageIndex(manifests) + def parsed = new JsonSlurper().parseText(json) + + then: + parsed.schemaVersion == 2 + parsed.mediaType == OCI_IMAGE_INDEX_V1 + parsed.manifests.size() == 2 + + and: + parsed.manifests[0].mediaType == OCI_IMAGE_MANIFEST_V1 + parsed.manifests[0].digest == 'sha256:aaa111' + parsed.manifests[0].size == 1234 + parsed.manifests[0].platform.architecture == 'amd64' + parsed.manifests[0].platform.os == 'linux' + + and: + parsed.manifests[1].mediaType == OCI_IMAGE_MANIFEST_V1 + parsed.manifests[1].digest == 'sha256:bbb222' + parsed.manifests[1].size == 5678 + parsed.manifests[1].platform.architecture == 'arm64' + parsed.manifests[1].platform.os == 'linux' + } + + def 'should build valid JSON with single manifest'() { + given: + def manifests = [ + [ + mediaType: 'application/vnd.docker.distribution.manifest.v2+json', + digest: 'sha256:deadbeef', + size: 999, + platform: [architecture: 'amd64', os: 'linux'] + ] + ] + + when: + def json = ManifestAssembler.buildImageIndex(manifests) + def parsed = new JsonSlurper().parseText(json) + + then: + parsed.schemaVersion == 2 + parsed.manifests.size() == 1 + parsed.manifests[0].digest == 'sha256:deadbeef' + } +} diff --git a/src/test/groovy/io/seqera/wave/service/builder/MultiBuildEntryTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/MultiBuildEntryTest.groovy new file mode 100644 index 0000000000..afea043360 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/builder/MultiBuildEntryTest.groovy @@ -0,0 +1,112 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import spock.lang.Specification + +import java.time.Duration +import java.time.Instant + +import io.seqera.wave.tower.PlatformId +import io.seqera.wave.tower.User + +class MultiBuildEntryTest extends Specification { + + def 'should create multi-build entry'() { + given: + def identity = new PlatformId(new User(id: 1, email: 'foo@user.com')) + def request = MultiBuildRequest.create( + 'container123', + 'docker.io/wave:multi123', + 'bd-container123_0', + 'docker.io/wave:amd64', + 'docker.io/wave:arm64', + false, + false, + identity, + Duration.ofMinutes(5) + ) + + when: + def entry = MultiBuildEntry.of(request) + + then: + entry.key == 'docker.io/wave:multi123' + entry.requestId == request.multiBuildId + entry.request == request + !entry.done() + entry.succeeded() == null + } + + def 'should update with result'() { + given: + def identity = new PlatformId(new User(id: 1, email: 'foo@user.com')) + def request = MultiBuildRequest.create( + 'container123', + 'docker.io/wave:multi123', + 'bd-container123_0', + 'docker.io/wave:amd64', + 'docker.io/wave:arm64', + false, + false, + identity, + Duration.ofMinutes(5) + ) + + when: + def entry = MultiBuildEntry.of(request) + then: + !entry.done() + + when: + def result = BuildResult.completed('bd-container123_0', 0, 'ok', Instant.now(), null) + entry = entry.withResult(result) + then: + entry.done() + entry.succeeded() + entry.result == result + entry.request == request + } + + def 'should report failure'() { + given: + def identity = new PlatformId(new User(id: 1, email: 'foo@user.com')) + def request = MultiBuildRequest.create( + 'container123', + 'docker.io/wave:multi123', + 'bd-container123_0', + 'docker.io/wave:amd64', + 'docker.io/wave:arm64', + false, + false, + identity, + Duration.ofMinutes(5) + ) + + when: + def entry = MultiBuildEntry.of(request) + def result = BuildResult.failed('bd-container123_0', 'something went wrong', Instant.now()) + entry = entry.withResult(result) + + then: + entry.done() + !entry.succeeded() + entry.result.failed() + } +} diff --git a/src/test/groovy/io/seqera/wave/service/builder/MultiBuildRequestTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/MultiBuildRequestTest.groovy new file mode 100644 index 0000000000..90c0ccf76c --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/builder/MultiBuildRequestTest.groovy @@ -0,0 +1,81 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import spock.lang.Specification + +import java.time.Duration + +import io.seqera.wave.tower.PlatformId +import io.seqera.wave.tower.User + +class MultiBuildRequestTest extends Specification { + + def 'should create multi-build request'() { + given: + def identity = new PlatformId(new User(id: 1, email: 'foo@user.com')) + + when: + def request = MultiBuildRequest.create( + 'container123', + 'docker.io/wave:multi123', + 'bd-container123_0', + 'docker.io/wave:amd64', + 'docker.io/wave:arm64', + false, + false, + identity, + Duration.ofMinutes(5) + ) + + then: + request.multiBuildId.startsWith(MultiBuildRequest.ID_PREFIX) + request.targetImage == 'docker.io/wave:multi123' + request.containerId == 'container123' + request.buildId == 'bd-container123_0' + request.amd64TargetImage == 'docker.io/wave:amd64' + request.arm64TargetImage == 'docker.io/wave:arm64' + request.amd64Cached == false + request.arm64Cached == false + request.identity == identity + request.creationTime != null + request.maxDuration == Duration.ofMinutes(5) + } + + def 'should create from map'() { + when: + def request = MultiBuildRequest.of( + multiBuildId: 'mb-abc123', + targetImage: 'docker.io/wave:multi', + containerId: 'cid', + buildId: 'bd-cid_0', + amd64TargetImage: 'docker.io/wave:amd64', + arm64TargetImage: 'docker.io/wave:arm64', + amd64Cached: true, + arm64Cached: false, + maxDuration: Duration.ofMinutes(10) + ) + + then: + request.multiBuildId == 'mb-abc123' + request.targetImage == 'docker.io/wave:multi' + request.amd64Cached == true + request.arm64Cached == false + } +} diff --git a/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy new file mode 100644 index 0000000000..ee42644f9b --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy @@ -0,0 +1,393 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import spock.lang.Specification + +import java.nio.file.Path +import java.time.Duration +import java.time.Instant + +import io.micronaut.context.event.ApplicationEventPublisher +import io.seqera.wave.api.ContainerConfig +import io.seqera.wave.api.ContainerLayer +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.service.job.JobService +import io.seqera.wave.service.job.JobSpec +import io.seqera.wave.service.job.JobState +import io.seqera.wave.service.persistence.PersistenceService +import io.seqera.wave.service.persistence.WaveBuildRecord +import io.seqera.wave.service.scan.ContainerScanService +import io.seqera.wave.tower.PlatformId +import io.seqera.wave.tower.User + +class MultiPlatformBuildServiceTest extends Specification { + + static final PlatformId TEST_IDENTITY = new PlatformId(new User(id: 1, email: 'foo@user.com')) + + private BuildEntry makeExistingBuildEntry() { + BuildEntry.create(BuildRequest.of( + buildId: 'bd-cid_0', + containerId: 'cid', + targetImage: 'docker.io/wave:multi', + startTime: Instant.now(), + identity: TEST_IDENTITY, + platform: ContainerPlatform.MULTI_PLATFORM + )) + } + + def 'should create platform-specific build requests with noEmail flag'() { + given: + def service = new MultiPlatformBuildService() + def template = BuildRequest.of( + containerId: 'abc123', + containerFile: 'FROM ubuntu:latest', + condaFile: null, + workspace: Path.of('/tmp'), + targetImage: 'docker.io/wave:abc123', + identity: TEST_IDENTITY, + platform: ContainerPlatform.of('linux/amd64'), + cacheRepository: 'docker.io/cache', + ip: '10.0.0.1', + configJson: '{}', + offsetId: '+0', + format: BuildFormat.DOCKER, + maxDuration: Duration.ofMinutes(5), + buildTemplate: null + ) + + when: + def amd64Req = service.createPlatformRequest(template, MultiPlatformBuildService.PLATFORM_AMD64, '-linux-amd64') + def arm64Req = service.createPlatformRequest(template, MultiPlatformBuildService.PLATFORM_ARM64, '-linux-arm64') + + then: + amd64Req.platform == ContainerPlatform.of('linux/amd64') + arm64Req.platform == ContainerPlatform.of('linux/arm64') + amd64Req.containerFile == 'FROM ubuntu:latest' + arm64Req.containerFile == 'FROM ubuntu:latest' + amd64Req.containerId != arm64Req.containerId + amd64Req.targetImage != arm64Req.targetImage + amd64Req.cacheRepository == 'docker.io/cache' + arm64Req.cacheRepository == 'docker.io/cache' + and: 'sub-builds should have noEmail flag set' + amd64Req.noEmail == true + arm64Req.noEmail == true + and: 'sub-builds should not trigger individual scans' + amd64Req.scanId == null + arm64Req.scanId == null + } + + def 'should return in-progress build track and launch multi-build job'() { + given: + def buildService = Mock(ContainerBuildService) + def buildStore = Mock(BuildStateStore) + def multiBuildStore = Mock(MultiBuildStateStore) + def manifestAssembler = Mock(ManifestAssembler) + def jobService = Mock(JobService) + + def service = new MultiPlatformBuildService( + buildService: buildService, + buildStore: buildStore, + multiBuildStore: multiBuildStore, + manifestAssembler: manifestAssembler, + jobService: jobService + ) + + def template = BuildRequest.of( + containerId: 'abc123', + containerFile: 'FROM ubuntu:latest', + workspace: Path.of('/tmp'), + targetImage: 'docker.io/wave:abc123', + identity: TEST_IDENTITY, + platform: ContainerPlatform.of('linux/amd64'), + format: BuildFormat.DOCKER, + maxDuration: Duration.ofMinutes(5) + ) + + and: + buildService.buildImage(_) >> new BuildTrack('bd-amd64_0', 'docker.io/wave:amd64', false, null) >> new BuildTrack('bd-arm64_0', 'docker.io/wave:arm64', false, null) + + when: + def track = service.buildMultiPlatformImage(template, 'multi123', 'docker.io/wave:multi123', PlatformId.NULL) + + then: + track.targetImage == 'docker.io/wave:multi123' + track.cached == false + track.succeeded == null + track.id == 'bd-multi123_0' + and: + 1 * buildStore.storeIfAbsent('docker.io/wave:multi123', _) >> true + 1 * multiBuildStore.put('docker.io/wave:multi123', _) + 1 * jobService.launchMultiBuild(_) + } + + def 'should assemble manifest and persist record on job completion with success'() { + given: + def buildStore = Mock(BuildStateStore) + def multiBuildStore = Mock(MultiBuildStateStore) + def manifestAssembler = Mock(ManifestAssembler) + def eventPublisher = Mock(ApplicationEventPublisher) + def persistenceService = Mock(PersistenceService) + def scanService = Mock(ContainerScanService) + + def service = new MultiPlatformBuildService( + buildStore: buildStore, + multiBuildStore: multiBuildStore, + manifestAssembler: manifestAssembler, + eventPublisher: eventPublisher, + persistenceService: persistenceService, + scanService: scanService + ) + + def request = MultiBuildRequest.of( + multiBuildId: 'mb-abc123', + targetImage: 'docker.io/wave:multi', + containerId: 'cid', + buildId: 'bd-cid_0', + amd64TargetImage: 'docker.io/wave:amd64', + arm64TargetImage: 'docker.io/wave:arm64', + amd64Cached: false, + arm64Cached: false, + identity: TEST_IDENTITY, + creationTime: Instant.now(), + maxDuration: Duration.ofMinutes(5) + ) + def entry = MultiBuildEntry.of(request) + def job = JobSpec.multiBuild('docker.io/wave:multi', 'mb-abc123', Instant.now(), Duration.ofMinutes(5)) + def state = JobState.succeeded(null) + + when: + service.onJobCompletion(job, entry, state) + + then: + 1 * manifestAssembler.createAndPushManifestList('docker.io/wave:multi', _, TEST_IDENTITY) + and: + 1 * buildStore.getBuild('docker.io/wave:multi') >> makeExistingBuildEntry() + 1 * buildStore.storeBuild('docker.io/wave:multi', _) + 1 * multiBuildStore.put('docker.io/wave:multi', _) + 1 * scanService.scanOnBuild({ BuildEntry e -> e.request.targetImage == 'docker.io/wave:multi' }) + 1 * persistenceService.saveBuildAsync({ WaveBuildRecord r -> r.platform == 'linux/amd64,linux/arm64' }) + 1 * eventPublisher.publishEvent({ BuildEvent e -> e.request.platform?.isMultiArch() }) + } + + def 'should persist failure record when sub-build fails'() { + given: + def buildStore = Mock(BuildStateStore) + def multiBuildStore = Mock(MultiBuildStateStore) + def manifestAssembler = Mock(ManifestAssembler) + def eventPublisher = Mock(ApplicationEventPublisher) + def persistenceService = Mock(PersistenceService) + + def service = new MultiPlatformBuildService( + buildStore: buildStore, + multiBuildStore: multiBuildStore, + manifestAssembler: manifestAssembler, + eventPublisher: eventPublisher, + persistenceService: persistenceService + ) + + def request = MultiBuildRequest.of( + multiBuildId: 'mb-abc123', + targetImage: 'docker.io/wave:multi', + containerId: 'cid', + buildId: 'bd-cid_0', + amd64TargetImage: 'docker.io/wave:amd64', + arm64TargetImage: 'docker.io/wave:arm64', + amd64Cached: false, + arm64Cached: false, + identity: TEST_IDENTITY, + creationTime: Instant.now(), + maxDuration: Duration.ofMinutes(5) + ) + def entry = MultiBuildEntry.of(request) + def job = JobSpec.multiBuild('docker.io/wave:multi', 'mb-abc123', Instant.now(), Duration.ofMinutes(5)) + def state = JobState.failed(-1, 'sub-build failed') + + when: + service.onJobCompletion(job, entry, state) + + then: + 0 * manifestAssembler.createAndPushManifestList(_, _, _) + and: + 1 * buildStore.getBuild('docker.io/wave:multi') >> makeExistingBuildEntry() + 1 * buildStore.storeBuild('docker.io/wave:multi', _) + 1 * multiBuildStore.put('docker.io/wave:multi', _) + 1 * persistenceService.saveBuildAsync(_) + 1 * eventPublisher.publishEvent(_ as BuildEvent) + } + + def 'should handle job timeout'() { + given: + def buildStore = Mock(BuildStateStore) + def multiBuildStore = Mock(MultiBuildStateStore) + def eventPublisher = Mock(ApplicationEventPublisher) + def persistenceService = Mock(PersistenceService) + + def service = new MultiPlatformBuildService( + buildStore: buildStore, + multiBuildStore: multiBuildStore, + eventPublisher: eventPublisher, + persistenceService: persistenceService + ) + + def request = MultiBuildRequest.of( + multiBuildId: 'mb-abc123', + targetImage: 'docker.io/wave:multi', + containerId: 'cid', + buildId: 'bd-cid_0', + amd64TargetImage: 'docker.io/wave:amd64', + arm64TargetImage: 'docker.io/wave:arm64', + amd64Cached: false, + arm64Cached: false, + creationTime: Instant.now(), + maxDuration: Duration.ofMinutes(5) + ) + def entry = MultiBuildEntry.of(request) + def job = JobSpec.multiBuild('docker.io/wave:multi', 'mb-abc123', Instant.now(), Duration.ofMinutes(5)) + + when: + service.onJobTimeout(job, entry) + + then: + 1 * buildStore.getBuild('docker.io/wave:multi') >> makeExistingBuildEntry() + 1 * buildStore.storeBuild('docker.io/wave:multi', _) + 1 * multiBuildStore.put('docker.io/wave:multi', _) + 1 * persistenceService.saveBuildAsync(_) + 1 * eventPublisher.publishEvent(_ as BuildEvent) + } + + def 'should handle job exception'() { + given: + def buildStore = Mock(BuildStateStore) + def multiBuildStore = Mock(MultiBuildStateStore) + def eventPublisher = Mock(ApplicationEventPublisher) + def persistenceService = Mock(PersistenceService) + + def service = new MultiPlatformBuildService( + buildStore: buildStore, + multiBuildStore: multiBuildStore, + eventPublisher: eventPublisher, + persistenceService: persistenceService + ) + + def request = MultiBuildRequest.of( + multiBuildId: 'mb-abc123', + targetImage: 'docker.io/wave:multi', + containerId: 'cid', + buildId: 'bd-cid_0', + amd64TargetImage: 'docker.io/wave:amd64', + arm64TargetImage: 'docker.io/wave:arm64', + amd64Cached: false, + arm64Cached: false, + creationTime: Instant.now(), + maxDuration: Duration.ofMinutes(5) + ) + def entry = MultiBuildEntry.of(request) + def job = JobSpec.multiBuild('docker.io/wave:multi', 'mb-abc123', Instant.now(), Duration.ofMinutes(5)) + + when: + service.onJobException(job, entry, new RuntimeException('boom')) + + then: + 1 * buildStore.getBuild('docker.io/wave:multi') >> makeExistingBuildEntry() + 1 * buildStore.storeBuild('docker.io/wave:multi', _) + 1 * multiBuildStore.put('docker.io/wave:multi', _) + 1 * persistenceService.saveBuildAsync(_) + 1 * eventPublisher.publishEvent(_ as BuildEvent) + } + + def 'should filter fusion layers for target platform'() { + given: + def amd64Layer = new ContainerLayer( + location: 'https://fusionfs.seqera.io/releases/pkg/2/4/20/fusion-amd64.tar.gz', + gzipDigest: 'sha256:aaa', + gzipSize: 100, + tarDigest: 'sha256:bbb' + ) + def arm64Layer = new ContainerLayer( + location: 'https://fusionfs.seqera.io/releases/pkg/2/4/20/fusion-arm64.tar.gz', + gzipDigest: 'sha256:ccc', + gzipSize: 200, + tarDigest: 'sha256:ddd' + ) + def dataLayer = new ContainerLayer( + location: 'data:H4sIAAAAAAAA/+2STQ6C', + gzipDigest: 'sha256:eee', + gzipSize: 50, + tarDigest: 'sha256:fff' + ) + def config = new ContainerConfig( + ['/usr/bin/fusion'] as List, + null, + ['FUSION_CONFIG_PROFILE=nextflow'] as List, + null, + [dataLayer, amd64Layer, arm64Layer] + ) + + when: 'filtering for amd64' + def amd64Config = MultiPlatformBuildService.filterLayersForPlatform(config, ContainerPlatform.of('linux/amd64')) + then: + amd64Config.layers.size() == 2 + amd64Config.layers[0].location == dataLayer.location + amd64Config.layers[1].location == amd64Layer.location + amd64Config.entrypoint == ['/usr/bin/fusion'] + amd64Config.env == ['FUSION_CONFIG_PROFILE=nextflow'] + + when: 'filtering for arm64' + def arm64Config = MultiPlatformBuildService.filterLayersForPlatform(config, ContainerPlatform.of('linux/arm64')) + then: + arm64Config.layers.size() == 2 + arm64Config.layers[0].location == dataLayer.location + arm64Config.layers[1].location == arm64Layer.location + } + + def 'should return config as-is when no fusion layers'() { + given: + def dataLayer = new ContainerLayer(location: 'data:H4sIAAAAAAAA/+2STQ6C') + def config = new ContainerConfig(null, null, null, null, [dataLayer]) + + when: + def result = MultiPlatformBuildService.filterLayersForPlatform(config, ContainerPlatform.of('linux/amd64')) + + then: + result.layers.size() == 1 + result.layers[0].location == dataLayer.location + } + + def 'should handle null and empty config'() { + expect: + MultiPlatformBuildService.filterLayersForPlatform(null, ContainerPlatform.of('linux/amd64')) == null + MultiPlatformBuildService.filterLayersForPlatform(new ContainerConfig(), ContainerPlatform.of('linux/amd64')).layers == [] + } + + def 'launchJob should be a no-op returning job with launch time'() { + given: + def service = new MultiPlatformBuildService() + def job = JobSpec.multiBuild('docker.io/wave:multi', 'mb-abc123', Instant.now(), Duration.ofMinutes(5)) + def entry = Mock(MultiBuildEntry) + + when: + def result = service.launchJob(job, entry) + + then: + result.id == job.id + result.type == job.type + result.launchTime != null + } +} diff --git a/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy index cc0069ecc1..36310712bc 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy @@ -27,6 +27,7 @@ import java.time.Instant import io.seqera.wave.api.BuildCompression import io.seqera.wave.api.BuildStatusResponse +import io.seqera.wave.core.ChildRefs import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildFormat @@ -56,7 +57,15 @@ class WaveBuildRecordTest extends Specification { scanId: 'scan12345', format: BuildFormat.DOCKER, maxDuration: Duration.ofMinutes(1), - compression: BuildCompression.gzip + compression: BuildCompression.gzip, + buildChildIds: new ChildRefs([ + new ChildRefs.Ref('bd-abc_0', 'linux/amd64'), + new ChildRefs.Ref('bd-def_0', 'linux/arm64') + ]), + scanChildIds: new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') + ]) ) final result = new BuildResult(request.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) final event = new BuildEvent(request, result) @@ -65,7 +74,19 @@ class WaveBuildRecordTest extends Specification { when: def json = JacksonHelper.toJson(record) then: - JacksonHelper.fromJson(json, WaveBuildRecord) == record + def restored = JacksonHelper.fromJson(json, WaveBuildRecord) + restored.buildId == record.buildId + restored.targetImage == record.targetImage + restored.buildChildIds.size() == 2 + restored.buildChildIds[0].id == 'bd-abc_0' + restored.buildChildIds[0].value == 'linux/amd64' + restored.buildChildIds[1].id == 'bd-def_0' + restored.buildChildIds[1].value == 'linux/arm64' + restored.scanChildIds.size() == 2 + restored.scanChildIds[0].id == 'sc-abc_1' + restored.scanChildIds[0].value == 'linux/amd64' + restored.scanChildIds[1].id == 'sc-def_2' + restored.scanChildIds[1].value == 'linux/arm64' } diff --git a/src/test/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentServiceTest.groovy index d9eea73bef..340fe4f5e1 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentServiceTest.groovy @@ -340,7 +340,7 @@ class PostgresPersistentServiceTest extends Specification { def SCAN_ID = 'a1' def BUILD_ID = '100' def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/amd64') + def PLATFORM = 'linux/amd64' def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') @@ -374,7 +374,7 @@ class PostgresPersistentServiceTest extends Specification { def SCAN_ID = 'a1' def BUILD_ID = '100' def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/amd64') + def PLATFORM = 'linux/amd64' def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') def scan = new WaveScanRecord(SCAN_ID, BUILD_ID, null, null, CONTAINER_IMAGE, PLATFORM, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1], null, null, null) @@ -390,7 +390,7 @@ class PostgresPersistentServiceTest extends Specification { def 'should find all scans' () { given: def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/amd64') + def PLATFORM = 'linux/amd64' def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') diff --git a/src/test/groovy/io/seqera/wave/service/request/ContainerRequestTest.groovy b/src/test/groovy/io/seqera/wave/service/request/ContainerRequestTest.groovy index 98852c432a..548cc549e1 100644 --- a/src/test/groovy/io/seqera/wave/service/request/ContainerRequestTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/request/ContainerRequestTest.groovy @@ -67,6 +67,7 @@ class ContainerRequestTest extends Specification { BUILD_NEW, FREEZE, 'scan-1234', + null, // scanChildIds ScanMode.required, List.of(ScanLevel.HIGH), DRY_RUN, diff --git a/src/test/groovy/io/seqera/wave/service/request/ContainerStatusServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/request/ContainerStatusServiceTest.groovy index 2533d9b874..54b0a69002 100644 --- a/src/test/groovy/io/seqera/wave/service/request/ContainerStatusServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/request/ContainerStatusServiceTest.groovy @@ -45,6 +45,7 @@ class ContainerStatusServiceTest extends Specification { def res1 = service.scanResult(request, scan) then: scan.succeeded() >> false + scan.getScanId() >> 'scan-123' request.getScanId() >> 'scan-123' and: res1 == new ContainerStatusServiceImpl.StageResult( @@ -58,6 +59,7 @@ class ContainerStatusServiceTest extends Specification { then: scan.succeeded() >> true scan.summary() >> [MEDIUM: 5] + scan.getScanId() >> 'scan-123' request.getScanId() >> 'scan-123' request.getScanLevels() >> null and: @@ -72,6 +74,7 @@ class ContainerStatusServiceTest extends Specification { then: scan.succeeded() >> true scan.summary() >> [HIGH:2, CRITICAL:1, MEDIUM: 5] + scan.getScanId() >> 'scan-123' request.getScanId() >> 'scan-123' request.getScanLevels() >> [ScanLevel.LOW, ScanLevel.MEDIUM] and: @@ -86,6 +89,7 @@ class ContainerStatusServiceTest extends Specification { then: scan.succeeded() >> true scan.summary() >> [MEDIUM: 5] + scan.getScanId() >> 'scan-123' request.getScanId() >> 'scan-123' request.getScanLevels() >> [ScanLevel.LOW, ScanLevel.MEDIUM] and: @@ -100,6 +104,7 @@ class ContainerStatusServiceTest extends Specification { then: scan.succeeded() >> true scan.summary() >> [:] + scan.getScanId() >> 'scan-123' request.getScanId() >> 'scan-123' request.getScanLevels() >> [ScanLevel.LOW, ScanLevel.MEDIUM] and: @@ -345,7 +350,7 @@ class ContainerStatusServiceTest extends Specification { requestData.scanMode >> ScanMode.required requestData.scanLevels >> List.of() and: - service.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>false } + service.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>false; getScanId()>>'scan-abc' } then: resp.id == requestId resp.status == ContainerStatus.DONE @@ -384,7 +389,7 @@ class ContainerStatusServiceTest extends Specification { requestData.scanMode >> ScanMode.required requestData.scanLevels >> List.of() and: - service.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true; summary()>>[HIGH:1] } + service.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true; summary()>>[HIGH:1]; getScanId()>>'scan-abc' } then: resp.id == requestId resp.status == ContainerStatus.DONE @@ -423,7 +428,7 @@ class ContainerStatusServiceTest extends Specification { requestData.scanMode >> ScanMode.required requestData.scanLevels >> List.of(ScanLevel.MEDIUM,ScanLevel.HIGH) and: - service.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true; summary()>>[HIGH:1] } + service.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true; summary()>>[HIGH:1]; getScanId()>>'scan-abc' } then: resp.id == requestId resp.status == ContainerStatus.DONE @@ -500,7 +505,7 @@ class ContainerStatusServiceTest extends Specification { requestData.scanMode >> ScanMode.required requestData.scanLevels >> List.of() and: - service.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true; summary()>>[HIGH:1]; getStartTime()>>startTime } + service.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true; summary()>>[HIGH:1]; getScanId()>>'scan-abc'; getStartTime()>>startTime } then: resp.id == requestId resp.status == ContainerStatus.DONE diff --git a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy index c43c8e2190..b08dc3931e 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy @@ -29,9 +29,12 @@ import java.time.Instant import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.api.ScanMode import io.seqera.wave.configuration.ScanConfig +import io.seqera.wave.core.ChildRefs import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildFormat import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.builder.BuildResult import io.seqera.wave.service.cleanup.CleanupService import io.seqera.wave.service.inspect.ContainerInspectService import io.seqera.wave.service.job.JobService @@ -432,6 +435,72 @@ class ContainerScanServiceImplTest extends Specification { 'sc-123'| 'bd-123' | false | null | ScanMode.async | true | false | 0 } + def 'should create scan request from build with overridden platform' () { + given: + def scanService = new ContainerScanServiceImpl() + def containerId = 'container1234' + and: + def workspace = Path.of('/some/workspace') + def platform = ContainerPlatform.of('linux/amd64,linux/arm64') + final build = + BuildRequest.of( + containerId: containerId, + containerFile: 'FROM ubuntu', + workspace: workspace, + targetImage: 'docker.io/my/repo:container1234', + identity: PlatformId.NULL, + platform: platform, + configJson: '{"config":"json"}', + scanId: 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64', + format: BuildFormat.DOCKER, + buildId: "${containerId}_1", + ) + + when: + def scan = scanService.fromBuild(build, 'sc-abc_1', ContainerPlatform.of('linux/amd64')) + then: + scan.scanId == 'sc-abc_1' + scan.buildId == build.buildId + scan.configJson == build.configJson + scan.targetImage == build.targetImage + scan.platform == ContainerPlatform.of('linux/amd64') + scan.workDir.startsWith(workspace) + scan.workDir.fileName.toString() == 'sc-abc_1' + } + + def 'should fan out multi-platform scans on build' () { + given: + def scanService = Spy(new ContainerScanServiceImpl()) + def containerId = 'container1234' + def workspace = Path.of('/some/workspace') + def platform = ContainerPlatform.of('linux/amd64,linux/arm64') + def scanChildIds = new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') + ]) + final build = + BuildRequest.of( + containerId: containerId, + containerFile: 'FROM ubuntu', + workspace: workspace, + targetImage: 'docker.io/my/repo:container1234', + identity: PlatformId.NULL, + platform: platform, + configJson: '{"config":"json"}', + scanChildIds: scanChildIds, + format: BuildFormat.DOCKER, + buildId: "${containerId}_1", + ) + + and: + def entry = new BuildEntry(build, BuildResult.completed(build.buildId, 0, '', Instant.now(), null)) + + when: + scanService.scanOnBuild(entry) + then: + 2 * scanService.scan(_) + } + def 'should store scan entry' () { given: def duration = Duration.ofSeconds(1) diff --git a/src/test/groovy/io/seqera/wave/service/scan/ScanEntryTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ScanEntryTest.groovy index 5a343599da..8a082fa0f2 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/ScanEntryTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/ScanEntryTest.groovy @@ -274,7 +274,7 @@ class ScanEntryTest extends Specification { 'mr-12345', 'cr-12345', 'docker.io/some:image', - ContainerPlatform.DEFAULT, + ContainerPlatform.DEFAULT.toString(), Instant.now(), Duration.ofMinutes(1), ScanEntry.SUCCEEDED, @@ -292,7 +292,7 @@ class ScanEntryTest extends Specification { entry.mirrorId == recrd.mirrorId entry.requestId == recrd.requestId entry.containerImage == recrd.containerImage - entry.platform == recrd.platform + entry.platform == ContainerPlatform.DEFAULT entry.startTime == recrd.startTime entry.duration == recrd.duration entry.status == recrd.status diff --git a/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy b/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy index 8ade673627..9dedebe0b9 100644 --- a/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy +++ b/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy @@ -36,6 +36,7 @@ import io.seqera.wave.service.request.ContainerRequest import io.seqera.wave.service.builder.BuildFormat import io.seqera.wave.service.request.TokenData +import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.request.ContainerRequest.Type /** @@ -904,4 +905,30 @@ class ContainerHelperTest extends Specification { e.message == "Unexpected or missing package type 'CONDA' or build template 'invalid-template'" } + def 'should create multi-platform container id distinct from single platform'() { + given: + def containerFile = 'FROM ubuntu:latest' + def repo = 'docker.io/wave' + + when: + def singleId = ContainerHelper.makeContainerId(containerFile, null, ContainerPlatform.of('linux/amd64'), repo, null, null) + def multiId = ContainerHelper.makeMultiPlatformContainerId(containerFile, null, repo, null, null) + + then: + singleId != multiId + } + + def 'should create stable multi-platform container id'() { + given: + def containerFile = 'FROM ubuntu:latest' + def repo = 'docker.io/wave' + + when: + def id1 = ContainerHelper.makeMultiPlatformContainerId(containerFile, null, repo, null, null) + def id2 = ContainerHelper.makeMultiPlatformContainerId(containerFile, null, repo, null, null) + + then: + id1 == id2 + } + } diff --git a/wave-api/src/main/java/io/seqera/wave/api/SubmitContainerTokenRequest.java b/wave-api/src/main/java/io/seqera/wave/api/SubmitContainerTokenRequest.java index 83a5fee015..99d85d4eb2 100644 --- a/wave-api/src/main/java/io/seqera/wave/api/SubmitContainerTokenRequest.java +++ b/wave-api/src/main/java/io/seqera/wave/api/SubmitContainerTokenRequest.java @@ -165,6 +165,7 @@ public class SubmitContainerTokenRequest implements Cloneable { */ public String buildTemplate; + public SubmitContainerTokenRequest copyWith(Map opts) { try { final SubmitContainerTokenRequest copy = (SubmitContainerTokenRequest) this.clone();