From 5201d27eb835d1e73a659ce83aa999b7c5c16197 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 2 Mar 2026 21:04:19 +0100 Subject: [PATCH 01/15] POC 1 Signed-off-by: Paolo Di Tommaso --- .../wave/auth/RegistryAuthService.groovy | 10 + .../wave/auth/RegistryAuthServiceImpl.groovy | 87 ++++++-- .../controller/ContainerController.groovy | 52 ++++- .../io/seqera/wave/proxy/ProxyClient.groovy | 59 +++++ .../service/builder/ManifestAssembler.groovy | 151 +++++++++++++ .../builder/MultiPlatformBuildService.groovy | 211 ++++++++++++++++++ .../seqera/wave/util/ContainerHelper.groovy | 13 ++ .../builder/ManifestAssemblerTest.groovy | 110 +++++++++ .../MultiPlatformBuildServiceTest.groovy | 187 ++++++++++++++++ .../wave/util/ContainerHelperTest.groovy | 27 +++ .../wave/api/SubmitContainerTokenRequest.java | 13 ++ 11 files changed, 896 insertions(+), 24 deletions(-) create mode 100644 src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy create mode 100644 src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/builder/ManifestAssemblerTest.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthService.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthService.groovy index 9e1c01082f..5e689ebb16 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthService.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthService.groovy @@ -58,6 +58,16 @@ interface RegistryAuthService { */ String getAuthorization(String image, RegistryAuth auth, RegistryCredentials creds) throws RegistryUnauthorizedAccessException + /** + * Get the authorization header for push operations (pull,push scope). + * + * @param image The image name for which the authorisation is needed + * @param auth The {@link RegistryAuth} information modelling the target registry + * @param creds The user credentials + * @return The authorization header including the 'Basic' or 'Bearer' prefix + */ + String getAuthorizationForPush(String image, RegistryAuth auth, RegistryCredentials creds) throws RegistryUnauthorizedAccessException + /** * Invalidate a cached authorization token * diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy index 89a6261799..756602230a 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy @@ -110,6 +110,15 @@ class RegistryAuthServiceImpl implements RegistryAuthService { private LoadingCache cacheTokens + private CacheLoader pushLoader = new CacheLoader() { + @Override + String load(CacheKey key) throws Exception { + return getPushToken(key) + } + } + + private LoadingCache cachePushTokens + @Inject private RegistryLookupService lookupService @@ -124,6 +133,13 @@ class RegistryAuthServiceImpl implements RegistryAuthService { .expireAfterAccess(_1_HOUR.toMillis(), TimeUnit.MILLISECONDS) .executor(ioExecutor) .build(loader) + + cachePushTokens = Caffeine + .newBuilder() + .maximumSize(1_000) + .expireAfterAccess(_1_HOUR.toMillis(), TimeUnit.MILLISECONDS) + .executor(ioExecutor) + .build(pushLoader) } /** @@ -215,23 +231,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService { */ @Override String getAuthorization(String image, RegistryAuth auth, RegistryCredentials creds) throws RegistryUnauthorizedAccessException { - if( !auth ) - throw new RegistryUnauthorizedAccessException("Missing authentication credentials") - - if( !auth.type ) - return null - - if( auth.type == RegistryAuth.Type.Bearer ) { - final token = getAuthToken(image, auth, creds) - return "Bearer $token" - } - - if( auth.type == RegistryAuth.Type.Basic ) { - final String basic = creds ? "$creds.username:$creds.password".bytes.encodeBase64() : null - return basic ? "Basic $basic" : null - } - - throw new RegistryUnauthorizedAccessException("Unknown authentication type: $auth.type") + return getAuthorization0(image, auth, creds, false) } /** @@ -242,9 +242,12 @@ class RegistryAuthServiceImpl implements RegistryAuthService { * @return The resulting bearer token to authorise a pull request */ protected String getToken0(CacheKey key) { + return fetchToken(buildLoginUrl(key.auth.realm, key.image, key.auth.service), key.creds) + } + + private String fetchToken(String login, RegistryCredentials creds) { final httpClient = HttpClientFactory.followRedirectsHttpClient() - final login = buildLoginUrl(key.auth.realm, key.image, key.auth.service) - final req = makeRequest(login, key.creds) + final req = makeRequest(login, creds) log.trace "Token request=$req" // retry strategy @@ -272,18 +275,30 @@ class RegistryAuthServiceImpl implements RegistryAuthService { throw new RegistryForwardException("Unexpected response acquiring token for '$login' [${response.statusCode()}]", response) } - String buildLoginUrl(URI realm, String image, String service){ - String result = "${realm}?scope=repository:${image}:pull" - if(service) { + String buildLoginUrl(URI realm, String image, String service, String scope='pull') { + String result = "${realm}?scope=repository:${image}:${scope}" + if( service ) { result += "&service=$service" } return result } protected String getAuthToken(String image, RegistryAuth auth, RegistryCredentials creds) { + return getCachedToken(cacheTokens, image, auth, creds) + } + + protected String getPushToken(CacheKey key) { + return fetchToken(buildLoginUrl(key.auth.realm, key.image, key.auth.service, 'pull,push'), key.creds) + } + + protected String getPushAuthToken(String image, RegistryAuth auth, RegistryCredentials creds) { + return getCachedToken(cachePushTokens, image, auth, creds) + } + + private String getCachedToken(LoadingCache cache, String image, RegistryAuth auth, RegistryCredentials creds) { final key = new CacheKey(image, auth, creds) try { - return cacheTokens.get(key) + return cache.get(key) } catch (CompletionException e) { // this catches the exception thrown in the cache loader lookup @@ -292,6 +307,31 @@ class RegistryAuthServiceImpl implements RegistryAuthService { } } + @Override + String getAuthorizationForPush(String image, RegistryAuth auth, RegistryCredentials creds) throws RegistryUnauthorizedAccessException { + return getAuthorization0(image, auth, creds, true) + } + + private String getAuthorization0(String image, RegistryAuth auth, RegistryCredentials creds, boolean push) { + if( !auth ) + throw new RegistryUnauthorizedAccessException("Missing authentication credentials") + + if( !auth.type ) + return null + + if( auth.type == RegistryAuth.Type.Bearer ) { + final token = push ? getPushAuthToken(image, auth, creds) : getAuthToken(image, auth, creds) + return "Bearer $token" + } + + if( auth.type == RegistryAuth.Type.Basic ) { + final String basic = creds ? "$creds.username:$creds.password".bytes.encodeBase64() : null + return basic ? "Basic $basic" : null + } + + throw new RegistryUnauthorizedAccessException("Unknown authentication type: $auth.type") + } + /** * Invalidate a cached authorization token * @@ -302,6 +342,7 @@ class RegistryAuthServiceImpl implements RegistryAuthService { void invalidateAuthorization(String image, RegistryAuth auth, RegistryCredentials creds) { final key = new CacheKey(image, auth, creds) cacheTokens.invalidate(key) + cachePushTokens.invalidate(key) tokenStore.remove(getStableKey(key)) } diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index 9958a79c5f..3fa468a5a4 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -63,6 +63,7 @@ import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildTrack import io.seqera.wave.service.builder.ContainerBuildService import io.seqera.wave.service.builder.FreezeService +import io.seqera.wave.service.builder.MultiPlatformBuildService import io.seqera.wave.service.inclusion.ContainerInclusionService import io.seqera.wave.service.inspect.ContainerInspectService import io.seqera.wave.service.mirror.ContainerMirrorService @@ -92,6 +93,7 @@ import static io.seqera.wave.util.ContainerHelper.condaFileFromRequest import static io.seqera.wave.util.ContainerHelper.containerFileFromRequest import static io.seqera.wave.util.ContainerHelper.decodeBase64OrFail import static io.seqera.wave.util.ContainerHelper.makeContainerId +import static io.seqera.wave.util.ContainerHelper.makeMultiPlatformContainerId import static io.seqera.wave.util.ContainerHelper.makeResponseV1 import static io.seqera.wave.util.ContainerHelper.makeResponseV2 import static io.seqera.wave.util.ContainerHelper.makeTargetImage @@ -178,6 +180,10 @@ class ContainerController { @Nullable private ContainerScanService scanService + @Inject + @Nullable + private MultiPlatformBuildService multiPlatformBuildService + @PostConstruct private void init() { log.info "Wave server url: $serverUrl; allowAnonymous: $allowAnonymous; tower-endpoint-url: $towerEndpointUrl; default-build-repo: ${buildConfig?.defaultBuildRepository}; default-cache-repo: ${buildConfig?.defaultCacheRepository}; default-public-repo: ${buildConfig?.defaultPublicRepository}" @@ -253,6 +259,16 @@ class ContainerController { if( v2 && req.packages && req.freeze && !validationService.isCustomRepo(req.buildRepository) && !buildConfig.defaultPublicRepository ) throw new BadRequestException("Attribute `buildRepository` must be specified when using freeze mode [3]") + // multi-platform validation + if( req.multiPlatform ) { + if( !req.containerFile && !req.packages ) + throw new BadRequestException("Multi-platform builds require either 'containerFile' or 'packages' attribute") + if( req.formatSingularity() ) + throw new BadRequestException("Multi-platform builds are not supported for Singularity format") + if( !multiPlatformBuildService ) + throw new UnsupportedBuildServiceException() + } + if( v2 && req.packages ) { // generate the container file required to assemble the container final generated = containerFileFromRequest(req) @@ -408,6 +424,27 @@ class ContainerController { } } + protected BuildTrack checkMultiPlatformBuild(BuildRequest templateBuild, SubmitContainerTokenRequest req, PlatformId identity, String ip) { + final containerSpec = templateBuild.containerFile + final condaContent = templateBuild.condaFile + final buildRepository = ContainerCoordinates.parse(templateBuild.targetImage).repository + + // compute multi-platform container ID and target image + final containerId = makeMultiPlatformContainerId(containerSpec, condaContent, buildRepository, req.buildContext, req.freeze ? req.containerConfig : null) + final targetImage = makeTargetImage(templateBuild.format, buildRepository, containerId, condaContent, req.nameStrategy) + + // check if the multi-platform image already exists + final digest = registryProxyService.getImageDigest(targetImage, identity) + if( digest ) { + log.debug "== Found cached multi-platform build for $targetImage" + final cache = persistenceService.loadBuildSucceed(targetImage, digest) + return new BuildTrack(cache?.buildId, targetImage, true, true) + } + + // delegate to multi-platform build service + return multiPlatformBuildService.buildMultiPlatformImage(templateBuild, containerId, targetImage, identity) + } + protected String getContainerDigest(String containerImage, PlatformId identity) { containerImage ? registryProxyService.getImageDigest(containerImage, identity) @@ -447,7 +484,20 @@ class ContainerController { boolean buildNew String scanId Boolean succeeded - if( req.containerFile ) { + if( req.containerFile && req.multiPlatform ) { + if( !buildService ) throw new UnsupportedBuildServiceException() + final build = makeBuildRequest(req, identity, ip) + final track = checkMultiPlatformBuild(build, req, identity, ip) + targetImage = track.targetImage + targetContent = build.containerFile + condaContent = build.condaFile + buildId = track.id + buildNew = !track.cached + scanId = build.scanId + succeeded = track.succeeded + type = ContainerRequest.Type.Build + } + else if( req.containerFile ) { if( !buildService ) throw new UnsupportedBuildServiceException() final build = makeBuildRequest(req, identity, ip) final track = checkBuild(build, req.dryRun) diff --git a/src/main/groovy/io/seqera/wave/proxy/ProxyClient.groovy b/src/main/groovy/io/seqera/wave/proxy/ProxyClient.groovy index 8027aa3869..c62205b188 100644 --- a/src/main/groovy/io/seqera/wave/proxy/ProxyClient.groovy +++ b/src/main/groovy/io/seqera/wave/proxy/ProxyClient.groovy @@ -61,6 +61,7 @@ class ProxyClient { private ContainerPath route private HttpClientConfig httpConfig private List retryableHttpErrors = HTTP_RETRYABLE_ERRORS + private boolean pushMode ProxyClient(HttpClient httpClient, HttpClientConfig httpConfig) { if( httpClient.followRedirects()!= HttpClient.Redirect.NEVER ) @@ -104,6 +105,11 @@ class ProxyClient { return this } + ProxyClient withPushMode(boolean value) { + this.pushMode = value + return this + } + URI makeUri(String path) { assert path.startsWith('/'), "Request past should start with a slash character — offending path: $path" return URI.create(registry.host.toString() + path) @@ -293,6 +299,59 @@ class ProxyClient { return response } + private String getAuthHeader() { + return pushMode + ? loginService.getAuthorizationForPush(image, registry.auth, credentials) + : loginService.getAuthorization(image, registry.auth, credentials) + } + + HttpResponse put(String path, byte[] body, String contentType, Map> headers=null) { + final uri = makeUri(path) + final retryable = Retryable + .>of(httpConfig) + .retryIf((resp) -> resp.statusCode() in retryableHttpErrors) + .onRetry((event) -> "Failure on PUT request: $uri - event: $event") + return retryable.apply(()-> put0(uri, body, contentType, headers)) + } + + private HttpResponse put0(URI uri, byte[] body, String contentType, Map> headers) { + int authRetry = 0 + while( true ) { + final result = put1(uri, body, contentType, headers) + if( result.statusCode()==401 && (authRetry++)<2 && registry.auth.isRefreshable() ) { + loginService.invalidateAuthorization(image, registry.auth, credentials) + continue + } + return result + } + } + + private HttpResponse put1(URI uri, byte[] body, String contentType, Map> headers) { + try { + final builder = HttpRequest.newBuilder(uri) + .PUT(HttpRequest.BodyPublishers.ofByteArray(body)) + if( contentType ) + builder.setHeader("Content-Type", contentType) + copyHeaders(headers, builder) + final header = getAuthHeader() + if( header ) + builder.setHeader("Authorization", header) + final request = builder.build() + final response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()) + traceResponse(response) + return response + } + catch (IOException e) { + throw e + } + catch (RegistryForwardException | RegistryUnauthorizedAccessException e) { + throw e + } + catch (Exception e) { + throw new InternalServerException("Unexpected error on HTTP PUT request '$uri'", e) + } + } + private void traceResponse(HttpResponse resp) { // dump response if( !log.isTraceEnabled() || !resp ) diff --git a/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy b/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy new file mode 100644 index 0000000000..f01edb4cc0 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy @@ -0,0 +1,151 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.seqera.wave.auth.RegistryAuthService +import io.seqera.wave.auth.RegistryCredentialsProvider +import io.seqera.wave.auth.RegistryLookupService +import io.seqera.wave.configuration.HttpClientConfig +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.core.RegistryProxyService +import io.seqera.wave.core.RoutePath +import io.seqera.wave.http.HttpClientFactory +import io.seqera.wave.model.ContainerCoordinates +import io.seqera.wave.proxy.ProxyClient +import io.seqera.wave.tower.PlatformId +import jakarta.inject.Inject +import jakarta.inject.Singleton +import static io.seqera.wave.WaveDefault.ACCEPT_HEADERS +import static io.seqera.wave.model.ContentType.OCI_IMAGE_INDEX_V1 +import static io.seqera.wave.model.ContentType.OCI_IMAGE_MANIFEST_V1 +/** + * Assembles an OCI Image Index (manifest list) from individual + * platform-specific manifests and pushes it to the registry. + * + * @author Paolo Di Tommaso + */ +@Slf4j +@Singleton +@CompileStatic +class ManifestAssembler { + + @Inject + RegistryLookupService registryLookup + + @Inject + RegistryCredentialsProvider credentialsProvider + + @Inject + RegistryAuthService loginService + + @Inject + HttpClientConfig httpConfig + + /** + * Create and push an OCI Image Index (manifest list) to the registry. + * + * @param targetImage The final tag for the manifest list (e.g. repo:tag) + * @param platformImages List of platform-specific images [amd64-image, arm64-image] + * @param identity The platform identity for credentials lookup + */ + void createAndPushManifestList(String targetImage, List platformEntries, PlatformId identity) { + log.debug "Creating manifest list for targetImage=$targetImage from platformEntries=$platformEntries" + + // 1. Fetch each platform manifest + final manifests = platformEntries.collect { Map entry -> fetchPlatformManifest(entry.image as String, entry.platform as ContainerPlatform, identity) } + + // 2. Build the OCI Image Index JSON + final indexJson = buildImageIndex(manifests) + log.debug "OCI Image Index JSON: $indexJson" + + // 3. Push the index to the registry under the target tag + pushManifest(targetImage, indexJson, identity) + log.info "Successfully pushed manifest list for targetImage=$targetImage" + } + + protected Map fetchPlatformManifest(String image, ContainerPlatform platform, PlatformId identity) { + final coords = ContainerCoordinates.parse(image) + final route = RoutePath.v2manifestPath(coords, identity) + final client = createClient(route, identity, false) + + final resp = client.getString(route.path, ACCEPT_HEADERS) + if( resp.statusCode() != 200 ) + throw new IllegalStateException("Failed to GET manifest for '$image' — status: ${resp.statusCode()}, body: ${resp.body()}") + + final body = resp.body() + final contentType = resp.headers().firstValue('content-type').orElse(OCI_IMAGE_MANIFEST_V1) + final digest = resp.headers().firstValue('docker-content-digest').orElse(null) + final size = body.bytes.length + + return [ + mediaType: contentType, + digest: digest, + size: size, + platform: [architecture: platform.arch, os: platform.os] + ] + } + + static String buildImageIndex(List manifests) { + final index = [ + schemaVersion: 2, + mediaType: OCI_IMAGE_INDEX_V1, + manifests: manifests.collect { Map m -> + [ + mediaType: m.mediaType, + digest: m.digest, + size: m.size, + platform: m.platform + ] + } + ] + return JsonOutput.prettyPrint(JsonOutput.toJson(index)) + } + + protected void pushManifest(String targetImage, String indexJson, PlatformId identity) { + final coords = ContainerCoordinates.parse(targetImage) + final route = RoutePath.v2manifestPath(coords, identity) + final client = createClient(route, identity, true) + + final body = indexJson.bytes + final path = "/v2/${coords.image}/manifests/${coords.reference}" + final resp = client.put(path, body, OCI_IMAGE_INDEX_V1) + + if( resp.statusCode() != 201 && resp.statusCode() != 200 ) { + throw new IllegalStateException("Failed to PUT manifest list for '$targetImage' — status: ${resp.statusCode()}, body: ${resp.body()}") + } + log.debug "PUT manifest list for '$targetImage' — status: ${resp.statusCode()}" + } + + private ProxyClient createClient(RoutePath route, PlatformId identity, boolean pushMode) { + final httpClient = HttpClientFactory.neverRedirectsHttpClient() + final registry = registryLookup.lookup(route.registry) + final creds = credentialsProvider.getCredentials(route, identity) + new ProxyClient(httpClient, httpConfig) + .withRoute(route) + .withImage(route.image) + .withRegistry(registry) + .withCredentials(creds) + .withLoginService(loginService) + .withPushMode(pushMode) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy new file mode 100644 index 0000000000..7c008af8ee --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy @@ -0,0 +1,211 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import java.time.Instant +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ExecutorService + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.micronaut.scheduling.TaskExecutors +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.tower.PlatformId +import jakarta.inject.Inject +import jakarta.inject.Named +import jakarta.inject.Singleton +import static io.seqera.wave.util.ContainerHelper.makeContainerId +import static io.seqera.wave.util.ContainerHelper.makeTargetImage +/** + * Orchestrates multi-platform container builds by fanning out + * two single-platform builds (linux/amd64 + linux/arm64) and + * assembling the results into an OCI Image Index. + * + * @author Paolo Di Tommaso + */ +@Slf4j +@Singleton +@CompileStatic +class MultiPlatformBuildService { + + @Inject + ContainerBuildService buildService + + @Inject + BuildStateStore buildStore + + @Inject + ManifestAssembler manifestAssembler + + @Inject + @Named(TaskExecutors.BLOCKING) + ExecutorService executor + + static final ContainerPlatform PLATFORM_AMD64 = ContainerPlatform.of('linux/amd64') + static final ContainerPlatform PLATFORM_ARM64 = ContainerPlatform.of('linux/arm64') + + /** + * Build a multi-platform image by orchestrating two single-platform builds + * and assembling the result into a manifest list. + * + * @param templateRequest The original build request used as template + * @param finalContainerId The container ID for the multi-platform image + * @param finalTargetImage The final target image tag for the manifest list + * @param identity The platform identity + * @return A BuildTrack with the final target image (succeeded=null means in-progress) + */ + BuildTrack buildMultiPlatformImage(BuildRequest templateRequest, String finalContainerId, String finalTargetImage, PlatformId identity) { + log.debug "Starting multi-platform build: finalTargetImage=$finalTargetImage, finalContainerId=$finalContainerId" + + // Create platform-specific build requests + final amd64Request = createPlatformRequest(templateRequest, PLATFORM_AMD64, '-linux-amd64') + final arm64Request = createPlatformRequest(templateRequest, PLATFORM_ARM64, '-linux-arm64') + + // Submit both platform builds + final amd64Track = buildService.buildImage(amd64Request) + final arm64Track = buildService.buildImage(arm64Request) + + log.debug "Submitted platform builds: amd64=${amd64Track.targetImage} (cached=${amd64Track.cached}), arm64=${arm64Track.targetImage} (cached=${arm64Track.cached})" + + final buildId = BuildRequest.ID_PREFIX + finalContainerId + BuildRequest.SEP + '0' + final startTime = Instant.now() + + // Store an in-progress build entry so status polling can find it + final syntheticRequest = BuildRequest.of( + buildId: buildId, + containerId: finalContainerId, + targetImage: finalTargetImage, + startTime: startTime, + identity: templateRequest.identity, + containerFile: templateRequest.containerFile, + condaFile: templateRequest.condaFile, + platform: templateRequest.platform, + ip: templateRequest.ip, + offsetId: templateRequest.offsetId, + scanId: templateRequest.scanId, + format: templateRequest.format, + compression: templateRequest.compression, + buildTemplate: templateRequest.buildTemplate + ) + final initialEntry = BuildEntry.create(syntheticRequest) + buildStore.storeIfAbsent(finalTargetImage, initialEntry) + + // Async: wait for both builds to complete, then assemble manifest list + CompletableFuture.runAsync({ + try { + awaitAndAssemble(amd64Track, arm64Track, finalTargetImage, buildId, startTime, identity) + } + catch (Exception e) { + log.error "Multi-platform build failed for $finalTargetImage", e + // Store failure so status polling sees completion + final failedResult = BuildResult.failed(buildId, e.message, startTime) + storeBuildResult(finalTargetImage, buildId, failedResult) + } + }, executor) + + // Return immediately — build is in progress (succeeded=null) + return new BuildTrack(buildId, finalTargetImage, false, null) + } + + protected void awaitAndAssemble(BuildTrack amd64Track, BuildTrack arm64Track, String finalTargetImage, String buildId, Instant startTime, PlatformId identity) { + // await both platform builds in parallel + final amd64Future = amd64Track.cached + ? CompletableFuture.completedFuture(true) + : CompletableFuture.supplyAsync({ awaitBuildResult(amd64Track, 'amd64', finalTargetImage) }, executor) + final arm64Future = arm64Track.cached + ? CompletableFuture.completedFuture(true) + : CompletableFuture.supplyAsync({ awaitBuildResult(arm64Track, 'arm64', finalTargetImage) }, executor) + + CompletableFuture.allOf(amd64Future, arm64Future).join() + final boolean amd64Ok = amd64Future.get() + final boolean arm64Ok = arm64Future.get() + + log.debug "Platform build results: amd64=$amd64Ok, arm64=$arm64Ok" + + if( amd64Ok && arm64Ok ) { + final List platformEntries = [ + [image: amd64Track.targetImage, platform: PLATFORM_AMD64], + [image: arm64Track.targetImage, platform: PLATFORM_ARM64] + ] + manifestAssembler.createAndPushManifestList(finalTargetImage, platformEntries, identity) + log.info "Multi-platform manifest list assembled for $finalTargetImage" + // store completed build result + final completedResult = BuildResult.completed(buildId, 0, 'Multi-platform build completed', startTime, null) + storeBuildResult(finalTargetImage, buildId, completedResult) + } + else { + log.error "Multi-platform build failed — amd64Ok=$amd64Ok, arm64Ok=$arm64Ok" + final failedResult = BuildResult.failed(buildId, "Multi-platform build failed — amd64=$amd64Ok, arm64=$arm64Ok", startTime) + storeBuildResult(finalTargetImage, buildId, failedResult) + } + } + + private void storeBuildResult(String targetImage, String buildId, BuildResult result) { + final existing = buildStore.getBuild(targetImage) + if( existing ) { + buildStore.storeBuild(targetImage, existing.withResult(result)) + } + else { + log.warn "Multi-platform build entry not found for $targetImage, buildId=$buildId" + } + } + + private boolean awaitBuildResult(BuildTrack track, String arch, String finalTargetImage) { + final future = buildStore.awaitBuild(track.targetImage) + if( !future ) { + log.error "Multi-platform build: unable to await $arch build for $finalTargetImage" + return false + } + return future.get()?.succeeded() + } + + protected BuildRequest createPlatformRequest(BuildRequest template, ContainerPlatform platform, String suffix) { + final repo = template.targetImage.split(':')[0] + final platformId = makeContainerId( + template.containerFile, + template.condaFile, + platform, + repo, + template.buildContext, + template.containerConfig + ) + final platformImage = makeTargetImage(template.format, repo, platformId, template.condaFile, null) + + return new BuildRequest( + platformId, + template.containerFile, + template.condaFile, + template.workspace, + platformImage, + template.identity, + platform, + template.cacheRepository, + template.ip, + template.configJson, + template.offsetId, + template.containerConfig, + template.scanId, + template.buildContext, + template.format, + template.maxDuration, + template.compression, + template.buildTemplate + ) + } +} diff --git a/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy b/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy index ed0b55609e..6a071b2a82 100644 --- a/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy +++ b/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy @@ -298,6 +298,19 @@ class ContainerHelper { return RegHelper.sipHash(attrs) } + static final String MULTI_PLATFORM = 'linux/amd64,linux/arm64' + + static String makeMultiPlatformContainerId(String containerFile, String condaFile, String repository, BuildContext buildContext, ContainerConfig containerConfig) { + final attrs = new LinkedHashMap(10) + attrs.containerFile = containerFile + attrs.condaFile = condaFile + attrs.platform = MULTI_PLATFORM + attrs.repository = repository + if( buildContext ) attrs.buildContext = buildContext.tarDigest + if( containerConfig ) attrs.containerConfig = String.valueOf(containerConfig.hashCode()) + return RegHelper.sipHash(attrs) + } + static void checkContainerSpec(String file) { if( !file ) return diff --git a/src/test/groovy/io/seqera/wave/service/builder/ManifestAssemblerTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/ManifestAssemblerTest.groovy new file mode 100644 index 0000000000..6db0ce50f7 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/builder/ManifestAssemblerTest.groovy @@ -0,0 +1,110 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import spock.lang.Specification + +import groovy.json.JsonSlurper + +import static io.seqera.wave.model.ContentType.OCI_IMAGE_INDEX_V1 +import static io.seqera.wave.model.ContentType.OCI_IMAGE_MANIFEST_V1 + +class ManifestAssemblerTest extends Specification { + + def 'should build OCI image index JSON'() { + given: + def manifests = [ + [ + mediaType: OCI_IMAGE_MANIFEST_V1, + digest: 'sha256:aaa111', + size: 1234, + platform: [architecture: 'amd64', os: 'linux'] + ], + [ + mediaType: OCI_IMAGE_MANIFEST_V1, + digest: 'sha256:bbb222', + size: 5678, + platform: [architecture: 'arm64', os: 'linux'] + ] + ] + + when: + def json = ManifestAssembler.buildImageIndex(manifests) + def parsed = new JsonSlurper().parseText(json) + + then: + parsed.schemaVersion == 2 + parsed.mediaType == OCI_IMAGE_INDEX_V1 + parsed.manifests.size() == 2 + + and: + parsed.manifests[0].mediaType == OCI_IMAGE_MANIFEST_V1 + parsed.manifests[0].digest == 'sha256:aaa111' + parsed.manifests[0].size == 1234 + parsed.manifests[0].platform.architecture == 'amd64' + parsed.manifests[0].platform.os == 'linux' + + and: + parsed.manifests[1].mediaType == OCI_IMAGE_MANIFEST_V1 + parsed.manifests[1].digest == 'sha256:bbb222' + parsed.manifests[1].size == 5678 + parsed.manifests[1].platform.architecture == 'arm64' + parsed.manifests[1].platform.os == 'linux' + } + + def 'should extract platform from image name'() { + expect: + ManifestAssembler.extractPlatform([:], IMAGE) == EXPECTED + + where: + IMAGE | EXPECTED + 'repo:abc123-linux-amd64' | [architecture: 'amd64', os: 'linux'] + 'docker.io/wave:tag-linux-arm64' | [architecture: 'arm64', os: 'linux'] + 'quay.io/repo:foo-linux-amd64-suffix' | [architecture: 'amd64', os: 'linux'] + } + + def 'should fail to extract platform from unknown image name'() { + when: + ManifestAssembler.extractPlatform([:], 'repo:some-tag') + + then: + thrown(IllegalStateException) + } + + def 'should build valid JSON with single manifest'() { + given: + def manifests = [ + [ + mediaType: 'application/vnd.docker.distribution.manifest.v2+json', + digest: 'sha256:deadbeef', + size: 999, + platform: [architecture: 'amd64', os: 'linux'] + ] + ] + + when: + def json = ManifestAssembler.buildImageIndex(manifests) + def parsed = new JsonSlurper().parseText(json) + + then: + parsed.schemaVersion == 2 + parsed.manifests.size() == 1 + parsed.manifests[0].digest == 'sha256:deadbeef' + } +} diff --git a/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy new file mode 100644 index 0000000000..85d8180dd3 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy @@ -0,0 +1,187 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import spock.lang.Specification + +import java.nio.file.Path +import java.time.Duration +import java.time.Instant +import java.util.concurrent.CompletableFuture +import java.util.concurrent.Executors + +import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.tower.PlatformId +import io.seqera.wave.tower.User + +class MultiPlatformBuildServiceTest extends Specification { + + def 'should create platform-specific build requests'() { + given: + def service = new MultiPlatformBuildService() + def template = BuildRequest.of( + containerId: 'abc123', + containerFile: 'FROM ubuntu:latest', + condaFile: null, + workspace: Path.of('/tmp'), + targetImage: 'docker.io/wave:abc123', + identity: new PlatformId(new User(id: 1, email: 'foo@user.com')), + platform: ContainerPlatform.of('linux/amd64'), + cacheRepository: 'docker.io/cache', + ip: '10.0.0.1', + configJson: '{}', + offsetId: '+0', + format: BuildFormat.DOCKER, + maxDuration: Duration.ofMinutes(5), + buildTemplate: null + ) + + when: + def amd64Req = service.createPlatformRequest(template, MultiPlatformBuildService.PLATFORM_AMD64, '-linux-amd64') + def arm64Req = service.createPlatformRequest(template, MultiPlatformBuildService.PLATFORM_ARM64, '-linux-arm64') + + then: + amd64Req.platform == ContainerPlatform.of('linux/amd64') + arm64Req.platform == ContainerPlatform.of('linux/arm64') + amd64Req.containerFile == 'FROM ubuntu:latest' + arm64Req.containerFile == 'FROM ubuntu:latest' + amd64Req.containerId != arm64Req.containerId + amd64Req.targetImage != arm64Req.targetImage + amd64Req.cacheRepository == 'docker.io/cache' + arm64Req.cacheRepository == 'docker.io/cache' + } + + def 'should return in-progress build track'() { + given: + def buildService = Mock(ContainerBuildService) + def buildStore = Mock(BuildStateStore) + def manifestAssembler = Mock(ManifestAssembler) + + def service = new MultiPlatformBuildService( + buildService: buildService, + buildStore: buildStore, + manifestAssembler: manifestAssembler, + executor: Executors.newSingleThreadExecutor() + ) + + def template = BuildRequest.of( + containerId: 'abc123', + containerFile: 'FROM ubuntu:latest', + workspace: Path.of('/tmp'), + targetImage: 'docker.io/wave:abc123', + identity: new PlatformId(new User(id: 1, email: 'foo@user.com')), + platform: ContainerPlatform.of('linux/amd64'), + format: BuildFormat.DOCKER, + maxDuration: Duration.ofMinutes(5) + ) + + and: + buildService.buildImage(_) >> new BuildTrack('bd-amd64_0', 'docker.io/wave:amd64', false, null) >> new BuildTrack('bd-arm64_0', 'docker.io/wave:arm64', false, null) + + when: + def track = service.buildMultiPlatformImage(template, 'multi123', 'docker.io/wave:multi123', PlatformId.NULL) + + then: + track.targetImage == 'docker.io/wave:multi123' + track.cached == false + track.succeeded == null + track.id == 'bd-multi123_0' + } + + def 'should assemble manifest list on both builds succeeding'() { + given: + def buildStore = Mock(BuildStateStore) + def manifestAssembler = Mock(ManifestAssembler) + + def service = new MultiPlatformBuildService( + buildStore: buildStore, + manifestAssembler: manifestAssembler, + executor: Executors.newSingleThreadExecutor() + ) + + def amd64Track = new BuildTrack('bd-amd64_0', 'docker.io/wave:amd64', false, null) + def arm64Track = new BuildTrack('bd-arm64_0', 'docker.io/wave:arm64', false, null) + def identity = PlatformId.NULL + + and: + def now = Instant.now() + def amd64Result = BuildResult.completed('bd-amd64_0', 0, 'ok', now, 'sha256:aaa') + def arm64Result = BuildResult.completed('bd-arm64_0', 0, 'ok', now, 'sha256:bbb') + buildStore.awaitBuild('docker.io/wave:amd64') >> CompletableFuture.completedFuture(amd64Result) + buildStore.awaitBuild('docker.io/wave:arm64') >> CompletableFuture.completedFuture(arm64Result) + + when: + service.awaitAndAssemble(amd64Track, arm64Track, 'docker.io/wave:final', identity) + + then: + 1 * manifestAssembler.createAndPushManifestList('docker.io/wave:final', ['docker.io/wave:amd64', 'docker.io/wave:arm64'], identity) + } + + def 'should not assemble manifest list when a build fails'() { + given: + def buildStore = Mock(BuildStateStore) + def manifestAssembler = Mock(ManifestAssembler) + + def service = new MultiPlatformBuildService( + buildStore: buildStore, + manifestAssembler: manifestAssembler, + executor: Executors.newSingleThreadExecutor() + ) + + def amd64Track = new BuildTrack('bd-amd64_0', 'docker.io/wave:amd64', false, null) + def arm64Track = new BuildTrack('bd-arm64_0', 'docker.io/wave:arm64', false, null) + def identity = PlatformId.NULL + + and: + def now = Instant.now() + def amd64Result = BuildResult.completed('bd-amd64_0', 0, 'ok', now, 'sha256:aaa') + def arm64Result = BuildResult.failed('bd-arm64_0', 'error', now) + buildStore.awaitBuild('docker.io/wave:amd64') >> CompletableFuture.completedFuture(amd64Result) + buildStore.awaitBuild('docker.io/wave:arm64') >> CompletableFuture.completedFuture(arm64Result) + + when: + service.awaitAndAssemble(amd64Track, arm64Track, 'docker.io/wave:final', identity) + + then: + 0 * manifestAssembler.createAndPushManifestList(_, _, _) + } + + def 'should handle cached platform builds'() { + given: + def buildStore = Mock(BuildStateStore) + def manifestAssembler = Mock(ManifestAssembler) + + def service = new MultiPlatformBuildService( + buildStore: buildStore, + manifestAssembler: manifestAssembler, + executor: Executors.newSingleThreadExecutor() + ) + + def amd64Track = new BuildTrack('bd-amd64_0', 'docker.io/wave:amd64', true, true) + def arm64Track = new BuildTrack('bd-arm64_0', 'docker.io/wave:arm64', true, true) + def identity = PlatformId.NULL + + when: + service.awaitAndAssemble(amd64Track, arm64Track, 'docker.io/wave:final', identity) + + then: + 0 * buildStore.awaitBuild(_) + 1 * manifestAssembler.createAndPushManifestList('docker.io/wave:final', ['docker.io/wave:amd64', 'docker.io/wave:arm64'], identity) + } +} diff --git a/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy b/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy index 78a596ce8b..75e1cc5c5e 100644 --- a/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy +++ b/src/test/groovy/io/seqera/wave/util/ContainerHelperTest.groovy @@ -36,6 +36,7 @@ import io.seqera.wave.service.request.ContainerRequest import io.seqera.wave.service.builder.BuildFormat import io.seqera.wave.service.request.TokenData +import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.request.ContainerRequest.Type /** @@ -896,4 +897,30 @@ class ContainerHelperTest extends Specification { e.message == "Unexpected or missing package type 'CONDA' or build template 'invalid-template'" } + def 'should create multi-platform container id distinct from single platform'() { + given: + def containerFile = 'FROM ubuntu:latest' + def repo = 'docker.io/wave' + + when: + def singleId = ContainerHelper.makeContainerId(containerFile, null, ContainerPlatform.of('linux/amd64'), repo, null, null) + def multiId = ContainerHelper.makeMultiPlatformContainerId(containerFile, null, repo, null, null) + + then: + singleId != multiId + } + + def 'should create stable multi-platform container id'() { + given: + def containerFile = 'FROM ubuntu:latest' + def repo = 'docker.io/wave' + + when: + def id1 = ContainerHelper.makeMultiPlatformContainerId(containerFile, null, repo, null, null) + def id2 = ContainerHelper.makeMultiPlatformContainerId(containerFile, null, repo, null, null) + + then: + id1 == id2 + } + } diff --git a/wave-api/src/main/java/io/seqera/wave/api/SubmitContainerTokenRequest.java b/wave-api/src/main/java/io/seqera/wave/api/SubmitContainerTokenRequest.java index 83a5fee015..05c29ce42a 100644 --- a/wave-api/src/main/java/io/seqera/wave/api/SubmitContainerTokenRequest.java +++ b/wave-api/src/main/java/io/seqera/wave/api/SubmitContainerTokenRequest.java @@ -165,6 +165,11 @@ public class SubmitContainerTokenRequest implements Cloneable { */ public String buildTemplate; + /** + * When {@code true}, build a multi-platform (linux/amd64 + linux/arm64) container image + */ + public boolean multiPlatform; + public SubmitContainerTokenRequest copyWith(Map opts) { try { final SubmitContainerTokenRequest copy = (SubmitContainerTokenRequest) this.clone(); @@ -220,6 +225,8 @@ public SubmitContainerTokenRequest copyWith(Map opts) { copy.buildCompression = (BuildCompression) opts.get("buildCompression"); if( opts.containsKey("buildTemplate")) copy.buildTemplate = (String) opts.get("buildTemplate"); + if( opts.containsKey("multiPlatform")) + copy.multiPlatform = (boolean) opts.get("multiPlatform"); // done return copy; } @@ -365,6 +372,11 @@ public SubmitContainerTokenRequest withBuildTemplate(String template) { return this; } + public SubmitContainerTokenRequest withMultiPlatform(boolean value) { + this.multiPlatform = value; + return this; + } + public boolean formatSingularity() { return "sif".equals(format); } @@ -398,6 +410,7 @@ public String toString() { ", scanLevels=" + scanLevels + ", buildCompression=" + buildCompression + ", buildTemplate=" + buildTemplate + + ", multiPlatform=" + multiPlatform + '}'; } } From 6a415c44942a485da979dc2d0889ee66ce46915e Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 2 Mar 2026 21:53:12 +0100 Subject: [PATCH 02/15] POC 2 Signed-off-by: Paolo Di Tommaso --- .../service/builder/MultiBuildEntry.groovy | 78 +++++++ .../service/builder/MultiBuildRequest.groovy | 98 +++++++++ .../builder/MultiBuildStateStore.groovy | 58 +++++ .../builder/MultiPlatformBuildService.groovy | 151 ++++++++----- .../service/cleanup/CleanupServiceImpl.groovy | 6 +- .../seqera/wave/service/job/JobFactory.groovy | 11 + .../seqera/wave/service/job/JobService.groovy | 3 + .../wave/service/job/JobServiceImpl.groovy | 59 ++++++ .../io/seqera/wave/service/job/JobSpec.groovy | 15 +- .../builder/MultiBuildEntryTest.groovy | 112 ++++++++++ .../builder/MultiBuildRequestTest.groovy | 81 +++++++ .../MultiPlatformBuildServiceTest.groovy | 198 ++++++++++++++---- 12 files changed, 771 insertions(+), 99 deletions(-) create mode 100644 src/main/groovy/io/seqera/wave/service/builder/MultiBuildEntry.groovy create mode 100644 src/main/groovy/io/seqera/wave/service/builder/MultiBuildRequest.groovy create mode 100644 src/main/groovy/io/seqera/wave/service/builder/MultiBuildStateStore.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/builder/MultiBuildEntryTest.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/builder/MultiBuildRequestTest.groovy diff --git a/src/main/groovy/io/seqera/wave/service/builder/MultiBuildEntry.groovy b/src/main/groovy/io/seqera/wave/service/builder/MultiBuildEntry.groovy new file mode 100644 index 0000000000..01e3bdc5de --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/builder/MultiBuildEntry.groovy @@ -0,0 +1,78 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString +import io.seqera.wave.service.job.JobEntry +import io.seqera.data.store.state.RequestIdAware +import io.seqera.data.store.state.StateEntry +/** + * Model a multi-platform build entry object + * + * @author Paolo Di Tommaso + */ +@ToString(includeNames = true, includePackage = false) +@EqualsAndHashCode +@CompileStatic +class MultiBuildEntry implements StateEntry, JobEntry, RequestIdAware { + + final MultiBuildRequest request + + final BuildResult result + + protected MultiBuildEntry() {} + + MultiBuildEntry(MultiBuildRequest request, BuildResult result) { + this.request = request + this.result = result + } + + MultiBuildRequest getRequest() { request } + + BuildResult getResult() { result } + + @Override + String getKey() { + return request.targetImage + } + + @Override + String getRequestId() { + return request.multiBuildId + } + + @Override + boolean done() { + result?.done() + } + + Boolean succeeded() { + return done() ? result.succeeded() : null + } + + MultiBuildEntry withResult(BuildResult result) { + new MultiBuildEntry(this.request, result) + } + + static MultiBuildEntry of(MultiBuildRequest request) { + new MultiBuildEntry(request, BuildResult.create(request.buildId)) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/builder/MultiBuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/MultiBuildRequest.groovy new file mode 100644 index 0000000000..c04a0b7b72 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/builder/MultiBuildRequest.groovy @@ -0,0 +1,98 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import java.time.Duration +import java.time.Instant + +import groovy.transform.Canonical +import groovy.transform.CompileStatic +import groovy.transform.ToString +import io.seqera.random.LongRndKey +import io.seqera.wave.tower.PlatformId +/** + * Model a multi-platform build request + * + * @author Paolo Di Tommaso + */ +@ToString(includeNames = true, includePackage = false) +@Canonical +@CompileStatic +class MultiBuildRequest { + + static final String ID_PREFIX = 'mb-' + + final String multiBuildId + final String targetImage + final String containerId + final String buildId + final String amd64TargetImage + final String arm64TargetImage + final boolean amd64Cached + final boolean arm64Cached + final PlatformId identity + final Instant creationTime + final Duration maxDuration + + static MultiBuildRequest create( + String containerId, + String targetImage, + String buildId, + String amd64TargetImage, + String arm64TargetImage, + boolean amd64Cached, + boolean arm64Cached, + PlatformId identity, + Duration maxDuration + ) { + assert targetImage, "Argument 'targetImage' cannot be empty" + assert buildId, "Argument 'buildId' cannot be empty" + + final multiBuildId = ID_PREFIX + LongRndKey.rndHex() + return new MultiBuildRequest( + multiBuildId, + targetImage, + containerId, + buildId, + amd64TargetImage, + arm64TargetImage, + amd64Cached, + arm64Cached, + identity, + Instant.now(), + maxDuration + ) + } + + static MultiBuildRequest of(Map opts) { + new MultiBuildRequest( + opts.multiBuildId as String, + opts.targetImage as String, + opts.containerId as String, + opts.buildId as String, + opts.amd64TargetImage as String, + opts.arm64TargetImage as String, + opts.amd64Cached as boolean, + opts.arm64Cached as boolean, + opts.identity as PlatformId, + opts.creationTime as Instant, + opts.maxDuration as Duration + ) + } +} diff --git a/src/main/groovy/io/seqera/wave/service/builder/MultiBuildStateStore.groovy b/src/main/groovy/io/seqera/wave/service/builder/MultiBuildStateStore.groovy new file mode 100644 index 0000000000..635f1e6140 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/builder/MultiBuildStateStore.groovy @@ -0,0 +1,58 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import java.time.Duration + +import groovy.transform.CompileStatic +import io.micronaut.context.annotation.Requires +import io.seqera.wave.configuration.BuildConfig +import io.seqera.wave.configuration.BuildEnabled +import io.seqera.serde.moshi.MoshiEncodeStrategy +import io.seqera.data.store.state.AbstractStateStore +import io.seqera.data.store.state.impl.StateProvider +import jakarta.inject.Inject +import jakarta.inject.Singleton +/** + * Implement a {@link io.seqera.data.store.state.StateStore} for {@link MultiBuildEntry} objects + * + * @author Paolo Di Tommaso + */ +@Singleton +@Requires(bean = BuildEnabled) +@CompileStatic +class MultiBuildStateStore extends AbstractStateStore { + + @Inject + private BuildConfig config + + MultiBuildStateStore(StateProvider provider) { + super(provider, new MoshiEncodeStrategy() {}) + } + + @Override + protected String getPrefix() { + return 'wave-multibuild/v1' + } + + @Override + protected Duration getDuration() { + return config.statusDuration + } +} diff --git a/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy index 7c008af8ee..10f2678198 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy @@ -19,13 +19,15 @@ package io.seqera.wave.service.builder import java.time.Instant -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ExecutorService import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.service.job.JobEntry +import io.seqera.wave.service.job.JobHandler +import io.seqera.wave.service.job.JobService +import io.seqera.wave.service.job.JobSpec +import io.seqera.wave.service.job.JobState import io.seqera.wave.tower.PlatformId import jakarta.inject.Inject import jakarta.inject.Named @@ -41,8 +43,9 @@ import static io.seqera.wave.util.ContainerHelper.makeTargetImage */ @Slf4j @Singleton +@Named('MultiBuild') @CompileStatic -class MultiPlatformBuildService { +class MultiPlatformBuildService implements JobHandler { @Inject ContainerBuildService buildService @@ -50,12 +53,14 @@ class MultiPlatformBuildService { @Inject BuildStateStore buildStore + @Inject + MultiBuildStateStore multiBuildStore + @Inject ManifestAssembler manifestAssembler @Inject - @Named(TaskExecutors.BLOCKING) - ExecutorService executor + JobService jobService static final ContainerPlatform PLATFORM_AMD64 = ContainerPlatform.of('linux/amd64') static final ContainerPlatform PLATFORM_ARM64 = ContainerPlatform.of('linux/arm64') @@ -106,73 +111,105 @@ class MultiPlatformBuildService { final initialEntry = BuildEntry.create(syntheticRequest) buildStore.storeIfAbsent(finalTargetImage, initialEntry) - // Async: wait for both builds to complete, then assemble manifest list - CompletableFuture.runAsync({ - try { - awaitAndAssemble(amd64Track, arm64Track, finalTargetImage, buildId, startTime, identity) - } - catch (Exception e) { - log.error "Multi-platform build failed for $finalTargetImage", e - // Store failure so status polling sees completion - final failedResult = BuildResult.failed(buildId, e.message, startTime) - storeBuildResult(finalTargetImage, buildId, failedResult) - } - }, executor) + // Create the multi-build request and entry, then launch via job queue + final multiBuildRequest = MultiBuildRequest.create( + finalContainerId, + finalTargetImage, + buildId, + amd64Track.targetImage, + arm64Track.targetImage, + amd64Track.cached, + arm64Track.cached, + identity, + templateRequest.maxDuration + ) + final multiBuildEntry = MultiBuildEntry.of(multiBuildRequest) + multiBuildStore.put(finalTargetImage, multiBuildEntry) + + // Launch the multi-build job — goes directly to processing queue + jobService.launchMultiBuild(multiBuildRequest) // Return immediately — build is in progress (succeeded=null) return new BuildTrack(buildId, finalTargetImage, false, null) } - protected void awaitAndAssemble(BuildTrack amd64Track, BuildTrack arm64Track, String finalTargetImage, String buildId, Instant startTime, PlatformId identity) { - // await both platform builds in parallel - final amd64Future = amd64Track.cached - ? CompletableFuture.completedFuture(true) - : CompletableFuture.supplyAsync({ awaitBuildResult(amd64Track, 'amd64', finalTargetImage) }, executor) - final arm64Future = arm64Track.cached - ? CompletableFuture.completedFuture(true) - : CompletableFuture.supplyAsync({ awaitBuildResult(arm64Track, 'arm64', finalTargetImage) }, executor) - - CompletableFuture.allOf(amd64Future, arm64Future).join() - final boolean amd64Ok = amd64Future.get() - final boolean arm64Ok = arm64Future.get() - - log.debug "Platform build results: amd64=$amd64Ok, arm64=$arm64Ok" - - if( amd64Ok && arm64Ok ) { - final List platformEntries = [ - [image: amd64Track.targetImage, platform: PLATFORM_AMD64], - [image: arm64Track.targetImage, platform: PLATFORM_ARM64] - ] - manifestAssembler.createAndPushManifestList(finalTargetImage, platformEntries, identity) - log.info "Multi-platform manifest list assembled for $finalTargetImage" - // store completed build result - final completedResult = BuildResult.completed(buildId, 0, 'Multi-platform build completed', startTime, null) - storeBuildResult(finalTargetImage, buildId, completedResult) + // ************************************************************** + // ** JobHandler implementation + // ************************************************************** + + @Override + MultiBuildEntry getJobEntry(JobSpec job) { + multiBuildStore.get(job.entryKey) + } + + @Override + JobSpec launchJob(JobSpec job, MultiBuildEntry entry) { + // no-op: multi-build jobs don't launch K8s/Docker processes + return job.withLaunchTime(Instant.now()) + } + + @Override + void onJobCompletion(JobSpec job, MultiBuildEntry entry, JobState state) { + final request = entry.request + final buildId = request.buildId + final startTime = request.creationTime + + if( state.succeeded() ) { + try { + final List platformEntries = [ + [image: request.amd64TargetImage, platform: PLATFORM_AMD64], + [image: request.arm64TargetImage, platform: PLATFORM_ARM64] + ] + manifestAssembler.createAndPushManifestList(request.targetImage, platformEntries, request.identity) + log.info "Multi-platform manifest list assembled for ${request.targetImage}" + + final completedResult = BuildResult.completed(buildId, 0, 'Multi-platform build completed', startTime, null) + updateStores(entry, completedResult) + } + catch (Exception e) { + log.error "Multi-platform manifest assembly failed for ${request.targetImage}", e + final failedResult = BuildResult.failed(buildId, "Manifest assembly failed: ${e.message}", startTime) + updateStores(entry, failedResult) + } } else { - log.error "Multi-platform build failed — amd64Ok=$amd64Ok, arm64Ok=$arm64Ok" - final failedResult = BuildResult.failed(buildId, "Multi-platform build failed — amd64=$amd64Ok, arm64=$arm64Ok", startTime) - storeBuildResult(finalTargetImage, buildId, failedResult) + final failedResult = BuildResult.failed(buildId, state.stdout ?: "Multi-platform build failed", startTime) + updateStores(entry, failedResult) } } - private void storeBuildResult(String targetImage, String buildId, BuildResult result) { + @Override + void onJobException(JobSpec job, MultiBuildEntry entry, Throwable error) { + final request = entry.request + final result = BuildResult.failed(request.buildId, error.message, request.creationTime) + updateStores(entry, result) + log.error("Multi-platform build exception for '${request.targetImage}' - operation=${job.operationName}; cause=${error.message}", error) + } + + @Override + void onJobTimeout(JobSpec job, MultiBuildEntry entry) { + final request = entry.request + final result = BuildResult.failed(request.buildId, "Multi-platform build timed out '${request.targetImage}'", request.creationTime) + updateStores(entry, result) + log.warn "Multi-platform build timed out '${request.targetImage}'; operation=${job.operationName}; duration=${result.duration}" + } + + // ************************************************************** + // ** helper methods + // ************************************************************** + + private void updateStores(MultiBuildEntry entry, BuildResult result) { + final targetImage = entry.request.targetImage + // update the multi-build state store + multiBuildStore.put(targetImage, entry.withResult(result)) + // update the build state store for status polling final existing = buildStore.getBuild(targetImage) if( existing ) { buildStore.storeBuild(targetImage, existing.withResult(result)) } else { - log.warn "Multi-platform build entry not found for $targetImage, buildId=$buildId" - } - } - - private boolean awaitBuildResult(BuildTrack track, String arch, String finalTargetImage) { - final future = buildStore.awaitBuild(track.targetImage) - if( !future ) { - log.error "Multi-platform build: unable to await $arch build for $finalTargetImage" - return false + log.warn "Multi-platform build entry not found in build store for $targetImage" } - return future.get()?.succeeded() } protected BuildRequest createPlatformRequest(BuildRequest template, ContainerPlatform platform, String suffix) { diff --git a/src/main/groovy/io/seqera/wave/service/cleanup/CleanupServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/cleanup/CleanupServiceImpl.groovy index ee168237b7..1af56be2d7 100644 --- a/src/main/groovy/io/seqera/wave/service/cleanup/CleanupServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/cleanup/CleanupServiceImpl.groovy @@ -150,8 +150,10 @@ class CleanupServiceImpl implements Runnable, CleanupService { ? config.succeededDuration : config.failedDuration final expirationSecs = Instant.now().plus(ttl).epochSecond - // schedule the job deletion - store.add(JOB_PREFIX + job.operationName, expirationSecs) + // schedule the job deletion (skip for MultiBuild — no K8s/Docker job to delete) + if( job.type != JobSpec.Type.MultiBuild ) { + store.add(JOB_PREFIX + job.operationName, expirationSecs) + } // schedule work dir path deletion if( job.workDir ) { store.add(DIR_PREFIX + job.workDir, expirationSecs) diff --git a/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy b/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy index efd1acdf66..afec2716f4 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobFactory.groovy @@ -28,6 +28,7 @@ import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.configuration.WaveLite import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.builder.MultiBuildRequest import io.seqera.wave.configuration.MirrorConfig import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.scan.ScanRequest @@ -95,6 +96,16 @@ class JobFactory { ) } + JobSpec multiBuild(MultiBuildRequest request) { + assert request.multiBuildId.startsWith(MultiBuildRequest.ID_PREFIX) + JobSpec.multiBuild( + request.targetImage, + request.multiBuildId.replace('_', '-'), + request.creationTime, + request.maxDuration + ) + } + static private String generate(String type, String id, Instant creationTime) { final prefix = type.toLowerCase() return prefix + '-' + Hashing diff --git a/src/main/groovy/io/seqera/wave/service/job/JobService.groovy b/src/main/groovy/io/seqera/wave/service/job/JobService.groovy index ff8d62dfe0..61b5bc634a 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobService.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobService.groovy @@ -20,6 +20,7 @@ package io.seqera.wave.service.job import io.seqera.wave.service.blob.BlobEntry import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.builder.MultiBuildRequest import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.scan.ScanRequest @@ -38,6 +39,8 @@ interface JobService { JobSpec launchMirror(MirrorRequest request) + JobSpec launchMultiBuild(MultiBuildRequest request) + JobState status(JobSpec jobSpec) void cleanup(JobSpec jobSpec, Integer exitStatus) diff --git a/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy index bd502997a6..86324ca5f2 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy @@ -27,6 +27,10 @@ import io.seqera.wave.configuration.WaveLite import io.seqera.wave.service.blob.BlobEntry import io.seqera.wave.service.blob.TransferStrategy import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.builder.BuildStateStore +import io.seqera.wave.service.builder.MultiBuildEntry +import io.seqera.wave.service.builder.MultiBuildRequest +import io.seqera.wave.service.builder.MultiBuildStateStore import io.seqera.wave.service.cleanup.CleanupService import io.seqera.wave.service.mirror.MirrorRequest import io.seqera.wave.service.scan.ScanRequest @@ -62,6 +66,14 @@ class JobServiceImpl implements JobService { @Nullable private TransferStrategy transferStrategy + @Inject + @Nullable + private BuildStateStore buildStateStore + + @Inject + @Nullable + private MultiBuildStateStore multiBuildStateStore + @Override JobSpec launchTransfer(BlobEntry blob, List command) { if( !transferStrategy ) @@ -107,8 +119,20 @@ class JobServiceImpl implements JobService { return job } + @Override + JobSpec launchMultiBuild(MultiBuildRequest request) { + // create the unique job id for the multi-platform build + final job = jobFactory.multiBuild(request) + // multi-build jobs go directly to processing queue (no K8s resources needed) + processingQueue.offer(job.withLaunchTime(java.time.Instant.now())) + return job + } + @Override JobState status(JobSpec job) { + if( job.type == JobSpec.Type.MultiBuild ) { + return multiBuildStatus(job) + } try { return operations.status(job) } @@ -122,4 +146,39 @@ class JobServiceImpl implements JobService { void cleanup(JobSpec job, Integer exitStatus) { cleanupService.cleanupJob(job, exitStatus) } + + protected JobState multiBuildStatus(JobSpec job) { + try { + final entry = multiBuildStateStore?.get(job.entryKey) as MultiBuildEntry + if( !entry ) + return new JobState(JobState.Status.UNKNOWN, null, null) + + // check if both sub-builds are done + final amd64Entry = buildStateStore?.getBuild(entry.request.amd64TargetImage) + final arm64Entry = buildStateStore?.getBuild(entry.request.arm64TargetImage) + + final amd64Done = entry.request.amd64Cached || amd64Entry?.done() + final arm64Done = entry.request.arm64Cached || arm64Entry?.done() + + if( !amd64Done || !arm64Done ) { + return JobState.running() + } + + // both done — check if both succeeded + final amd64Ok = entry.request.amd64Cached || amd64Entry?.result?.succeeded() + final arm64Ok = entry.request.arm64Cached || arm64Entry?.result?.succeeded() + + if( amd64Ok && arm64Ok ) { + return JobState.succeeded(null) + } + else { + final msg = "Multi-platform sub-build failed — amd64=${amd64Ok}, arm64=${arm64Ok}" + return JobState.failed(null, msg) + } + } + catch (Throwable t) { + log.warn "Unable to obtain multi-build status for job=${job.operationName} - cause: ${t.message}", t + return new JobState(JobState.Status.UNKNOWN, null, t.message) + } + } } diff --git a/src/main/groovy/io/seqera/wave/service/job/JobSpec.groovy b/src/main/groovy/io/seqera/wave/service/job/JobSpec.groovy index 96f72a41c8..af21ebd94f 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobSpec.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobSpec.groovy @@ -37,7 +37,7 @@ import io.seqera.random.LongRndKey @CompileStatic class JobSpec { - enum Type { Transfer, Build, Scan, Mirror } + enum Type { Transfer, Build, Scan, Mirror, MultiBuild } /** * The job unique identifier @@ -144,6 +144,19 @@ class JobSpec { ) } + static JobSpec multiBuild(String recordId, String operationName, Instant creationTime, Duration maxDuration) { + new JobSpec( + LongRndKey.rndHex(), + Type.MultiBuild, + recordId, + operationName, + creationTime, + null, + maxDuration, + null + ) + } + JobSpec withLaunchTime(Instant instant) { new JobSpec( this.id, diff --git a/src/test/groovy/io/seqera/wave/service/builder/MultiBuildEntryTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/MultiBuildEntryTest.groovy new file mode 100644 index 0000000000..afea043360 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/builder/MultiBuildEntryTest.groovy @@ -0,0 +1,112 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import spock.lang.Specification + +import java.time.Duration +import java.time.Instant + +import io.seqera.wave.tower.PlatformId +import io.seqera.wave.tower.User + +class MultiBuildEntryTest extends Specification { + + def 'should create multi-build entry'() { + given: + def identity = new PlatformId(new User(id: 1, email: 'foo@user.com')) + def request = MultiBuildRequest.create( + 'container123', + 'docker.io/wave:multi123', + 'bd-container123_0', + 'docker.io/wave:amd64', + 'docker.io/wave:arm64', + false, + false, + identity, + Duration.ofMinutes(5) + ) + + when: + def entry = MultiBuildEntry.of(request) + + then: + entry.key == 'docker.io/wave:multi123' + entry.requestId == request.multiBuildId + entry.request == request + !entry.done() + entry.succeeded() == null + } + + def 'should update with result'() { + given: + def identity = new PlatformId(new User(id: 1, email: 'foo@user.com')) + def request = MultiBuildRequest.create( + 'container123', + 'docker.io/wave:multi123', + 'bd-container123_0', + 'docker.io/wave:amd64', + 'docker.io/wave:arm64', + false, + false, + identity, + Duration.ofMinutes(5) + ) + + when: + def entry = MultiBuildEntry.of(request) + then: + !entry.done() + + when: + def result = BuildResult.completed('bd-container123_0', 0, 'ok', Instant.now(), null) + entry = entry.withResult(result) + then: + entry.done() + entry.succeeded() + entry.result == result + entry.request == request + } + + def 'should report failure'() { + given: + def identity = new PlatformId(new User(id: 1, email: 'foo@user.com')) + def request = MultiBuildRequest.create( + 'container123', + 'docker.io/wave:multi123', + 'bd-container123_0', + 'docker.io/wave:amd64', + 'docker.io/wave:arm64', + false, + false, + identity, + Duration.ofMinutes(5) + ) + + when: + def entry = MultiBuildEntry.of(request) + def result = BuildResult.failed('bd-container123_0', 'something went wrong', Instant.now()) + entry = entry.withResult(result) + + then: + entry.done() + !entry.succeeded() + entry.result.failed() + } +} diff --git a/src/test/groovy/io/seqera/wave/service/builder/MultiBuildRequestTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/MultiBuildRequestTest.groovy new file mode 100644 index 0000000000..90c0ccf76c --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/builder/MultiBuildRequestTest.groovy @@ -0,0 +1,81 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +import spock.lang.Specification + +import java.time.Duration + +import io.seqera.wave.tower.PlatformId +import io.seqera.wave.tower.User + +class MultiBuildRequestTest extends Specification { + + def 'should create multi-build request'() { + given: + def identity = new PlatformId(new User(id: 1, email: 'foo@user.com')) + + when: + def request = MultiBuildRequest.create( + 'container123', + 'docker.io/wave:multi123', + 'bd-container123_0', + 'docker.io/wave:amd64', + 'docker.io/wave:arm64', + false, + false, + identity, + Duration.ofMinutes(5) + ) + + then: + request.multiBuildId.startsWith(MultiBuildRequest.ID_PREFIX) + request.targetImage == 'docker.io/wave:multi123' + request.containerId == 'container123' + request.buildId == 'bd-container123_0' + request.amd64TargetImage == 'docker.io/wave:amd64' + request.arm64TargetImage == 'docker.io/wave:arm64' + request.amd64Cached == false + request.arm64Cached == false + request.identity == identity + request.creationTime != null + request.maxDuration == Duration.ofMinutes(5) + } + + def 'should create from map'() { + when: + def request = MultiBuildRequest.of( + multiBuildId: 'mb-abc123', + targetImage: 'docker.io/wave:multi', + containerId: 'cid', + buildId: 'bd-cid_0', + amd64TargetImage: 'docker.io/wave:amd64', + arm64TargetImage: 'docker.io/wave:arm64', + amd64Cached: true, + arm64Cached: false, + maxDuration: Duration.ofMinutes(10) + ) + + then: + request.multiBuildId == 'mb-abc123' + request.targetImage == 'docker.io/wave:multi' + request.amd64Cached == true + request.arm64Cached == false + } +} diff --git a/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy index 85d8180dd3..7acc56aba7 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy @@ -23,10 +23,11 @@ import spock.lang.Specification import java.nio.file.Path import java.time.Duration import java.time.Instant -import java.util.concurrent.CompletableFuture -import java.util.concurrent.Executors import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.service.job.JobService +import io.seqera.wave.service.job.JobSpec +import io.seqera.wave.service.job.JobState import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User @@ -67,17 +68,20 @@ class MultiPlatformBuildServiceTest extends Specification { arm64Req.cacheRepository == 'docker.io/cache' } - def 'should return in-progress build track'() { + def 'should return in-progress build track and launch multi-build job'() { given: def buildService = Mock(ContainerBuildService) def buildStore = Mock(BuildStateStore) + def multiBuildStore = Mock(MultiBuildStateStore) def manifestAssembler = Mock(ManifestAssembler) + def jobService = Mock(JobService) def service = new MultiPlatformBuildService( buildService: buildService, buildStore: buildStore, + multiBuildStore: multiBuildStore, manifestAssembler: manifestAssembler, - executor: Executors.newSingleThreadExecutor() + jobService: jobService ) def template = BuildRequest.of( @@ -102,86 +106,202 @@ class MultiPlatformBuildServiceTest extends Specification { track.cached == false track.succeeded == null track.id == 'bd-multi123_0' + and: + 1 * buildStore.storeIfAbsent('docker.io/wave:multi123', _) + 1 * multiBuildStore.put('docker.io/wave:multi123', _) + 1 * jobService.launchMultiBuild(_) } - def 'should assemble manifest list on both builds succeeding'() { + def 'should assemble manifest on job completion with success'() { given: def buildStore = Mock(BuildStateStore) + def multiBuildStore = Mock(MultiBuildStateStore) def manifestAssembler = Mock(ManifestAssembler) def service = new MultiPlatformBuildService( buildStore: buildStore, - manifestAssembler: manifestAssembler, - executor: Executors.newSingleThreadExecutor() + multiBuildStore: multiBuildStore, + manifestAssembler: manifestAssembler ) - def amd64Track = new BuildTrack('bd-amd64_0', 'docker.io/wave:amd64', false, null) - def arm64Track = new BuildTrack('bd-arm64_0', 'docker.io/wave:arm64', false, null) - def identity = PlatformId.NULL + def identity = new PlatformId(new User(id: 1, email: 'foo@user.com')) + def request = MultiBuildRequest.of( + multiBuildId: 'mb-abc123', + targetImage: 'docker.io/wave:multi', + containerId: 'cid', + buildId: 'bd-cid_0', + amd64TargetImage: 'docker.io/wave:amd64', + arm64TargetImage: 'docker.io/wave:arm64', + amd64Cached: false, + arm64Cached: false, + identity: identity, + creationTime: Instant.now(), + maxDuration: Duration.ofMinutes(5) + ) + def entry = MultiBuildEntry.of(request) + def job = JobSpec.multiBuild('docker.io/wave:multi', 'mb-abc123', Instant.now(), Duration.ofMinutes(5)) + def state = JobState.succeeded(null) and: - def now = Instant.now() - def amd64Result = BuildResult.completed('bd-amd64_0', 0, 'ok', now, 'sha256:aaa') - def arm64Result = BuildResult.completed('bd-arm64_0', 0, 'ok', now, 'sha256:bbb') - buildStore.awaitBuild('docker.io/wave:amd64') >> CompletableFuture.completedFuture(amd64Result) - buildStore.awaitBuild('docker.io/wave:arm64') >> CompletableFuture.completedFuture(arm64Result) + def existingBuildEntry = BuildEntry.create(BuildRequest.of( + buildId: 'bd-cid_0', + containerId: 'cid', + targetImage: 'docker.io/wave:multi', + startTime: Instant.now() + )) when: - service.awaitAndAssemble(amd64Track, arm64Track, 'docker.io/wave:final', identity) + service.onJobCompletion(job, entry, state) then: - 1 * manifestAssembler.createAndPushManifestList('docker.io/wave:final', ['docker.io/wave:amd64', 'docker.io/wave:arm64'], identity) + 1 * manifestAssembler.createAndPushManifestList('docker.io/wave:multi', _, identity) + 1 * multiBuildStore.put('docker.io/wave:multi', _) + 1 * buildStore.getBuild('docker.io/wave:multi') >> existingBuildEntry + 1 * buildStore.storeBuild('docker.io/wave:multi', _) } - def 'should not assemble manifest list when a build fails'() { + def 'should not assemble manifest on job completion with failure'() { given: def buildStore = Mock(BuildStateStore) + def multiBuildStore = Mock(MultiBuildStateStore) def manifestAssembler = Mock(ManifestAssembler) def service = new MultiPlatformBuildService( buildStore: buildStore, - manifestAssembler: manifestAssembler, - executor: Executors.newSingleThreadExecutor() + multiBuildStore: multiBuildStore, + manifestAssembler: manifestAssembler ) - def amd64Track = new BuildTrack('bd-amd64_0', 'docker.io/wave:amd64', false, null) - def arm64Track = new BuildTrack('bd-arm64_0', 'docker.io/wave:arm64', false, null) - def identity = PlatformId.NULL + def request = MultiBuildRequest.of( + multiBuildId: 'mb-abc123', + targetImage: 'docker.io/wave:multi', + containerId: 'cid', + buildId: 'bd-cid_0', + amd64TargetImage: 'docker.io/wave:amd64', + arm64TargetImage: 'docker.io/wave:arm64', + amd64Cached: false, + arm64Cached: false, + creationTime: Instant.now(), + maxDuration: Duration.ofMinutes(5) + ) + def entry = MultiBuildEntry.of(request) + def job = JobSpec.multiBuild('docker.io/wave:multi', 'mb-abc123', Instant.now(), Duration.ofMinutes(5)) + def state = JobState.failed(-1, 'sub-build failed') and: - def now = Instant.now() - def amd64Result = BuildResult.completed('bd-amd64_0', 0, 'ok', now, 'sha256:aaa') - def arm64Result = BuildResult.failed('bd-arm64_0', 'error', now) - buildStore.awaitBuild('docker.io/wave:amd64') >> CompletableFuture.completedFuture(amd64Result) - buildStore.awaitBuild('docker.io/wave:arm64') >> CompletableFuture.completedFuture(arm64Result) + def existingBuildEntry = BuildEntry.create(BuildRequest.of( + buildId: 'bd-cid_0', + containerId: 'cid', + targetImage: 'docker.io/wave:multi', + startTime: Instant.now() + )) when: - service.awaitAndAssemble(amd64Track, arm64Track, 'docker.io/wave:final', identity) + service.onJobCompletion(job, entry, state) then: 0 * manifestAssembler.createAndPushManifestList(_, _, _) + 1 * multiBuildStore.put('docker.io/wave:multi', _) + 1 * buildStore.getBuild('docker.io/wave:multi') >> existingBuildEntry + 1 * buildStore.storeBuild('docker.io/wave:multi', _) } - def 'should handle cached platform builds'() { + def 'should handle job timeout'() { given: def buildStore = Mock(BuildStateStore) - def manifestAssembler = Mock(ManifestAssembler) + def multiBuildStore = Mock(MultiBuildStateStore) def service = new MultiPlatformBuildService( buildStore: buildStore, - manifestAssembler: manifestAssembler, - executor: Executors.newSingleThreadExecutor() + multiBuildStore: multiBuildStore + ) + + def request = MultiBuildRequest.of( + multiBuildId: 'mb-abc123', + targetImage: 'docker.io/wave:multi', + containerId: 'cid', + buildId: 'bd-cid_0', + amd64TargetImage: 'docker.io/wave:amd64', + arm64TargetImage: 'docker.io/wave:arm64', + amd64Cached: false, + arm64Cached: false, + creationTime: Instant.now(), + maxDuration: Duration.ofMinutes(5) + ) + def entry = MultiBuildEntry.of(request) + def job = JobSpec.multiBuild('docker.io/wave:multi', 'mb-abc123', Instant.now(), Duration.ofMinutes(5)) + + and: + def existingBuildEntry = BuildEntry.create(BuildRequest.of( + buildId: 'bd-cid_0', + containerId: 'cid', + targetImage: 'docker.io/wave:multi', + startTime: Instant.now() + )) + + when: + service.onJobTimeout(job, entry) + + then: + 1 * multiBuildStore.put('docker.io/wave:multi', _) + 1 * buildStore.getBuild('docker.io/wave:multi') >> existingBuildEntry + 1 * buildStore.storeBuild('docker.io/wave:multi', _) + } + + def 'should handle job exception'() { + given: + def buildStore = Mock(BuildStateStore) + def multiBuildStore = Mock(MultiBuildStateStore) + + def service = new MultiPlatformBuildService( + buildStore: buildStore, + multiBuildStore: multiBuildStore + ) + + def request = MultiBuildRequest.of( + multiBuildId: 'mb-abc123', + targetImage: 'docker.io/wave:multi', + containerId: 'cid', + buildId: 'bd-cid_0', + amd64TargetImage: 'docker.io/wave:amd64', + arm64TargetImage: 'docker.io/wave:arm64', + amd64Cached: false, + arm64Cached: false, + creationTime: Instant.now(), + maxDuration: Duration.ofMinutes(5) ) + def entry = MultiBuildEntry.of(request) + def job = JobSpec.multiBuild('docker.io/wave:multi', 'mb-abc123', Instant.now(), Duration.ofMinutes(5)) - def amd64Track = new BuildTrack('bd-amd64_0', 'docker.io/wave:amd64', true, true) - def arm64Track = new BuildTrack('bd-arm64_0', 'docker.io/wave:arm64', true, true) - def identity = PlatformId.NULL + and: + def existingBuildEntry = BuildEntry.create(BuildRequest.of( + buildId: 'bd-cid_0', + containerId: 'cid', + targetImage: 'docker.io/wave:multi', + startTime: Instant.now() + )) + + when: + service.onJobException(job, entry, new RuntimeException('boom')) + + then: + 1 * multiBuildStore.put('docker.io/wave:multi', _) + 1 * buildStore.getBuild('docker.io/wave:multi') >> existingBuildEntry + 1 * buildStore.storeBuild('docker.io/wave:multi', _) + } + + def 'launchJob should be a no-op returning job with launch time'() { + given: + def service = new MultiPlatformBuildService() + def job = JobSpec.multiBuild('docker.io/wave:multi', 'mb-abc123', Instant.now(), Duration.ofMinutes(5)) + def entry = Mock(MultiBuildEntry) when: - service.awaitAndAssemble(amd64Track, arm64Track, 'docker.io/wave:final', identity) + def result = service.launchJob(job, entry) then: - 0 * buildStore.awaitBuild(_) - 1 * manifestAssembler.createAndPushManifestList('docker.io/wave:final', ['docker.io/wave:amd64', 'docker.io/wave:arm64'], identity) + result.id == job.id + result.type == job.type + result.launchTime != null } } From 390e1f53be02ce7f180b188a24fd00419bd1a2cc Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Mar 2026 00:28:15 +0100 Subject: [PATCH 03/15] POC 3 Signed-off-by: Paolo Di Tommaso --- .../wave/core/MultiContainerPlatform.groovy | 45 +++++++ .../wave/service/builder/BuildRequest.groovy | 17 ++- .../builder/MultiPlatformBuildService.groovy | 42 +++++- .../service/mail/impl/MailServiceImpl.groovy | 5 +- .../persistence/WaveBuildRecord.groovy | 3 +- .../service/persistence/WaveScanRecord.groovy | 8 +- .../scan/ContainerScanServiceImpl.groovy | 13 +- .../seqera/wave/service/scan/ScanEntry.groovy | 39 +++++- .../wave/service/scan/ScanRequest.groovy | 8 +- .../wave/controller/ViewControllerTest.groovy | 8 +- .../MultiPlatformBuildServiceTest.groovy | 124 ++++++++++-------- .../impl/SurrealPersistenceServiceTest.groovy | 6 +- .../PostgresPersistentServiceTest.groovy | 6 +- .../wave/service/scan/ScanEntryTest.groovy | 4 +- 14 files changed, 238 insertions(+), 90 deletions(-) create mode 100644 src/main/groovy/io/seqera/wave/core/MultiContainerPlatform.groovy diff --git a/src/main/groovy/io/seqera/wave/core/MultiContainerPlatform.groovy b/src/main/groovy/io/seqera/wave/core/MultiContainerPlatform.groovy new file mode 100644 index 0000000000..1adaa010f0 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/core/MultiContainerPlatform.groovy @@ -0,0 +1,45 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.core + +import groovy.transform.CompileStatic +/** + * A composite container platform representing multiple architectures. + * + * @author Paolo Di Tommaso + */ +@CompileStatic +class MultiContainerPlatform extends ContainerPlatform { + + static final MultiContainerPlatform MULTI_PLATFORM = new MultiContainerPlatform( + [ContainerPlatform.of('linux/amd64'), ContainerPlatform.of('linux/arm64')] + ) + + final List platforms + + MultiContainerPlatform(List platforms) { + super(platforms[0].os, platforms[0].arch, platforms[0].variant) + this.platforms = List.copyOf(platforms) + } + + @Override + String toString() { + platforms.collect { it.toString() }.join(',') + } +} diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy index 61e1428308..365f0995b5 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy @@ -149,6 +149,16 @@ class BuildRequest { */ final String buildTemplate + /** + * When {@code true}, email notifications should not be sent for this build + */ + final boolean noEmail + + /** + * When {@code true}, this is a multi-platform composite build (linux/amd64 + linux/arm64) + */ + final boolean multiPlatform + BuildRequest( String containerId, String containerFile, @@ -167,7 +177,8 @@ class BuildRequest { BuildFormat format, Duration maxDuration, BuildCompression compression, - String buildTemplate + String buildTemplate, + boolean noEmail = false ) { this.containerId = containerId @@ -189,6 +200,8 @@ class BuildRequest { this.maxDuration = maxDuration this.compression = compression this.buildTemplate = buildTemplate + this.noEmail = noEmail + this.multiPlatform = false // NOTE: this is meant to be updated - automatically - when the request is submitted this.buildId = computeBuildId(containerId) } @@ -217,6 +230,8 @@ class BuildRequest { this.compression = opts.compression as BuildCompression this.buildId = opts.buildId ?: computeBuildId(containerId) this.buildTemplate = opts.buildTemplate + this.noEmail = opts.noEmail as boolean + this.multiPlatform = opts.multiPlatform as boolean } static BuildRequest of(Map opts) { diff --git a/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy index 10f2678198..52dc3c1bda 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy @@ -22,13 +22,18 @@ import java.time.Instant import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import io.micronaut.context.event.ApplicationEventPublisher import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.service.job.JobEntry +import io.seqera.wave.core.MultiContainerPlatform import io.seqera.wave.service.job.JobHandler import io.seqera.wave.service.job.JobService import io.seqera.wave.service.job.JobSpec import io.seqera.wave.service.job.JobState +import io.seqera.wave.service.persistence.PersistenceService +import io.seqera.wave.service.persistence.WaveBuildRecord +import io.seqera.wave.service.scan.ContainerScanService import io.seqera.wave.tower.PlatformId +import jakarta.annotation.Nullable import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton @@ -62,6 +67,15 @@ class MultiPlatformBuildService implements JobHandler { @Inject JobService jobService + @Inject + ApplicationEventPublisher eventPublisher + + @Inject + PersistenceService persistenceService + + @Inject @Nullable + ContainerScanService scanService + static final ContainerPlatform PLATFORM_AMD64 = ContainerPlatform.of('linux/amd64') static final ContainerPlatform PLATFORM_ARM64 = ContainerPlatform.of('linux/arm64') @@ -100,13 +114,16 @@ class MultiPlatformBuildService implements JobHandler { identity: templateRequest.identity, containerFile: templateRequest.containerFile, condaFile: templateRequest.condaFile, - platform: templateRequest.platform, + workspace: templateRequest.workspace, + platform: MultiContainerPlatform.MULTI_PLATFORM, + configJson: templateRequest.configJson, ip: templateRequest.ip, offsetId: templateRequest.offsetId, scanId: templateRequest.scanId, format: templateRequest.format, compression: templateRequest.compression, - buildTemplate: templateRequest.buildTemplate + buildTemplate: templateRequest.buildTemplate, + multiPlatform: true ) final initialEntry = BuildEntry.create(syntheticRequest) buildStore.storeIfAbsent(finalTargetImage, initialEntry) @@ -202,10 +219,20 @@ class MultiPlatformBuildService implements JobHandler { final targetImage = entry.request.targetImage // update the multi-build state store multiBuildStore.put(targetImage, entry.withResult(result)) - // update the build state store for status polling + // update the build state store for status polling and publish event final existing = buildStore.getBuild(targetImage) if( existing ) { - buildStore.storeBuild(targetImage, existing.withResult(result)) + final updatedEntry = existing.withResult(result) + buildStore.storeBuild(targetImage, updatedEntry) + // trigger scan on the composite image before persisting the record + // (scan record must exist before build record to avoid status polling race condition) + scanService?.scanOnBuild(updatedEntry) + // persist the build record and publish event (triggers email notification) + // note: the multiPlatform flag on the request survives serialization and is used + // by WaveBuildRecord.create0() to render the correct platform string + final event = new BuildEvent(updatedEntry.request, result) + persistenceService.saveBuildAsync(WaveBuildRecord.fromEvent(event)) + eventPublisher.publishEvent(event) } else { log.warn "Multi-platform build entry not found in build store for $targetImage" @@ -237,12 +264,13 @@ class MultiPlatformBuildService implements JobHandler { template.configJson, template.offsetId, template.containerConfig, - template.scanId, + null, // scanId - suppress per-platform scans; scan runs on the composite image template.buildContext, template.format, template.maxDuration, template.compression, - template.buildTemplate + template.buildTemplate, + true // noEmail - suppress individual sub-build notifications ) } } diff --git a/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy index 24f66bffe8..36356ae4a9 100644 --- a/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy @@ -28,6 +28,7 @@ import io.seqera.mail.Mail import io.seqera.mail.MailAttachment import io.seqera.mail.MailHelper import io.seqera.mail.MailerConfig +import io.seqera.wave.core.MultiContainerPlatform import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildResult @@ -59,6 +60,8 @@ class MailServiceImpl implements MailService { @EventListener void onBuildEvent(BuildEvent event) { + if( event.request.noEmail ) + return try { sendCompletionEmail(event.request, event.result) } @@ -95,7 +98,7 @@ class MailServiceImpl implements MailService { binding.build_image = preventLinkFormatting(req.targetImage) binding.build_format = req.format?.render() ?: 'Docker' binding.build_compression = req.compression?.mode ?: '(default)' - binding.build_platform = req.platform + binding.build_platform = req.multiPlatform ? MultiContainerPlatform.MULTI_PLATFORM.toString() : req.platform binding.build_template = req.buildTemplate ?: '(default)' binding.build_containerfile = req.containerFile ?: '-' binding.build_condafile = req.condaFile diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy index 415439c6fe..1cc02b1e6c 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy @@ -26,6 +26,7 @@ import groovy.transform.EqualsAndHashCode import groovy.transform.ToString import io.seqera.wave.api.BuildCompression import io.seqera.wave.api.BuildStatusResponse +import io.seqera.wave.core.MultiContainerPlatform import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildFormat @@ -91,7 +92,7 @@ class WaveBuildRecord { userId: request.identity.user?.id, requestIp: request.ip, startTime: request.startTime, - platform: request.platform, + platform: request.multiPlatform ? MultiContainerPlatform.MULTI_PLATFORM.toString() : request.platform?.toString(), offsetId: request.offsetId, scanId: request.scanId, format: request.format, diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy index d1d5fab88b..05dbe81ac7 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy @@ -26,7 +26,7 @@ import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString import groovy.util.logging.Slf4j -import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.core.MultiContainerPlatform import io.seqera.wave.service.scan.ScanEntry import io.seqera.wave.service.scan.ScanVulnerability import io.seqera.wave.util.StringUtils @@ -47,7 +47,7 @@ class WaveScanRecord implements Cloneable { String mirrorId String requestId String containerImage - ContainerPlatform platform + String platform Instant startTime Duration duration String status @@ -65,7 +65,7 @@ class WaveScanRecord implements Cloneable { String mirrorId, String requestId, String containerImage, - ContainerPlatform platform, + String platform, Instant startTime, Duration duration, String status, @@ -98,7 +98,7 @@ class WaveScanRecord implements Cloneable { this.mirrorId = scan.mirrorId this.requestId = scan.requestId this.containerImage = scan.containerImage - this.platform = scan.platform + this.platform = scan.multiPlatform ? MultiContainerPlatform.MULTI_PLATFORM.toString() : scan.platform?.toString() this.startTime = scan.startTime this.duration = scan.duration this.status = scan.status diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index 53541f8396..b1610c576c 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -36,6 +36,7 @@ import io.micronaut.objectstorage.request.UploadRequest import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.api.ScanMode import io.seqera.wave.configuration.ScanConfig +import io.seqera.wave.core.MultiContainerPlatform import io.seqera.wave.configuration.ScanEnabled import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildRequest @@ -230,6 +231,7 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler, JobEntry { */ String logs + /** + * When {@code true}, this scan targets a multi-platform composite image (linux/amd64 + linux/arm64) + */ + boolean multiPlatform + @Override String getKey() { return scanId @@ -142,7 +148,10 @@ class ScanEntry implements StateEntry, JobEntry { request.creationTime, null, PENDING, - List.of()) + List.of(), + null, + null, + request.multiPlatform) } ScanEntry success(List vulnerabilities){ @@ -159,7 +168,9 @@ class ScanEntry implements StateEntry, JobEntry { Duration.between(this.startTime, Instant.now()), SUCCEEDED, vulnerabilities, - 0 ) + 0, + null, + this.multiPlatform) } ScanEntry failure(Integer exitCode, String logs){ @@ -177,7 +188,8 @@ class ScanEntry implements StateEntry, JobEntry { FAILED, List.of(), exitCode, - logs) + logs, + this.multiPlatform) } static ScanEntry failure(ScanRequest request){ @@ -193,7 +205,10 @@ class ScanEntry implements StateEntry, JobEntry { request.creationTime, Duration.between(request.creationTime, Instant.now()), FAILED, - List.of()) + List.of(), + null, + null, + request.multiPlatform) } @@ -223,7 +238,8 @@ class ScanEntry implements StateEntry, JobEntry { opts.status as String, opts.vulnerabilities as List, opts.exitCode as Integer, - opts.logs as String + opts.logs as String, + opts.multiPlatform as boolean ) } @@ -234,7 +250,7 @@ class ScanEntry implements StateEntry, JobEntry { record.mirrorId, record.requestId, record.containerImage, - record.platform, + parsePlatform0(record.platform), record.workDir, null, record.startTime, @@ -242,8 +258,17 @@ class ScanEntry implements StateEntry, JobEntry { record.status, record.vulnerabilities, record.exitCode, - record.logs + record.logs, + record.platform?.contains(',') ?: false ) } + static private ContainerPlatform parsePlatform0(String value) { + if( !value ) + return null + if( value.contains(',') ) + return new MultiContainerPlatform(value.tokenize(',').collect { ContainerPlatform.of(it.trim()) }) + return ContainerPlatform.of(value) + } + } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy index 041187a196..947a52b057 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy @@ -86,6 +86,11 @@ class ScanRequest { */ final PlatformId identity + /** + * When {@code true}, this scan targets a multi-platform composite image (linux/amd64 + linux/arm64) + */ + final boolean multiPlatform + static ScanRequest of(Map opts) { new ScanRequest( opts.scanId as String, @@ -97,7 +102,8 @@ class ScanRequest { opts.platform as ContainerPlatform, opts.workDir as Path, opts.creationTime as Instant, - opts.identity as PlatformId + opts.identity as PlatformId, + opts.multiPlatform as boolean ) } diff --git a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy index 004a3ee813..54a29d0848 100644 --- a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy @@ -371,7 +371,7 @@ class ViewControllerTest extends Specification { 'mr-12345', 'cr-12345', 'docker.io/some:image', - ContainerPlatform.DEFAULT, + ContainerPlatform.DEFAULT.toString(), Instant.now(), Duration.ofMinutes(1), ScanEntry.SUCCEEDED, @@ -410,7 +410,7 @@ class ViewControllerTest extends Specification { 'mr-12345', 'cr-12345', 'docker.io/some:image', - ContainerPlatform.DEFAULT, + ContainerPlatform.DEFAULT.toString(), Instant.now(), Duration.ofMinutes(1), ScanEntry.SUCCEEDED, @@ -684,7 +684,7 @@ class ViewControllerTest extends Specification { def service = Mock(ContainerScanService) def controller = new ViewController(scanService: service) def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/arm64') + def PLATFORM = 'linux/arm64' def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') @@ -730,7 +730,7 @@ class ViewControllerTest extends Specification { def 'should find all scans' () { given: def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/arm64') + def PLATFORM = 'linux/arm64' def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') diff --git a/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy index 7acc56aba7..07904126b5 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy @@ -24,16 +24,33 @@ import java.nio.file.Path import java.time.Duration import java.time.Instant +import io.micronaut.context.event.ApplicationEventPublisher import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.job.JobService import io.seqera.wave.service.job.JobSpec import io.seqera.wave.service.job.JobState +import io.seqera.wave.service.persistence.PersistenceService +import io.seqera.wave.service.persistence.WaveBuildRecord +import io.seqera.wave.service.scan.ContainerScanService import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User class MultiPlatformBuildServiceTest extends Specification { - def 'should create platform-specific build requests'() { + static final PlatformId TEST_IDENTITY = new PlatformId(new User(id: 1, email: 'foo@user.com')) + + private BuildEntry makeExistingBuildEntry() { + BuildEntry.create(BuildRequest.of( + buildId: 'bd-cid_0', + containerId: 'cid', + targetImage: 'docker.io/wave:multi', + startTime: Instant.now(), + identity: TEST_IDENTITY, + multiPlatform: true + )) + } + + def 'should create platform-specific build requests with noEmail flag'() { given: def service = new MultiPlatformBuildService() def template = BuildRequest.of( @@ -42,7 +59,7 @@ class MultiPlatformBuildServiceTest extends Specification { condaFile: null, workspace: Path.of('/tmp'), targetImage: 'docker.io/wave:abc123', - identity: new PlatformId(new User(id: 1, email: 'foo@user.com')), + identity: TEST_IDENTITY, platform: ContainerPlatform.of('linux/amd64'), cacheRepository: 'docker.io/cache', ip: '10.0.0.1', @@ -66,6 +83,12 @@ class MultiPlatformBuildServiceTest extends Specification { amd64Req.targetImage != arm64Req.targetImage amd64Req.cacheRepository == 'docker.io/cache' arm64Req.cacheRepository == 'docker.io/cache' + and: 'sub-builds should have noEmail flag set' + amd64Req.noEmail == true + arm64Req.noEmail == true + and: 'sub-builds should not trigger individual scans' + amd64Req.scanId == null + arm64Req.scanId == null } def 'should return in-progress build track and launch multi-build job'() { @@ -89,7 +112,7 @@ class MultiPlatformBuildServiceTest extends Specification { containerFile: 'FROM ubuntu:latest', workspace: Path.of('/tmp'), targetImage: 'docker.io/wave:abc123', - identity: new PlatformId(new User(id: 1, email: 'foo@user.com')), + identity: TEST_IDENTITY, platform: ContainerPlatform.of('linux/amd64'), format: BuildFormat.DOCKER, maxDuration: Duration.ofMinutes(5) @@ -112,19 +135,24 @@ class MultiPlatformBuildServiceTest extends Specification { 1 * jobService.launchMultiBuild(_) } - def 'should assemble manifest on job completion with success'() { + def 'should assemble manifest and persist record on job completion with success'() { given: def buildStore = Mock(BuildStateStore) def multiBuildStore = Mock(MultiBuildStateStore) def manifestAssembler = Mock(ManifestAssembler) + def eventPublisher = Mock(ApplicationEventPublisher) + def persistenceService = Mock(PersistenceService) + def scanService = Mock(ContainerScanService) def service = new MultiPlatformBuildService( buildStore: buildStore, multiBuildStore: multiBuildStore, - manifestAssembler: manifestAssembler + manifestAssembler: manifestAssembler, + eventPublisher: eventPublisher, + persistenceService: persistenceService, + scanService: scanService ) - def identity = new PlatformId(new User(id: 1, email: 'foo@user.com')) def request = MultiBuildRequest.of( multiBuildId: 'mb-abc123', targetImage: 'docker.io/wave:multi', @@ -134,7 +162,7 @@ class MultiPlatformBuildServiceTest extends Specification { arm64TargetImage: 'docker.io/wave:arm64', amd64Cached: false, arm64Cached: false, - identity: identity, + identity: TEST_IDENTITY, creationTime: Instant.now(), maxDuration: Duration.ofMinutes(5) ) @@ -142,34 +170,34 @@ class MultiPlatformBuildServiceTest extends Specification { def job = JobSpec.multiBuild('docker.io/wave:multi', 'mb-abc123', Instant.now(), Duration.ofMinutes(5)) def state = JobState.succeeded(null) - and: - def existingBuildEntry = BuildEntry.create(BuildRequest.of( - buildId: 'bd-cid_0', - containerId: 'cid', - targetImage: 'docker.io/wave:multi', - startTime: Instant.now() - )) - when: service.onJobCompletion(job, entry, state) then: - 1 * manifestAssembler.createAndPushManifestList('docker.io/wave:multi', _, identity) - 1 * multiBuildStore.put('docker.io/wave:multi', _) - 1 * buildStore.getBuild('docker.io/wave:multi') >> existingBuildEntry + 1 * manifestAssembler.createAndPushManifestList('docker.io/wave:multi', _, TEST_IDENTITY) + and: + 1 * buildStore.getBuild('docker.io/wave:multi') >> makeExistingBuildEntry() 1 * buildStore.storeBuild('docker.io/wave:multi', _) + 1 * multiBuildStore.put('docker.io/wave:multi', _) + 1 * scanService.scanOnBuild({ BuildEntry e -> e.request.targetImage == 'docker.io/wave:multi' }) + 1 * persistenceService.saveBuildAsync({ WaveBuildRecord r -> r.platform == 'linux/amd64,linux/arm64' }) + 1 * eventPublisher.publishEvent({ BuildEvent e -> e.request.multiPlatform == true }) } - def 'should not assemble manifest on job completion with failure'() { + def 'should persist failure record when sub-build fails'() { given: def buildStore = Mock(BuildStateStore) def multiBuildStore = Mock(MultiBuildStateStore) def manifestAssembler = Mock(ManifestAssembler) + def eventPublisher = Mock(ApplicationEventPublisher) + def persistenceService = Mock(PersistenceService) def service = new MultiPlatformBuildService( buildStore: buildStore, multiBuildStore: multiBuildStore, - manifestAssembler: manifestAssembler + manifestAssembler: manifestAssembler, + eventPublisher: eventPublisher, + persistenceService: persistenceService ) def request = MultiBuildRequest.of( @@ -181,6 +209,7 @@ class MultiPlatformBuildServiceTest extends Specification { arm64TargetImage: 'docker.io/wave:arm64', amd64Cached: false, arm64Cached: false, + identity: TEST_IDENTITY, creationTime: Instant.now(), maxDuration: Duration.ofMinutes(5) ) @@ -188,32 +217,31 @@ class MultiPlatformBuildServiceTest extends Specification { def job = JobSpec.multiBuild('docker.io/wave:multi', 'mb-abc123', Instant.now(), Duration.ofMinutes(5)) def state = JobState.failed(-1, 'sub-build failed') - and: - def existingBuildEntry = BuildEntry.create(BuildRequest.of( - buildId: 'bd-cid_0', - containerId: 'cid', - targetImage: 'docker.io/wave:multi', - startTime: Instant.now() - )) - when: service.onJobCompletion(job, entry, state) then: 0 * manifestAssembler.createAndPushManifestList(_, _, _) - 1 * multiBuildStore.put('docker.io/wave:multi', _) - 1 * buildStore.getBuild('docker.io/wave:multi') >> existingBuildEntry + and: + 1 * buildStore.getBuild('docker.io/wave:multi') >> makeExistingBuildEntry() 1 * buildStore.storeBuild('docker.io/wave:multi', _) + 1 * multiBuildStore.put('docker.io/wave:multi', _) + 1 * persistenceService.saveBuildAsync(_) + 1 * eventPublisher.publishEvent(_ as BuildEvent) } def 'should handle job timeout'() { given: def buildStore = Mock(BuildStateStore) def multiBuildStore = Mock(MultiBuildStateStore) + def eventPublisher = Mock(ApplicationEventPublisher) + def persistenceService = Mock(PersistenceService) def service = new MultiPlatformBuildService( buildStore: buildStore, - multiBuildStore: multiBuildStore + multiBuildStore: multiBuildStore, + eventPublisher: eventPublisher, + persistenceService: persistenceService ) def request = MultiBuildRequest.of( @@ -231,31 +259,29 @@ class MultiPlatformBuildServiceTest extends Specification { def entry = MultiBuildEntry.of(request) def job = JobSpec.multiBuild('docker.io/wave:multi', 'mb-abc123', Instant.now(), Duration.ofMinutes(5)) - and: - def existingBuildEntry = BuildEntry.create(BuildRequest.of( - buildId: 'bd-cid_0', - containerId: 'cid', - targetImage: 'docker.io/wave:multi', - startTime: Instant.now() - )) - when: service.onJobTimeout(job, entry) then: - 1 * multiBuildStore.put('docker.io/wave:multi', _) - 1 * buildStore.getBuild('docker.io/wave:multi') >> existingBuildEntry + 1 * buildStore.getBuild('docker.io/wave:multi') >> makeExistingBuildEntry() 1 * buildStore.storeBuild('docker.io/wave:multi', _) + 1 * multiBuildStore.put('docker.io/wave:multi', _) + 1 * persistenceService.saveBuildAsync(_) + 1 * eventPublisher.publishEvent(_ as BuildEvent) } def 'should handle job exception'() { given: def buildStore = Mock(BuildStateStore) def multiBuildStore = Mock(MultiBuildStateStore) + def eventPublisher = Mock(ApplicationEventPublisher) + def persistenceService = Mock(PersistenceService) def service = new MultiPlatformBuildService( buildStore: buildStore, - multiBuildStore: multiBuildStore + multiBuildStore: multiBuildStore, + eventPublisher: eventPublisher, + persistenceService: persistenceService ) def request = MultiBuildRequest.of( @@ -273,21 +299,15 @@ class MultiPlatformBuildServiceTest extends Specification { def entry = MultiBuildEntry.of(request) def job = JobSpec.multiBuild('docker.io/wave:multi', 'mb-abc123', Instant.now(), Duration.ofMinutes(5)) - and: - def existingBuildEntry = BuildEntry.create(BuildRequest.of( - buildId: 'bd-cid_0', - containerId: 'cid', - targetImage: 'docker.io/wave:multi', - startTime: Instant.now() - )) - when: service.onJobException(job, entry, new RuntimeException('boom')) then: - 1 * multiBuildStore.put('docker.io/wave:multi', _) - 1 * buildStore.getBuild('docker.io/wave:multi') >> existingBuildEntry + 1 * buildStore.getBuild('docker.io/wave:multi') >> makeExistingBuildEntry() 1 * buildStore.storeBuild('docker.io/wave:multi', _) + 1 * multiBuildStore.put('docker.io/wave:multi', _) + 1 * persistenceService.saveBuildAsync(_) + 1 * eventPublisher.publishEvent(_ as BuildEvent) } def 'launchJob should be a no-op returning job with launch time'() { diff --git a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy index 2147795690..d66a84f6be 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy @@ -320,7 +320,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def SCAN_ID = 'a1' def BUILD_ID = '100' def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/amd64') + def PLATFORM = 'linux/amd64' def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') @@ -369,7 +369,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def SCAN_ID = 'a1' def BUILD_ID = '100' def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/amd64') + def PLATFORM = 'linux/amd64' def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') def scan = new WaveScanRecord(SCAN_ID, BUILD_ID, null, null, CONTAINER_IMAGE, PLATFORM, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1], null, null, null) @@ -556,7 +556,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def persistence = applicationContext.getBean(SurrealPersistenceService) and: def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/amd64') + def PLATFORM = 'linux/amd64' def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') diff --git a/src/test/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentServiceTest.groovy index e09d0e74c5..c147e4ea78 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/postgres/PostgresPersistentServiceTest.groovy @@ -340,7 +340,7 @@ class PostgresPersistentServiceTest extends Specification { def SCAN_ID = 'a1' def BUILD_ID = '100' def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/amd64') + def PLATFORM = 'linux/amd64' def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') @@ -374,7 +374,7 @@ class PostgresPersistentServiceTest extends Specification { def SCAN_ID = 'a1' def BUILD_ID = '100' def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/amd64') + def PLATFORM = 'linux/amd64' def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') def scan = new WaveScanRecord(SCAN_ID, BUILD_ID, null, null, CONTAINER_IMAGE, PLATFORM, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1], null, null, null) @@ -390,7 +390,7 @@ class PostgresPersistentServiceTest extends Specification { def 'should find all scans' () { given: def CONTAINER_IMAGE = 'docker.io/my/repo:container1234' - def PLATFORM = ContainerPlatform.of('linux/amd64') + def PLATFORM = 'linux/amd64' def CVE1 = new ScanVulnerability('cve-1', 'x1', 'title1', 'package1', 'version1', 'fixed1', 'url1') def CVE2 = new ScanVulnerability('cve-2', 'x2', 'title2', 'package2', 'version2', 'fixed2', 'url2') def CVE3 = new ScanVulnerability('cve-3', 'x3', 'title3', 'package3', 'version3', 'fixed3', 'url3') diff --git a/src/test/groovy/io/seqera/wave/service/scan/ScanEntryTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ScanEntryTest.groovy index 5a343599da..8a082fa0f2 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/ScanEntryTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/ScanEntryTest.groovy @@ -274,7 +274,7 @@ class ScanEntryTest extends Specification { 'mr-12345', 'cr-12345', 'docker.io/some:image', - ContainerPlatform.DEFAULT, + ContainerPlatform.DEFAULT.toString(), Instant.now(), Duration.ofMinutes(1), ScanEntry.SUCCEEDED, @@ -292,7 +292,7 @@ class ScanEntryTest extends Specification { entry.mirrorId == recrd.mirrorId entry.requestId == recrd.requestId entry.containerImage == recrd.containerImage - entry.platform == recrd.platform + entry.platform == ContainerPlatform.DEFAULT entry.startTime == recrd.startTime entry.duration == recrd.duration entry.status == recrd.status From 8adb8acf4723af2b1817c493f287e9dfd19caf2e Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Mar 2026 17:20:01 +0100 Subject: [PATCH 04/15] Add multi-platform container builds with per-architecture scanning Refactor ContainerPlatform to support multi-arch builds natively, replacing MultiContainerPlatform with a unified model. Fan out security scans per architecture since Trivy only accepts a single --platform flag. Key changes: - Consolidate ContainerPlatform to handle both single and multi-arch - Add ScanIds helper for encoding/decoding per-platform scan IDs - Fan out scans in ContainerScanServiceImpl per architecture - Add BuildRequest.withScanId() for propagating multi-scan IDs - Update views and email templates for per-arch scan links - Poll all per-arch scans in ContainerStatusServiceImpl - Extract ScanIds.populateScanBinding() to DRY scan binding logic Co-Authored-By: Claude Opus 4.6 --- .../controller/ContainerController.groovy | 35 +++- .../wave/controller/ViewController.groovy | 10 +- .../seqera/wave/core/ContainerPlatform.groovy | 46 ++++- .../wave/core/MultiContainerPlatform.groovy | 45 ----- .../io/seqera/wave/proxy/ProxyClient.groovy | 10 +- .../wave/service/builder/BuildRequest.groovy | 33 +++- .../service/builder/ManifestAssembler.groovy | 2 +- .../service/builder/MultiBuildRequest.groovy | 3 + .../builder/MultiPlatformBuildService.groovy | 18 +- .../wave/service/job/JobServiceImpl.groovy | 14 +- .../service/mail/impl/MailServiceImpl.groovy | 7 +- .../persistence/WaveBuildRecord.groovy | 3 +- .../service/persistence/WaveScanRecord.groovy | 3 +- .../request/ContainerStatusServiceImpl.groovy | 48 ++++- .../scan/ContainerScanServiceImpl.groovy | 30 +-- .../seqera/wave/service/scan/ScanEntry.groovy | 34 +--- .../seqera/wave/service/scan/ScanIds.groovy | 122 ++++++++++++ .../wave/service/scan/ScanRequest.groovy | 8 +- .../seqera/wave/util/ContainerHelper.groovy | 11 +- .../io/seqera/wave/build-notification.html | 9 +- .../resources/io/seqera/wave/build-view.hbs | 11 +- .../io/seqera/wave/container-view.hbs | 9 +- .../resources/io/seqera/wave/mirror-view.hbs | 9 +- .../wave/core/ContainerPlatformTest.groovy | 48 +++++ .../builder/ManifestAssemblerTest.groovy | 19 -- .../MultiPlatformBuildServiceTest.groovy | 6 +- .../scan/ContainerScanServiceImplTest.groovy | 65 +++++++ .../wave/service/scan/ScanIdsTest.groovy | 178 ++++++++++++++++++ 28 files changed, 662 insertions(+), 174 deletions(-) delete mode 100644 src/main/groovy/io/seqera/wave/core/MultiContainerPlatform.groovy create mode 100644 src/main/groovy/io/seqera/wave/service/scan/ScanIds.groovy create mode 100644 src/test/groovy/io/seqera/wave/service/scan/ScanIdsTest.groovy diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index 3fa468a5a4..7aed070997 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -77,6 +77,7 @@ import io.seqera.wave.service.request.ContainerRequestService import io.seqera.wave.service.request.ContainerStatusService import io.seqera.wave.service.request.TokenData import io.seqera.wave.service.scan.ContainerScanService +import io.seqera.wave.service.scan.ScanIds import io.seqera.wave.service.validation.ValidationService import io.seqera.wave.service.validation.ValidationServiceImpl import io.seqera.wave.tower.PlatformId @@ -404,6 +405,21 @@ class ContainerController { ) } + protected String makeMultiPlatformScanId(BuildRequest build, SubmitContainerTokenRequest req) { + if( !scanService || !req.multiPlatform ) + return build.scanId + final multiPlatform = ContainerPlatform.MULTI_PLATFORM + final scanMode = req.scanMode!=null ? req.scanMode : ScanMode.async + final scanIdByPlatform = new LinkedHashMap() + for( String arch : multiPlatform.archs ) { + final platform = "${multiPlatform.os}/${arch}" as String + final id = scanService.getScanId("${build.targetImage}#${platform}", null, scanMode, req.format) + if( id ) + scanIdByPlatform.put(id, platform) + } + return scanIdByPlatform ? ScanIds.encode(scanIdByPlatform) : null + } + protected BuildTrack checkBuild(BuildRequest build, boolean dryRun) { final digest = registryProxyService.getImageDigest(build) // check for dry-run execution @@ -424,7 +440,7 @@ class ContainerController { } } - protected BuildTrack checkMultiPlatformBuild(BuildRequest templateBuild, SubmitContainerTokenRequest req, PlatformId identity, String ip) { + protected BuildTrack checkMultiPlatformBuild(BuildRequest templateBuild, SubmitContainerTokenRequest req, PlatformId identity, boolean dryRun) { final containerSpec = templateBuild.containerFile final condaContent = templateBuild.condaFile final buildRepository = ContainerCoordinates.parse(templateBuild.targetImage).repository @@ -433,6 +449,14 @@ class ContainerController { final containerId = makeMultiPlatformContainerId(containerSpec, condaContent, buildRepository, req.buildContext, req.freeze ? req.containerConfig : null) final targetImage = makeTargetImage(templateBuild.format, buildRepository, containerId, condaContent, req.nameStrategy) + // check for dry-run execution + if( dryRun ) { + log.debug "== Dry-run multi-platform build request for $targetImage" + final dryId = containerId + BuildRequest.SEP + '0' + final digest = registryProxyService.getImageDigest(targetImage, identity) + return new BuildTrack(dryId, targetImage, digest!=null, true) + } + // check if the multi-platform image already exists final digest = registryProxyService.getImageDigest(targetImage, identity) if( digest ) { @@ -486,8 +510,13 @@ class ContainerController { Boolean succeeded if( req.containerFile && req.multiPlatform ) { if( !buildService ) throw new UnsupportedBuildServiceException() - final build = makeBuildRequest(req, identity, ip) - final track = checkMultiPlatformBuild(build, req, identity, ip) + final build0 = makeBuildRequest(req, identity, ip) + // replace the single scanId with per-platform scanIds for multi-arch builds + final multiScanId = makeMultiPlatformScanId(build0, req) + final build = multiScanId != build0.scanId + ? build0.withScanId(multiScanId) + : build0 + final track = checkMultiPlatformBuild(build, req, identity, req.dryRun) targetImage = track.targetImage targetContent = build.containerFile condaContent = build.condaFile diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy index 6317f9088b..ea9a89a3ab 100644 --- a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy @@ -56,6 +56,7 @@ import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveScanRecord import io.seqera.wave.service.scan.ContainerScanService import io.seqera.wave.service.scan.ScanEntry +import io.seqera.wave.service.scan.ScanIds import io.seqera.wave.service.scan.ScanType import io.seqera.wave.service.scan.ScanVulnerability import io.seqera.wave.util.JacksonHelper @@ -137,8 +138,7 @@ class ViewController { binding.mirror_digest = result.digest ?: '-' binding.mirror_user = result.userName ?: '-' binding.put('server_url', serverUrl) - binding.scan_url = result.scanId && result.succeeded() ? "$serverUrl/view/scans/${result.scanId}" : null - binding.scan_id = result.scanId + ScanIds.populateScanBinding(binding, result.scanId, result.succeeded(), serverUrl) return binding } @@ -254,8 +254,7 @@ class ViewController { binding.build_condafile = result.condaFile binding.build_digest = result.digest ?: '-' binding.put('server_url', serverUrl) - binding.scan_url = result.scanId && result.succeeded() ? "$serverUrl/view/scans/${result.scanId}" : null - binding.scan_id = result.scanId + ScanIds.populateScanBinding(binding, result.scanId, result.succeeded(), serverUrl) // inspect uri binding.inspect_url = result.succeeded() ? "$serverUrl/view/inspect?image=${result.targetImage}&platform=${result.platform}" : null // configure build logs when available @@ -314,8 +313,7 @@ class ViewController { binding.build_url = data.buildId ? "$serverUrl/view/builds/${data.buildId}" : null binding.fusion_version = data.fusionVersion ?: '-' - binding.scan_id = data.scanId - binding.scan_url = data.scanId ? "$serverUrl/view/scans/${data.scanId}" : null + ScanIds.populateScanBinding(binding, data.scanId, true, serverUrl) binding.mirror_id = data.mirror ? data.buildId : null binding.mirror_url = data.mirror ? "$serverUrl/view/mirrors/${data.buildId}" : null diff --git a/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy b/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy index 6b90d16346..4c41629670 100644 --- a/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy +++ b/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy @@ -18,19 +18,17 @@ package io.seqera.wave.core -import groovy.transform.Canonical import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode import io.seqera.wave.exception.BadRequestException /** * Model a container platform * @author Paolo Di Tommaso */ -@Canonical +@EqualsAndHashCode @CompileStatic class ContainerPlatform { - public static final ContainerPlatform DEFAULT = new ContainerPlatform(DEFAULT_OS, DEFAULT_ARCH) - public static final List ARM64 = ['arm64', 'aarch64'] private static final List V8 = ['8','v8'] public static final List AMD64 = ['amd64', 'x86_64', 'x86-64'] @@ -38,11 +36,41 @@ class ContainerPlatform { public static final String DEFAULT_ARCH = 'amd64' public static final String DEFAULT_OS = 'linux' + public static final ContainerPlatform DEFAULT = new ContainerPlatform(DEFAULT_OS, DEFAULT_ARCH) + + /** + * A composite platform representing linux/amd64 + linux/arm64 multi-arch builds + */ + public static final ContainerPlatform MULTI_PLATFORM = new ContainerPlatform('linux', ['amd64', 'arm64']) + final String os final String arch final String variant + final List archs + + ContainerPlatform(String os, String arch, String variant=null) { + this.os = os + this.arch = arch + this.variant = variant + this.archs = List.of(arch) + } + + private ContainerPlatform(String os, List archs) { + assert archs.size() >= 2, "Multi-arch platform requires at least 2 architectures" + this.os = os + this.arch = archs[0] + this.variant = null + this.archs = List.copyOf(archs) + } + + boolean isMultiArch() { + return archs.size() > 1 + } String toString() { + if( isMultiArch() ) { + return archs.collect { "${os}/${it}" }.join(',') + } def result = os + "/" + arch if( variant ) result += "/" + variant @@ -88,6 +116,16 @@ class ContainerPlatform { if( !value ) throw new BadRequestException("Missing container platform attribute") + // handle comma-separated multi-platform values e.g. "linux/amd64,linux/arm64" + if( value.contains(',') ) { + final parts = value.tokenize(',').collect { it.trim() } + final platforms = parts.collect { of(it) } + // all platforms must share the same OS + final os = platforms[0].os + final archs = platforms.collect { it.arch } + return new ContainerPlatform(os, archs) + } + final items= value.tokenize('/') if( items.size()==1 ) items.add(0, DEFAULT_OS) diff --git a/src/main/groovy/io/seqera/wave/core/MultiContainerPlatform.groovy b/src/main/groovy/io/seqera/wave/core/MultiContainerPlatform.groovy deleted file mode 100644 index 1adaa010f0..0000000000 --- a/src/main/groovy/io/seqera/wave/core/MultiContainerPlatform.groovy +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.core - -import groovy.transform.CompileStatic -/** - * A composite container platform representing multiple architectures. - * - * @author Paolo Di Tommaso - */ -@CompileStatic -class MultiContainerPlatform extends ContainerPlatform { - - static final MultiContainerPlatform MULTI_PLATFORM = new MultiContainerPlatform( - [ContainerPlatform.of('linux/amd64'), ContainerPlatform.of('linux/arm64')] - ) - - final List platforms - - MultiContainerPlatform(List platforms) { - super(platforms[0].os, platforms[0].arch, platforms[0].variant) - this.platforms = List.copyOf(platforms) - } - - @Override - String toString() { - platforms.collect { it.toString() }.join(',') - } -} diff --git a/src/main/groovy/io/seqera/wave/proxy/ProxyClient.groovy b/src/main/groovy/io/seqera/wave/proxy/ProxyClient.groovy index c62205b188..c03f042547 100644 --- a/src/main/groovy/io/seqera/wave/proxy/ProxyClient.groovy +++ b/src/main/groovy/io/seqera/wave/proxy/ProxyClient.groovy @@ -231,7 +231,7 @@ class ProxyClient { copyHeaders(headers, builder) if( authorize ) { // add authorisation header - final header = loginService.getAuthorization(image, registry.auth, credentials) + final header = getAuthHeader() if( header ) builder.setHeader("Authorization", header) } @@ -285,10 +285,10 @@ class ProxyClient { // https://zetcode.com/java/httpclient/ final builder = HttpRequest.newBuilder(uri) .method("HEAD", HttpRequest.BodyPublishers.noBody()) - // copy headers + // copy headers copyHeaders(headers, builder) // add authorisation header - final header = loginService.getAuthorization(image, registry.auth, credentials) + final header = getAuthHeader() if( header ) builder.setHeader("Authorization", header) // build the request @@ -375,7 +375,7 @@ class ProxyClient { // copy headers copyHeaders(headers, request) // add authorisation header - final header = loginService.getAuthorization(image, registry.auth, credentials) + final header = getAuthHeader() if( header ) request.header("Authorization", header) @@ -395,7 +395,7 @@ class ProxyClient { result.add("${entry.key}: $entry.value") } // add authorisation header - final header = loginService.getAuthorization(image, registry.auth, credentials) + final header = getAuthHeader() if( header ) { result.add('-H') result.add("Authorization: $header") diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy index 365f0995b5..bc4b085f5c 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy @@ -154,11 +154,6 @@ class BuildRequest { */ final boolean noEmail - /** - * When {@code true}, this is a multi-platform composite build (linux/amd64 + linux/arm64) - */ - final boolean multiPlatform - BuildRequest( String containerId, String containerFile, @@ -201,7 +196,6 @@ class BuildRequest { this.compression = compression this.buildTemplate = buildTemplate this.noEmail = noEmail - this.multiPlatform = false // NOTE: this is meant to be updated - automatically - when the request is submitted this.buildId = computeBuildId(containerId) } @@ -231,13 +225,38 @@ class BuildRequest { this.buildId = opts.buildId ?: computeBuildId(containerId) this.buildTemplate = opts.buildTemplate this.noEmail = opts.noEmail as boolean - this.multiPlatform = opts.multiPlatform as boolean } static BuildRequest of(Map opts) { new BuildRequest(opts) } + BuildRequest withScanId(String scanId) { + return BuildRequest.of( + containerId: this.containerId, + containerFile: this.containerFile, + condaFile: this.condaFile, + workspace: this.workspace, + targetImage: this.targetImage, + identity: this.identity, + platform: this.platform, + cacheRepository: this.cacheRepository, + startTime: this.startTime, + ip: this.ip, + configJson: this.configJson, + offsetId: this.offsetId, + containerConfig: this.containerConfig, + scanId: scanId, + buildContext: this.buildContext, + format: this.format, + maxDuration: this.maxDuration, + compression: this.compression, + buildId: this.buildId, + buildTemplate: this.buildTemplate, + noEmail: this.noEmail + ) + } + @Override String toString() { return "BuildRequest[containerId=$containerId; targetImage=$targetImage; identity=$identity; dockerFile=${trunc(containerFile)}; condaFile=${trunc(condaFile)}; buildId=$buildId, maxDuration=$maxDuration]" diff --git a/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy b/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy index f01edb4cc0..d8ca6cf072 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy @@ -118,7 +118,7 @@ class ManifestAssembler { ] } ] - return JsonOutput.prettyPrint(JsonOutput.toJson(index)) + return JsonOutput.toJson(index) } protected void pushManifest(String targetImage, String indexJson, PlatformId identity) { diff --git a/src/main/groovy/io/seqera/wave/service/builder/MultiBuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/MultiBuildRequest.groovy index c04a0b7b72..fc8b954000 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/MultiBuildRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/MultiBuildRequest.groovy @@ -61,8 +61,11 @@ class MultiBuildRequest { PlatformId identity, Duration maxDuration ) { + assert containerId, "Argument 'containerId' cannot be empty" assert targetImage, "Argument 'targetImage' cannot be empty" assert buildId, "Argument 'buildId' cannot be empty" + assert amd64TargetImage, "Argument 'amd64TargetImage' cannot be empty" + assert arm64TargetImage, "Argument 'arm64TargetImage' cannot be empty" final multiBuildId = ID_PREFIX + LongRndKey.rndHex() return new MultiBuildRequest( diff --git a/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy index 52dc3c1bda..2210a97e30 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy @@ -24,7 +24,7 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.event.ApplicationEventPublisher import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.core.MultiContainerPlatform +import io.seqera.wave.model.ContainerCoordinates import io.seqera.wave.service.job.JobHandler import io.seqera.wave.service.job.JobService import io.seqera.wave.service.job.JobSpec @@ -115,18 +115,22 @@ class MultiPlatformBuildService implements JobHandler { containerFile: templateRequest.containerFile, condaFile: templateRequest.condaFile, workspace: templateRequest.workspace, - platform: MultiContainerPlatform.MULTI_PLATFORM, + platform: ContainerPlatform.MULTI_PLATFORM, configJson: templateRequest.configJson, ip: templateRequest.ip, offsetId: templateRequest.offsetId, scanId: templateRequest.scanId, format: templateRequest.format, compression: templateRequest.compression, - buildTemplate: templateRequest.buildTemplate, - multiPlatform: true + buildTemplate: templateRequest.buildTemplate ) final initialEntry = BuildEntry.create(syntheticRequest) - buildStore.storeIfAbsent(finalTargetImage, initialEntry) + final stored = buildStore.storeIfAbsent(finalTargetImage, initialEntry) + if( !stored ) { + // another request already started a build for this image — return the existing build track + log.debug "Multi-platform build already in progress for $finalTargetImage" + return new BuildTrack(buildId, finalTargetImage, false, null) + } // Create the multi-build request and entry, then launch via job queue final multiBuildRequest = MultiBuildRequest.create( @@ -228,8 +232,6 @@ class MultiPlatformBuildService implements JobHandler { // (scan record must exist before build record to avoid status polling race condition) scanService?.scanOnBuild(updatedEntry) // persist the build record and publish event (triggers email notification) - // note: the multiPlatform flag on the request survives serialization and is used - // by WaveBuildRecord.create0() to render the correct platform string final event = new BuildEvent(updatedEntry.request, result) persistenceService.saveBuildAsync(WaveBuildRecord.fromEvent(event)) eventPublisher.publishEvent(event) @@ -240,7 +242,7 @@ class MultiPlatformBuildService implements JobHandler { } protected BuildRequest createPlatformRequest(BuildRequest template, ContainerPlatform platform, String suffix) { - final repo = template.targetImage.split(':')[0] + final repo = ContainerCoordinates.parse(template.targetImage).repository final platformId = makeContainerId( template.containerFile, template.condaFile, diff --git a/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy index 86324ca5f2..3c322fdb73 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobServiceImpl.groovy @@ -153,9 +153,17 @@ class JobServiceImpl implements JobService { if( !entry ) return new JobState(JobState.Status.UNKNOWN, null, null) - // check if both sub-builds are done - final amd64Entry = buildStateStore?.getBuild(entry.request.amd64TargetImage) - final arm64Entry = buildStateStore?.getBuild(entry.request.arm64TargetImage) + // check if both sub-builds are done (skip store read for cached platforms) + final amd64Entry = entry.request.amd64Cached ? null : buildStateStore?.getBuild(entry.request.amd64TargetImage) + final arm64Entry = entry.request.arm64Cached ? null : buildStateStore?.getBuild(entry.request.arm64TargetImage) + + // fail if a non-cached sub-build entry has been evicted from the store + if( !entry.request.amd64Cached && amd64Entry == null ) { + return JobState.failed(null, "amd64 sub-build entry not found in store") + } + if( !entry.request.arm64Cached && arm64Entry == null ) { + return JobState.failed(null, "arm64 sub-build entry not found in store") + } final amd64Done = entry.request.amd64Cached || amd64Entry?.done() final arm64Done = entry.request.arm64Cached || arm64Entry?.done() diff --git a/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy index 36356ae4a9..fa1e80a147 100644 --- a/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy @@ -28,12 +28,12 @@ import io.seqera.mail.Mail import io.seqera.mail.MailAttachment import io.seqera.mail.MailHelper import io.seqera.mail.MailerConfig -import io.seqera.wave.core.MultiContainerPlatform import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildResult import io.seqera.wave.service.mail.MailService import io.seqera.wave.service.mail.MailSpooler +import io.seqera.wave.service.scan.ScanIds import jakarta.inject.Inject import jakarta.inject.Singleton import static io.seqera.wave.util.DataTimeUtils.formatDuration @@ -98,14 +98,13 @@ class MailServiceImpl implements MailService { binding.build_image = preventLinkFormatting(req.targetImage) binding.build_format = req.format?.render() ?: 'Docker' binding.build_compression = req.compression?.mode ?: '(default)' - binding.build_platform = req.multiPlatform ? MultiContainerPlatform.MULTI_PLATFORM.toString() : req.platform + binding.build_platform = req.platform binding.build_template = req.buildTemplate ?: '(default)' binding.build_containerfile = req.containerFile ?: '-' binding.build_condafile = req.condaFile binding.build_digest = result.digest ?: '-' binding.build_url = "$serverUrl/view/builds/${result.buildId}" - binding.scan_url = req.scanId && result.succeeded() ? "$serverUrl/view/scans/${req.scanId}" : null - binding.scan_id = req.scanId + ScanIds.populateScanBinding(binding, req.scanId, result.succeeded(), serverUrl) binding.put('server_url', serverUrl) // result the main object Mail mail = new Mail() diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy index 1cc02b1e6c..482edf5d06 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy @@ -26,7 +26,6 @@ import groovy.transform.EqualsAndHashCode import groovy.transform.ToString import io.seqera.wave.api.BuildCompression import io.seqera.wave.api.BuildStatusResponse -import io.seqera.wave.core.MultiContainerPlatform import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildFormat @@ -92,7 +91,7 @@ class WaveBuildRecord { userId: request.identity.user?.id, requestIp: request.ip, startTime: request.startTime, - platform: request.multiPlatform ? MultiContainerPlatform.MULTI_PLATFORM.toString() : request.platform?.toString(), + platform: request.platform?.toString(), offsetId: request.offsetId, scanId: request.scanId, format: request.format, diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy index 05dbe81ac7..012d42ab02 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveScanRecord.groovy @@ -26,7 +26,6 @@ import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString import groovy.util.logging.Slf4j -import io.seqera.wave.core.MultiContainerPlatform import io.seqera.wave.service.scan.ScanEntry import io.seqera.wave.service.scan.ScanVulnerability import io.seqera.wave.util.StringUtils @@ -98,7 +97,7 @@ class WaveScanRecord implements Cloneable { this.mirrorId = scan.mirrorId this.requestId = scan.requestId this.containerImage = scan.containerImage - this.platform = scan.multiPlatform ? MultiContainerPlatform.MULTI_PLATFORM.toString() : scan.platform?.toString() + this.platform = scan.platform?.toString() this.startTime = scan.startTime this.duration = scan.duration this.status = scan.status diff --git a/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy index c098d07f4b..b16f929fbf 100644 --- a/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy @@ -39,6 +39,7 @@ import io.seqera.wave.service.builder.ContainerBuildService import io.seqera.wave.service.mirror.ContainerMirrorService import io.seqera.wave.service.scan.ContainerScanService import io.seqera.wave.service.scan.ScanEntry +import io.seqera.wave.service.scan.ScanIds import jakarta.inject.Inject import jakarta.inject.Singleton /** @@ -104,6 +105,9 @@ class ContainerStatusServiceImpl implements ContainerStatusService { } if( request.scanId && request.scanMode == ScanMode.required && scanService ) { + if( ScanIds.isMulti(request.scanId) ) { + return handleMultiScanStatus(request, state) + } final scan = getScanState(request.scanId) if ( !scan ) throw new NotFoundException("Missing container scan record with id: ${request.scanId}") @@ -177,6 +181,43 @@ class ContainerStatusServiceImpl implements ContainerStatusService { ) } + protected ContainerStatusResponse handleMultiScanStatus(ContainerRequest request, ContainerState state) { + final entries = ScanIds.decode(request.scanId) + final List scans = new ArrayList<>(entries.size()) + boolean allDone = true + boolean allSucceeded = true + Duration maxScanDuration = Duration.ZERO + for( Map.Entry pair : entries ) { + final scan = getScanState(pair.key) + if( !scan ) + throw new NotFoundException("Missing container scan record with id: ${pair.key}") + scans.add(scan) + if( !scan.duration ) { + allDone = false + } + else { + if( scan.duration > maxScanDuration ) + maxScanDuration = scan.duration + if( !scan.succeeded() ) + allSucceeded = false + } + } + + if( !allDone ) { + return createResponse0(SCANNING, request, new ContainerState(state.startTime)) + } + + // all scans completed — pick the first failure or the last success for the response + final combinedScan = allSucceeded + ? scans.last() + : scans.find { !it.succeeded() } + // use max duration since per-arch scans run in parallel + final newState = state + ? new ContainerState(state.startTime, state.duration + maxScanDuration, allSucceeded) + : new ContainerState(combinedScan.startTime, maxScanDuration, allSucceeded) + return createScanResponse(request, newState, combinedScan) + } + protected StageResult buildResult(ContainerRequest request, ContainerState state) { if( state.succeeded ) return new StageResult(true) @@ -196,12 +237,13 @@ class ContainerStatusServiceImpl implements ContainerStatusService { } protected StageResult scanResult(ContainerRequest request, ScanEntry scan) { + final scanDetailId = scan.scanId // scan was not successful if (!scan.succeeded()) { return new StageResult( false, "Container security scan did not complete successfully", - "${serverUrl}/view/scans/${request.scanId}" + "${serverUrl}/view/scans/${scanDetailId}" ) } @@ -219,7 +261,7 @@ class ContainerStatusServiceImpl implements ContainerStatusService { return new StageResult( false, "Container security scan operation found one or more vulnerabilities with severity: ${foundLevels.join(',')}", - "${serverUrl}/view/scans/${request.scanId}" + "${serverUrl}/view/scans/${scanDetailId}" ) } @@ -227,7 +269,7 @@ class ContainerStatusServiceImpl implements ContainerStatusService { return new StageResult( true, "Container security scan operation found one or more vulnerabilities that are compatible with requested security levels: ${allowedLevels.join(',')}", - "${serverUrl}/view/scans/${request.scanId}" + "${serverUrl}/view/scans/${scanDetailId}" ) } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index b1610c576c..3f7198a36c 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -36,7 +36,7 @@ import io.micronaut.objectstorage.request.UploadRequest import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.api.ScanMode import io.seqera.wave.configuration.ScanConfig -import io.seqera.wave.core.MultiContainerPlatform +import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.configuration.ScanEnabled import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildRequest @@ -128,7 +128,15 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler pair : ScanIds.decode(entry.request.scanId) ) { + scan(fromBuild(entry.request, pair.key, ContainerPlatform.of(pair.value))) + } + } + else { + scan(fromBuild(entry.request)) + } } } catch (Exception e) { @@ -230,10 +238,13 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler, JobEntry { */ String logs - /** - * When {@code true}, this scan targets a multi-platform composite image (linux/amd64 + linux/arm64) - */ - boolean multiPlatform - @Override String getKey() { return scanId @@ -150,8 +144,7 @@ class ScanEntry implements StateEntry, JobEntry { PENDING, List.of(), null, - null, - request.multiPlatform) + null) } ScanEntry success(List vulnerabilities){ @@ -169,8 +162,7 @@ class ScanEntry implements StateEntry, JobEntry { SUCCEEDED, vulnerabilities, 0, - null, - this.multiPlatform) + null) } ScanEntry failure(Integer exitCode, String logs){ @@ -188,8 +180,7 @@ class ScanEntry implements StateEntry, JobEntry { FAILED, List.of(), exitCode, - logs, - this.multiPlatform) + logs) } static ScanEntry failure(ScanRequest request){ @@ -207,8 +198,7 @@ class ScanEntry implements StateEntry, JobEntry { FAILED, List.of(), null, - null, - request.multiPlatform) + null) } @@ -238,8 +228,7 @@ class ScanEntry implements StateEntry, JobEntry { opts.status as String, opts.vulnerabilities as List, opts.exitCode as Integer, - opts.logs as String, - opts.multiPlatform as boolean + opts.logs as String ) } @@ -250,7 +239,7 @@ class ScanEntry implements StateEntry, JobEntry { record.mirrorId, record.requestId, record.containerImage, - parsePlatform0(record.platform), + record.platform ? ContainerPlatform.of(record.platform) : null, record.workDir, null, record.startTime, @@ -258,17 +247,8 @@ class ScanEntry implements StateEntry, JobEntry { record.status, record.vulnerabilities, record.exitCode, - record.logs, - record.platform?.contains(',') ?: false + record.logs ) } - static private ContainerPlatform parsePlatform0(String value) { - if( !value ) - return null - if( value.contains(',') ) - return new MultiContainerPlatform(value.tokenize(',').collect { ContainerPlatform.of(it.trim()) }) - return ContainerPlatform.of(value) - } - } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanIds.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanIds.groovy new file mode 100644 index 0000000000..70d8581c7f --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanIds.groovy @@ -0,0 +1,122 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.scan + +import groovy.transform.CompileStatic + +/** + * Helper class for encoding/decoding multi-platform scan IDs. + * + * Single-platform scanId: "sc-abc_1" + * Multi-platform scanId: "sc-abc_1:linux/amd64,sc-def_2:linux/arm64" + * + * @author Paolo Di Tommaso + */ +@CompileStatic +class ScanIds { + + /** + * Check if the scanId represents multiple scans + */ + static boolean isMulti(String value) { + return value != null && value.contains(':') + } + + /** + * Encode a map of scanId-by-platform into a single string. + * @param scanIdByPlatform Map where keys are scanIds and values are platform strings (e.g. "linux/amd64") + * @return Encoded string e.g. "sc-abc_1:linux/amd64,sc-def_2:linux/arm64" + */ + static String encode(Map scanIdByPlatform) { + if( !scanIdByPlatform ) + return null + if( scanIdByPlatform.size() == 1 ) + return scanIdByPlatform.keySet().first() + return scanIdByPlatform + .collect { scanId, platform -> "${scanId}:${platform}" } + .join(',') + } + + /** + * Decode a scanId string into a list of (scanId, platform) pairs. + * For single-platform scanIds, returns a single entry with null platform. + */ + static List> decode(String value) { + if( !value ) + return Collections.>emptyList() + if( !isMulti(value) ) { + final Map.Entry entry = new AbstractMap.SimpleEntry(value, null) + return Collections.>singletonList(entry) + } + final List> result = new ArrayList<>() + for( String part : value.tokenize(',') ) { + final idx = part.indexOf(':') + final scanId = part.substring(0, idx) + final platform = part.substring(idx + 1) + result.add(new AbstractMap.SimpleEntry(scanId, platform)) + } + return result + } + + /** + * Get the first/primary scanId (works for both single and multi) + */ + static String primary(String value) { + if( !value ) + return null + if( !isMulti(value) ) + return value + final idx = value.indexOf(':') + return value.substring(0, idx) + } + + /** + * Get all scanIds from the encoded string + */ + static List allIds(String value) { + return decode(value).collect { it.key } + } + + /** + * Populate scan-related binding keys for view templates and emails. + * Handles both single and multi-platform scan IDs. + * + * Sets: scan_entries (list of maps), scan_url, scan_id + * + * @param binding The binding map to populate + * @param scanId The raw scanId (single or multi-platform encoded) + * @param succeeded Whether the associated build/mirror succeeded + * @param serverUrl The server base URL + */ + static void populateScanBinding(Map binding, String scanId, boolean succeeded, String serverUrl) { + if( scanId && succeeded && isMulti(scanId) ) { + binding.scan_entries = decode(scanId).collect { Map.Entry entry -> + [scan_id: entry.key, scan_platform: entry.value, scan_url: "${serverUrl}/view/scans/${entry.key}"] as Map + } + binding.scan_url = null + binding.scan_id = primary(scanId) + } + else { + binding.scan_entries = null + binding.scan_url = scanId && succeeded ? "${serverUrl}/view/scans/${scanId}" : null + binding.scan_id = scanId + } + } + +} diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy index 947a52b057..041187a196 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy @@ -86,11 +86,6 @@ class ScanRequest { */ final PlatformId identity - /** - * When {@code true}, this scan targets a multi-platform composite image (linux/amd64 + linux/arm64) - */ - final boolean multiPlatform - static ScanRequest of(Map opts) { new ScanRequest( opts.scanId as String, @@ -102,8 +97,7 @@ class ScanRequest { opts.platform as ContainerPlatform, opts.workDir as Path, opts.creationTime as Instant, - opts.identity as PlatformId, - opts.multiPlatform as boolean + opts.identity as PlatformId ) } diff --git a/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy b/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy index 6a071b2a82..ba745e1d46 100644 --- a/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy +++ b/src/main/groovy/io/seqera/wave/util/ContainerHelper.groovy @@ -298,17 +298,8 @@ class ContainerHelper { return RegHelper.sipHash(attrs) } - static final String MULTI_PLATFORM = 'linux/amd64,linux/arm64' - static String makeMultiPlatformContainerId(String containerFile, String condaFile, String repository, BuildContext buildContext, ContainerConfig containerConfig) { - final attrs = new LinkedHashMap(10) - attrs.containerFile = containerFile - attrs.condaFile = condaFile - attrs.platform = MULTI_PLATFORM - attrs.repository = repository - if( buildContext ) attrs.buildContext = buildContext.tarDigest - if( containerConfig ) attrs.containerConfig = String.valueOf(containerConfig.hashCode()) - return RegHelper.sipHash(attrs) + return makeContainerId(containerFile, condaFile, ContainerPlatform.MULTI_PLATFORM, repository, buildContext, containerConfig) } static void checkContainerSpec(String file) { diff --git a/src/main/resources/io/seqera/wave/build-notification.html b/src/main/resources/io/seqera/wave/build-notification.html index 291926eddf..268d2bfbdc 100644 --- a/src/main/resources/io/seqera/wave/build-notification.html +++ b/src/main/resources/io/seqera/wave/build-notification.html @@ -343,7 +343,14 @@

Build Summary

${build_exit_status} - <% if (scan_url) { %> + <% if (scan_entries) { %> + <% scan_entries.each { entry -> %> + + Security Scan (${entry.scan_platform}) + ${entry.scan_id} + + <% } %> + <% } else if (scan_url) { %> Security Scan ${scan_id} diff --git a/src/main/resources/io/seqera/wave/build-view.hbs b/src/main/resources/io/seqera/wave/build-view.hbs index e93feb3786..9123eebce9 100644 --- a/src/main/resources/io/seqera/wave/build-view.hbs +++ b/src/main/resources/io/seqera/wave/build-view.hbs @@ -112,7 +112,16 @@ Exit status {{build_exit_status}} - {{#if scan_url}} + {{#if scan_entries}} + {{#each scan_entries}} + + Security scan ({{scan_platform}}) + + {{scan_id}} + + + {{/each}} + {{else if scan_url}} Security scan diff --git a/src/main/resources/io/seqera/wave/container-view.hbs b/src/main/resources/io/seqera/wave/container-view.hbs index 50131951b6..848b10c973 100644 --- a/src/main/resources/io/seqera/wave/container-view.hbs +++ b/src/main/resources/io/seqera/wave/container-view.hbs @@ -244,7 +244,14 @@ {{mirror_cached}} {{/if}} - {{#if scan_id}} + {{#if scan_entries}} + {{#each scan_entries}} + + Security scan ({{scan_platform}}) + {{scan_id}} + + {{/each}} + {{else if scan_id}} Security scan {{scan_id}} diff --git a/src/main/resources/io/seqera/wave/mirror-view.hbs b/src/main/resources/io/seqera/wave/mirror-view.hbs index 52b5d40a84..b9695ef2c5 100644 --- a/src/main/resources/io/seqera/wave/mirror-view.hbs +++ b/src/main/resources/io/seqera/wave/mirror-view.hbs @@ -114,7 +114,14 @@ {{mirror_exitcode}} {{/if}} - {{#if scan_url}} + {{#if scan_entries}} + {{#each scan_entries}} + + Security scan ({{scan_platform}}) + {{scan_id}} + + {{/each}} + {{else if scan_url}} Security scan {{scan_id}} diff --git a/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy b/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy index eb27efc00e..1e7f87bacd 100644 --- a/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy +++ b/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy @@ -117,4 +117,52 @@ class ContainerPlatformTest extends Specification { 'linux/arm64/v8'| [os:'linux', architecture:'arm64', variant: 'v9'] } + + def 'should parse multi-arch platform' () { + when: + def platform = ContainerPlatform.of('linux/amd64,linux/arm64') + then: + platform.os == 'linux' + platform.arch == 'amd64' + platform.archs == ['amd64', 'arm64'] + platform.isMultiArch() + platform.toString() == 'linux/amd64,linux/arm64' + } + + def 'should detect single-arch platform' () { + when: + def platform = ContainerPlatform.of('linux/amd64') + then: + platform.os == 'linux' + platform.arch == 'amd64' + platform.archs == ['amd64'] + !platform.isMultiArch() + platform.toString() == 'linux/amd64' + } + + def 'should round-trip multi-arch through toString and of' () { + when: + def original = ContainerPlatform.of('linux/amd64,linux/arm64') + def roundTripped = ContainerPlatform.of(original.toString()) + then: + roundTripped == original + roundTripped.isMultiArch() + roundTripped.archs == ['amd64', 'arm64'] + } + + def 'should have MULTI_PLATFORM constant' () { + expect: + ContainerPlatform.MULTI_PLATFORM.isMultiArch() + ContainerPlatform.MULTI_PLATFORM.os == 'linux' + ContainerPlatform.MULTI_PLATFORM.arch == 'amd64' + ContainerPlatform.MULTI_PLATFORM.archs == ['amd64', 'arm64'] + ContainerPlatform.MULTI_PLATFORM.toString() == 'linux/amd64,linux/arm64' + } + + def 'should test equality for multi-arch' () { + expect: + ContainerPlatform.of('linux/amd64,linux/arm64') == ContainerPlatform.MULTI_PLATFORM + ContainerPlatform.of('linux/amd64,linux/arm64') == ContainerPlatform.of('linux/amd64,linux/arm64') + ContainerPlatform.of('linux/amd64') != ContainerPlatform.MULTI_PLATFORM + } } diff --git a/src/test/groovy/io/seqera/wave/service/builder/ManifestAssemblerTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/ManifestAssemblerTest.groovy index 6db0ce50f7..08f33eb2e9 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/ManifestAssemblerTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/ManifestAssemblerTest.groovy @@ -68,25 +68,6 @@ class ManifestAssemblerTest extends Specification { parsed.manifests[1].platform.os == 'linux' } - def 'should extract platform from image name'() { - expect: - ManifestAssembler.extractPlatform([:], IMAGE) == EXPECTED - - where: - IMAGE | EXPECTED - 'repo:abc123-linux-amd64' | [architecture: 'amd64', os: 'linux'] - 'docker.io/wave:tag-linux-arm64' | [architecture: 'arm64', os: 'linux'] - 'quay.io/repo:foo-linux-amd64-suffix' | [architecture: 'amd64', os: 'linux'] - } - - def 'should fail to extract platform from unknown image name'() { - when: - ManifestAssembler.extractPlatform([:], 'repo:some-tag') - - then: - thrown(IllegalStateException) - } - def 'should build valid JSON with single manifest'() { given: def manifests = [ diff --git a/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy index 07904126b5..24f9dd4996 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy @@ -46,7 +46,7 @@ class MultiPlatformBuildServiceTest extends Specification { targetImage: 'docker.io/wave:multi', startTime: Instant.now(), identity: TEST_IDENTITY, - multiPlatform: true + platform: ContainerPlatform.MULTI_PLATFORM )) } @@ -130,7 +130,7 @@ class MultiPlatformBuildServiceTest extends Specification { track.succeeded == null track.id == 'bd-multi123_0' and: - 1 * buildStore.storeIfAbsent('docker.io/wave:multi123', _) + 1 * buildStore.storeIfAbsent('docker.io/wave:multi123', _) >> true 1 * multiBuildStore.put('docker.io/wave:multi123', _) 1 * jobService.launchMultiBuild(_) } @@ -181,7 +181,7 @@ class MultiPlatformBuildServiceTest extends Specification { 1 * multiBuildStore.put('docker.io/wave:multi', _) 1 * scanService.scanOnBuild({ BuildEntry e -> e.request.targetImage == 'docker.io/wave:multi' }) 1 * persistenceService.saveBuildAsync({ WaveBuildRecord r -> r.platform == 'linux/amd64,linux/arm64' }) - 1 * eventPublisher.publishEvent({ BuildEvent e -> e.request.multiPlatform == true }) + 1 * eventPublisher.publishEvent({ BuildEvent e -> e.request.platform?.isMultiArch() }) } def 'should persist failure record when sub-build fails'() { diff --git a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy index c43c8e2190..57f54d5c3e 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy @@ -30,8 +30,10 @@ import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.api.ScanMode import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.core.ContainerPlatform +import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildFormat import io.seqera.wave.service.builder.BuildRequest +import io.seqera.wave.service.builder.BuildResult import io.seqera.wave.service.cleanup.CleanupService import io.seqera.wave.service.inspect.ContainerInspectService import io.seqera.wave.service.job.JobService @@ -432,6 +434,69 @@ class ContainerScanServiceImplTest extends Specification { 'sc-123'| 'bd-123' | false | null | ScanMode.async | true | false | 0 } + def 'should create scan request from build with overridden platform' () { + given: + def scanService = new ContainerScanServiceImpl() + def containerId = 'container1234' + and: + def workspace = Path.of('/some/workspace') + def platform = ContainerPlatform.of('linux/amd64,linux/arm64') + final build = + BuildRequest.of( + containerId: containerId, + containerFile: 'FROM ubuntu', + workspace: workspace, + targetImage: 'docker.io/my/repo:container1234', + identity: PlatformId.NULL, + platform: platform, + configJson: '{"config":"json"}', + scanId: 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64', + format: BuildFormat.DOCKER, + buildId: "${containerId}_1", + ) + + when: + def scan = scanService.fromBuild(build, 'sc-abc_1', ContainerPlatform.of('linux/amd64')) + then: + scan.scanId == 'sc-abc_1' + scan.buildId == build.buildId + scan.configJson == build.configJson + scan.targetImage == build.targetImage + scan.platform == ContainerPlatform.of('linux/amd64') + scan.workDir.startsWith(workspace) + scan.workDir.fileName.toString() == 'sc-abc_1' + } + + def 'should fan out multi-platform scans on build' () { + given: + def scanService = Spy(new ContainerScanServiceImpl()) + def containerId = 'container1234' + def workspace = Path.of('/some/workspace') + def platform = ContainerPlatform.of('linux/amd64,linux/arm64') + def multiScanId = 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64' + final build = + BuildRequest.of( + containerId: containerId, + containerFile: 'FROM ubuntu', + workspace: workspace, + targetImage: 'docker.io/my/repo:container1234', + identity: PlatformId.NULL, + platform: platform, + configJson: '{"config":"json"}', + scanId: multiScanId, + format: BuildFormat.DOCKER, + buildId: "${containerId}_1", + ) + + and: + def entry = new BuildEntry(build, BuildResult.completed(build.buildId, 0, '', Instant.now(), null)) + + when: + scanService.scanOnBuild(entry) + then: + 2 * scanService.scan(_) + } + def 'should store scan entry' () { given: def duration = Duration.ofSeconds(1) diff --git a/src/test/groovy/io/seqera/wave/service/scan/ScanIdsTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ScanIdsTest.groovy new file mode 100644 index 0000000000..8fcf936b60 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/service/scan/ScanIdsTest.groovy @@ -0,0 +1,178 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.scan + +import spock.lang.Specification +import spock.lang.Unroll + +/** + * Tests for {@link ScanIds} + * + * @author Paolo Di Tommaso + */ +class ScanIdsTest extends Specification { + + @Unroll + def 'should detect multi scanId: #VALUE'() { + expect: + ScanIds.isMulti(VALUE) == EXPECTED + + where: + VALUE | EXPECTED + null | false + 'sc-abc_1' | false + 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64' | true + 'sc-abc_1:linux/amd64' | true + } + + def 'should encode single scanId'() { + expect: + ScanIds.encode(['sc-abc_1': 'linux/amd64']) == 'sc-abc_1' + } + + def 'should encode multiple scanIds'() { + given: + def map = new LinkedHashMap() + map.put('sc-abc_1', 'linux/amd64') + map.put('sc-def_2', 'linux/arm64') + + expect: + ScanIds.encode(map) == 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64' + } + + def 'should encode null or empty'() { + expect: + ScanIds.encode(null) == null + ScanIds.encode([:]) == null + } + + def 'should decode single scanId'() { + when: + def result = ScanIds.decode('sc-abc_1') + then: + result.size() == 1 + result[0].key == 'sc-abc_1' + result[0].value == null + } + + def 'should decode multi scanId'() { + when: + def result = ScanIds.decode('sc-abc_1:linux/amd64,sc-def_2:linux/arm64') + then: + result.size() == 2 + result[0].key == 'sc-abc_1' + result[0].value == 'linux/amd64' + result[1].key == 'sc-def_2' + result[1].value == 'linux/arm64' + } + + def 'should decode null or empty'() { + expect: + ScanIds.decode(null) == [] + ScanIds.decode('') == [] + } + + @Unroll + def 'should get primary scanId from: #VALUE'() { + expect: + ScanIds.primary(VALUE) == EXPECTED + + where: + VALUE | EXPECTED + null | null + 'sc-abc_1' | 'sc-abc_1' + 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64' | 'sc-abc_1' + } + + def 'should get all ids'() { + expect: + ScanIds.allIds('sc-abc_1') == ['sc-abc_1'] + ScanIds.allIds('sc-abc_1:linux/amd64,sc-def_2:linux/arm64') == ['sc-abc_1', 'sc-def_2'] + } + + def 'should roundtrip encode/decode'() { + given: + def map = new LinkedHashMap() + map.put('sc-abc_1', 'linux/amd64') + map.put('sc-def_2', 'linux/arm64') + + when: + def encoded = ScanIds.encode(map) + def decoded = ScanIds.decode(encoded) + + then: + decoded.size() == 2 + decoded[0].key == 'sc-abc_1' + decoded[0].value == 'linux/amd64' + decoded[1].key == 'sc-def_2' + decoded[1].value == 'linux/arm64' + } + + def 'should populate scan binding for single scanId'() { + given: + def binding = new HashMap() + + when: + ScanIds.populateScanBinding(binding, 'sc-abc_1', true, 'https://wave.io') + then: + binding.scan_id == 'sc-abc_1' + binding.scan_url == 'https://wave.io/view/scans/sc-abc_1' + binding.scan_entries == null + } + + def 'should populate scan binding for multi scanId'() { + given: + def binding = new HashMap() + + when: + ScanIds.populateScanBinding(binding, 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64', true, 'https://wave.io') + then: + binding.scan_id == 'sc-abc_1' + binding.scan_url == null + binding.scan_entries.size() == 2 + binding.scan_entries[0].scan_id == 'sc-abc_1' + binding.scan_entries[0].scan_platform == 'linux/amd64' + binding.scan_entries[0].scan_url == 'https://wave.io/view/scans/sc-abc_1' + binding.scan_entries[1].scan_id == 'sc-def_2' + binding.scan_entries[1].scan_platform == 'linux/arm64' + } + + def 'should populate scan binding when not succeeded'() { + given: + def binding = new HashMap() + + when: + ScanIds.populateScanBinding(binding, 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64', false, 'https://wave.io') + then: + binding.scan_id == 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64' + binding.scan_url == null + binding.scan_entries == null + } + + def 'should populate scan binding when scanId is null'() { + given: + def binding = new HashMap() + + when: + ScanIds.populateScanBinding(binding, null, true, 'https://wave.io') + then: + binding.scan_id == null + binding.scan_url == null + } +} From 8fa40aec5f0e5de8ac434e0d565f578316b92479 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Mar 2026 18:47:41 +0100 Subject: [PATCH 05/15] Replace ScanIds with ChildEntries value type and add dedicated scanChildIds field - Create ChildEntries as a shared value type for encoding/decoding per-platform child IDs (builds and scans), with Jackson serialization support - Stop overloading scanId with multi-platform encoded IDs; add dedicated scanChildIds field to BuildRequest, ContainerRequest, WaveBuildRecord, and WaveContainerRecord - Change buildChildIds from String to ChildEntries type across all data classes - Remove scan/build child entries and scan info from mail notifications - Delete ScanIds utility class and its tests, replaced by ChildEntries Co-Authored-By: Claude Opus 4.6 --- .../controller/ContainerController.groovy | 19 +- .../wave/controller/ViewController.groovy | 9 +- .../io/seqera/wave/core/ChildEntries.groovy | 174 ++++++++++++++ .../wave/service/builder/BuildRequest.groovy | 45 +++- .../builder/MultiPlatformBuildService.groovy | 8 +- .../service/mail/impl/MailServiceImpl.groovy | 2 - .../persistence/WaveBuildRecord.groovy | 7 +- .../persistence/WaveContainerRecord.groovy | 8 + .../service/request/ContainerRequest.groovy | 10 + .../request/ContainerStatusServiceImpl.groovy | 5 +- .../scan/ContainerScanServiceImpl.groovy | 8 +- .../seqera/wave/service/scan/ScanIds.groovy | 122 ---------- .../io/seqera/wave/build-notification.html | 13 -- .../resources/io/seqera/wave/build-view.hbs | 10 + .../seqera/wave/core/ChildEntriesTest.groovy | 218 ++++++++++++++++++ .../request/ContainerRequestTest.groovy | 1 + .../scan/ContainerScanServiceImplTest.groovy | 5 +- .../wave/service/scan/ScanIdsTest.groovy | 178 -------------- 18 files changed, 502 insertions(+), 340 deletions(-) create mode 100644 src/main/groovy/io/seqera/wave/core/ChildEntries.groovy delete mode 100644 src/main/groovy/io/seqera/wave/service/scan/ScanIds.groovy create mode 100644 src/test/groovy/io/seqera/wave/core/ChildEntriesTest.groovy delete mode 100644 src/test/groovy/io/seqera/wave/service/scan/ScanIdsTest.groovy diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index 7aed070997..cec74b5d9c 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -77,7 +77,7 @@ import io.seqera.wave.service.request.ContainerRequestService import io.seqera.wave.service.request.ContainerStatusService import io.seqera.wave.service.request.TokenData import io.seqera.wave.service.scan.ContainerScanService -import io.seqera.wave.service.scan.ScanIds +import io.seqera.wave.core.ChildEntries import io.seqera.wave.service.validation.ValidationService import io.seqera.wave.service.validation.ValidationServiceImpl import io.seqera.wave.tower.PlatformId @@ -405,9 +405,9 @@ class ContainerController { ) } - protected String makeMultiPlatformScanId(BuildRequest build, SubmitContainerTokenRequest req) { + protected ChildEntries makeChildScanIds(BuildRequest build, SubmitContainerTokenRequest req) { if( !scanService || !req.multiPlatform ) - return build.scanId + return null final multiPlatform = ContainerPlatform.MULTI_PLATFORM final scanMode = req.scanMode!=null ? req.scanMode : ScanMode.async final scanIdByPlatform = new LinkedHashMap() @@ -417,7 +417,7 @@ class ContainerController { if( id ) scanIdByPlatform.put(id, platform) } - return scanIdByPlatform ? ScanIds.encode(scanIdByPlatform) : null + return ChildEntries.of(scanIdByPlatform) } protected BuildTrack checkBuild(BuildRequest build, boolean dryRun) { @@ -507,15 +507,14 @@ class ContainerController { String buildId boolean buildNew String scanId + ChildEntries scanChildIds Boolean succeeded if( req.containerFile && req.multiPlatform ) { if( !buildService ) throw new UnsupportedBuildServiceException() final build0 = makeBuildRequest(req, identity, ip) - // replace the single scanId with per-platform scanIds for multi-arch builds - final multiScanId = makeMultiPlatformScanId(build0, req) - final build = multiScanId != build0.scanId - ? build0.withScanId(multiScanId) - : build0 + // create per-platform scan IDs for multi-arch builds + final childScans = makeChildScanIds(build0, req) + final build = childScans ? build0.withChildScanIds(childScans) : build0 final track = checkMultiPlatformBuild(build, req, identity, req.dryRun) targetImage = track.targetImage targetContent = build.containerFile @@ -523,6 +522,7 @@ class ContainerController { buildId = track.id buildNew = !track.cached scanId = build.scanId + scanChildIds = build.scanChildIds succeeded = track.succeeded type = ContainerRequest.Type.Build } @@ -580,6 +580,7 @@ class ContainerController { buildNew, req.freeze, scanId, + scanChildIds, req.scanMode, req.scanLevels, req.dryRun, diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy index ea9a89a3ab..e0fe778763 100644 --- a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy @@ -56,7 +56,7 @@ import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveScanRecord import io.seqera.wave.service.scan.ContainerScanService import io.seqera.wave.service.scan.ScanEntry -import io.seqera.wave.service.scan.ScanIds +import io.seqera.wave.core.ChildEntries import io.seqera.wave.service.scan.ScanType import io.seqera.wave.service.scan.ScanVulnerability import io.seqera.wave.util.JacksonHelper @@ -138,7 +138,7 @@ class ViewController { binding.mirror_digest = result.digest ?: '-' binding.mirror_user = result.userName ?: '-' binding.put('server_url', serverUrl) - ScanIds.populateScanBinding(binding, result.scanId, result.succeeded(), serverUrl) + ChildEntries.populateScanBinding(binding, result.scanId, null, result.succeeded(), serverUrl) return binding } @@ -254,7 +254,8 @@ class ViewController { binding.build_condafile = result.condaFile binding.build_digest = result.digest ?: '-' binding.put('server_url', serverUrl) - ScanIds.populateScanBinding(binding, result.scanId, result.succeeded(), serverUrl) + ChildEntries.populateScanBinding(binding, result.scanId, result.scanChildIds, result.succeeded(), serverUrl) + ChildEntries.populateBuildBinding(binding, result.buildChildIds, serverUrl) // inspect uri binding.inspect_url = result.succeeded() ? "$serverUrl/view/inspect?image=${result.targetImage}&platform=${result.platform}" : null // configure build logs when available @@ -313,7 +314,7 @@ class ViewController { binding.build_url = data.buildId ? "$serverUrl/view/builds/${data.buildId}" : null binding.fusion_version = data.fusionVersion ?: '-' - ScanIds.populateScanBinding(binding, data.scanId, true, serverUrl) + ChildEntries.populateScanBinding(binding, data.scanId, data.scanChildIds, true, serverUrl) binding.mirror_id = data.mirror ? data.buildId : null binding.mirror_url = data.mirror ? "$serverUrl/view/mirrors/${data.buildId}" : null diff --git a/src/main/groovy/io/seqera/wave/core/ChildEntries.groovy b/src/main/groovy/io/seqera/wave/core/ChildEntries.groovy new file mode 100644 index 0000000000..cc98b2e65c --- /dev/null +++ b/src/main/groovy/io/seqera/wave/core/ChildEntries.groovy @@ -0,0 +1,174 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.core + +import com.fasterxml.jackson.annotation.JsonCreator +import com.fasterxml.jackson.annotation.JsonValue +import groovy.transform.CompileStatic + +/** + * Value type representing a set of per-platform child IDs (builds or scans). + * + * Serialises to/from an encoded string: + * "id1:platform1,id2:platform2" + * + * Example: "sc-abc_1:linux/amd64,sc-def_2:linux/arm64" + * + * Used as the type for both {@code buildChildIds} and {@code scanChildIds}. + * + * @author Paolo Di Tommaso + */ +@CompileStatic +class ChildEntries { + + /** + * The encoded string representation + */ + private final String encoded + + @JsonCreator + ChildEntries(String encoded) { + this.encoded = encoded + } + + /** + * Create from a map of id-by-platform. + * @param idByPlatform Map where keys are IDs and values are platform strings (e.g. "linux/amd64") + * @return A new ChildEntries, or null if the map is null/empty + */ + static ChildEntries of(Map idByPlatform) { + if( !idByPlatform ) + return null + if( idByPlatform.size() == 1 ) { + final entry = idByPlatform.entrySet().first() + return new ChildEntries("${entry.key}:${entry.value}") + } + final encoded = idByPlatform + .collect { id, platform -> "${id}:${platform}" } + .join(',') + return new ChildEntries(encoded) + } + + /** + * Decode into a list of (id, platform) pairs. + */ + List> decode() { + if( !encoded ) + return Collections.>emptyList() + final List> result = new ArrayList<>() + for( String part : encoded.tokenize(',') ) { + final idx = part.indexOf(':') + if( idx < 0 ) { + result.add(new AbstractMap.SimpleEntry(part, null)) + } + else { + final id = part.substring(0, idx) + final platform = part.substring(idx + 1) + result.add(new AbstractMap.SimpleEntry(id, platform)) + } + } + return result + } + + /** + * Get the first/primary ID + */ + String primary() { + if( !encoded ) + return null + final idx = encoded.indexOf(':') + if( idx < 0 ) + return encoded + return encoded.substring(0, idx) + } + + /** + * Get all IDs + */ + List allIds() { + return decode().collect { it.key } + } + + /** + * Jackson serialisation: serialise as the encoded string + */ + @JsonValue + String toString() { + return encoded + } + + /** + * Groovy truth: non-null and non-empty encoded string + */ + boolean asBoolean() { + return encoded != null && !encoded.isEmpty() + } + + @Override + boolean equals(Object o) { + if( this.is(o) ) return true + if( o == null || getClass() != o.getClass() ) return false + return encoded == ((ChildEntries) o).encoded + } + + @Override + int hashCode() { + return encoded != null ? encoded.hashCode() : 0 + } + + // -- template binding helpers -- + + /** + * Populate scan-related binding keys for view templates and emails. + * Handles both single-platform (scanId) and multi-platform (scanChildIds) scan IDs. + * + * Sets: scan_entries (list of maps), scan_url, scan_id + */ + static void populateScanBinding(Map binding, String scanId, ChildEntries scanChildIds, boolean succeeded, String serverUrl) { + if( scanChildIds && succeeded ) { + binding.scan_entries = scanChildIds.decode().collect { Map.Entry entry -> + [scan_id: entry.key, scan_platform: entry.value, scan_url: "${serverUrl}/view/scans/${entry.key}"] as Map + } + binding.scan_url = null + binding.scan_id = scanChildIds.primary() + } + else { + binding.scan_entries = null + binding.scan_url = scanId && succeeded ? "${serverUrl}/view/scans/${scanId}" : null + binding.scan_id = scanId + } + } + + /** + * Populate build-related binding keys for child (per-arch) build entries. + * + * Sets: build_entries (list of maps with build_id, build_platform, build_url) + */ + static void populateBuildBinding(Map binding, ChildEntries buildChildIds, String serverUrl) { + if( buildChildIds ) { + binding.build_entries = buildChildIds.decode().collect { Map.Entry entry -> + [build_id: entry.key, build_platform: entry.value, build_url: "${serverUrl}/view/builds/${entry.key}"] as Map + } + } + else { + binding.build_entries = null + } + } + +} diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy index bc4b085f5c..165c0e0c0c 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy @@ -29,6 +29,7 @@ import groovy.transform.EqualsAndHashCode import io.seqera.wave.api.BuildCompression import io.seqera.wave.api.BuildContext import io.seqera.wave.api.ContainerConfig +import io.seqera.wave.core.ChildEntries import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.tower.PlatformId import static io.seqera.wave.service.builder.BuildFormat.DOCKER @@ -154,6 +155,16 @@ class BuildRequest { */ final boolean noEmail + /** + * Child build IDs for multi-platform builds + */ + final ChildEntries buildChildIds + + /** + * Child scan IDs for multi-platform builds + */ + final ChildEntries scanChildIds + BuildRequest( String containerId, String containerFile, @@ -225,6 +236,8 @@ class BuildRequest { this.buildId = opts.buildId ?: computeBuildId(containerId) this.buildTemplate = opts.buildTemplate this.noEmail = opts.noEmail as boolean + this.buildChildIds = opts.buildChildIds as ChildEntries + this.scanChildIds = opts.scanChildIds as ChildEntries } static BuildRequest of(Map opts) { @@ -253,7 +266,37 @@ class BuildRequest { compression: this.compression, buildId: this.buildId, buildTemplate: this.buildTemplate, - noEmail: this.noEmail + noEmail: this.noEmail, + buildChildIds: this.buildChildIds, + scanChildIds: this.scanChildIds + ) + } + + BuildRequest withChildScanIds(ChildEntries scanChildIds) { + return BuildRequest.of( + containerId: this.containerId, + containerFile: this.containerFile, + condaFile: this.condaFile, + workspace: this.workspace, + targetImage: this.targetImage, + identity: this.identity, + platform: this.platform, + cacheRepository: this.cacheRepository, + startTime: this.startTime, + ip: this.ip, + configJson: this.configJson, + offsetId: this.offsetId, + containerConfig: this.containerConfig, + scanId: this.scanId, + buildContext: this.buildContext, + format: this.format, + maxDuration: this.maxDuration, + compression: this.compression, + buildId: this.buildId, + buildTemplate: this.buildTemplate, + noEmail: this.noEmail, + buildChildIds: this.buildChildIds, + scanChildIds: scanChildIds ) } diff --git a/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy index 2210a97e30..3abe7ebb6a 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy @@ -23,6 +23,7 @@ import java.time.Instant import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.event.ApplicationEventPublisher +import io.seqera.wave.core.ChildEntries import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.model.ContainerCoordinates import io.seqera.wave.service.job.JobHandler @@ -105,6 +106,9 @@ class MultiPlatformBuildService implements JobHandler { final buildId = BuildRequest.ID_PREFIX + finalContainerId + BuildRequest.SEP + '0' final startTime = Instant.now() + // Encode child build IDs for the parent build view + final buildChildIds = new ChildEntries("${amd64Track.id}:${PLATFORM_AMD64},${arm64Track.id}:${PLATFORM_ARM64}") + // Store an in-progress build entry so status polling can find it final syntheticRequest = BuildRequest.of( buildId: buildId, @@ -122,7 +126,9 @@ class MultiPlatformBuildService implements JobHandler { scanId: templateRequest.scanId, format: templateRequest.format, compression: templateRequest.compression, - buildTemplate: templateRequest.buildTemplate + buildTemplate: templateRequest.buildTemplate, + buildChildIds: buildChildIds, + scanChildIds: templateRequest.scanChildIds ) final initialEntry = BuildEntry.create(syntheticRequest) final stored = buildStore.storeIfAbsent(finalTargetImage, initialEntry) diff --git a/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy index fa1e80a147..7eb3c47313 100644 --- a/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/mail/impl/MailServiceImpl.groovy @@ -33,7 +33,6 @@ import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.builder.BuildResult import io.seqera.wave.service.mail.MailService import io.seqera.wave.service.mail.MailSpooler -import io.seqera.wave.service.scan.ScanIds import jakarta.inject.Inject import jakarta.inject.Singleton import static io.seqera.wave.util.DataTimeUtils.formatDuration @@ -104,7 +103,6 @@ class MailServiceImpl implements MailService { binding.build_condafile = req.condaFile binding.build_digest = result.digest ?: '-' binding.build_url = "$serverUrl/view/builds/${result.buildId}" - ScanIds.populateScanBinding(binding, req.scanId, result.succeeded(), serverUrl) binding.put('server_url', serverUrl) // result the main object Mail mail = new Mail() diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy index 482edf5d06..0d3c316ece 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy @@ -26,6 +26,7 @@ import groovy.transform.EqualsAndHashCode import groovy.transform.ToString import io.seqera.wave.api.BuildCompression import io.seqera.wave.api.BuildStatusResponse +import io.seqera.wave.core.ChildEntries import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildFormat @@ -61,6 +62,8 @@ class WaveBuildRecord { String digest BuildCompression compression String buildTemplate + ChildEntries buildChildIds + ChildEntries scanChildIds Boolean succeeded() { return duration != null ? (exitStatus==0) : null @@ -99,7 +102,9 @@ class WaveBuildRecord { exitStatus: result?.exitStatus, digest: result?.digest, compression: request?.compression, - buildTemplate: request?.buildTemplate + buildTemplate: request?.buildTemplate, + buildChildIds: request?.buildChildIds, + scanChildIds: request?.scanChildIds ) } diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy index e0ee15ddfa..d8c167715e 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy @@ -28,6 +28,7 @@ import groovy.transform.ToString import groovy.util.logging.Slf4j import io.seqera.wave.api.ContainerConfig import io.seqera.wave.api.SubmitContainerTokenRequest +import io.seqera.wave.core.ChildEntries import io.seqera.wave.service.request.ContainerRequest import io.seqera.wave.tower.User import io.seqera.wave.util.FusionVersionStringDeserializer @@ -177,6 +178,11 @@ class WaveContainerRecord { */ final String scanId + /** + * Child scan IDs for multi-platform builds + */ + final ChildEntries scanChildIds + WaveContainerRecord(SubmitContainerTokenRequest request, ContainerRequest data, String waveImage, String addr, Instant expiration) { this.id = data.requestId this.user = data.identity.user @@ -203,6 +209,7 @@ class WaveContainerRecord { this.fusionVersion = request?.containerConfig?.fusionVersion()?.number this.mirror = data.mirror this.scanId = data.scanId + this.scanChildIds = data.scanChildIds } WaveContainerRecord(WaveContainerRecord that, String sourceDigest, String waveDigest) { @@ -229,6 +236,7 @@ class WaveContainerRecord { this.fusionVersion = that.fusionVersion this.mirror == that.mirror this.scanId = that.scanId + this.scanChildIds = that.scanChildIds // -- digest part this.sourceDigest = sourceDigest this.waveDigest = waveDigest diff --git a/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy index c59898b4bd..7ab2b8b89f 100644 --- a/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy @@ -25,6 +25,7 @@ import groovy.transform.CompileStatic import io.seqera.wave.api.ContainerConfig import io.seqera.wave.api.ScanLevel import io.seqera.wave.api.ScanMode +import io.seqera.wave.core.ChildEntries import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.model.ContainerCoordinates import io.seqera.wave.tower.PlatformId @@ -53,6 +54,7 @@ class ContainerRequest { Boolean buildNew Boolean freeze String scanId + ChildEntries scanChildIds ScanMode scanMode List scanLevels boolean dryRun @@ -88,6 +90,7 @@ class ContainerRequest { "buildNew=${buildNew}; " + "freeze=${freeze}; " + "scanId=${scanId}; " + + "scanChildIds=${scanChildIds}; " + "scanMode=${scanMode}; " + "scanLevels=${scanLevels}; " + "dryRun=${dryRun}; " + @@ -144,6 +147,10 @@ class ContainerRequest { return scanId } + ChildEntries getChildScanIds() { + return scanChildIds + } + ScanMode getScanMode() { return scanMode } @@ -176,6 +183,7 @@ class ContainerRequest { Boolean buildNew, Boolean freeze, String scanId, + ChildEntries scanChildIds, ScanMode scanMode, List scanLevels, boolean dryRun, @@ -196,6 +204,7 @@ class ContainerRequest { buildNew, freeze, scanId, + scanChildIds, scanMode, scanLevels, dryRun, @@ -222,6 +231,7 @@ class ContainerRequest { (Boolean) data.buildNew, (Boolean) data.freeze, data.scanId as String, + data.scanChildIds as ChildEntries, data.scanMode as ScanMode, data.scanLevels as List, data.dryRun as boolean, diff --git a/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy index b16f929fbf..a48f21da68 100644 --- a/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy @@ -39,7 +39,6 @@ import io.seqera.wave.service.builder.ContainerBuildService import io.seqera.wave.service.mirror.ContainerMirrorService import io.seqera.wave.service.scan.ContainerScanService import io.seqera.wave.service.scan.ScanEntry -import io.seqera.wave.service.scan.ScanIds import jakarta.inject.Inject import jakarta.inject.Singleton /** @@ -105,7 +104,7 @@ class ContainerStatusServiceImpl implements ContainerStatusService { } if( request.scanId && request.scanMode == ScanMode.required && scanService ) { - if( ScanIds.isMulti(request.scanId) ) { + if( request.scanChildIds ) { return handleMultiScanStatus(request, state) } final scan = getScanState(request.scanId) @@ -182,7 +181,7 @@ class ContainerStatusServiceImpl implements ContainerStatusService { } protected ContainerStatusResponse handleMultiScanStatus(ContainerRequest request, ContainerState state) { - final entries = ScanIds.decode(request.scanId) + final entries = request.scanChildIds.decode() final List scans = new ArrayList<>(entries.size()) boolean allDone = true boolean allSucceeded = true diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index 3f7198a36c..5cf8fb5ec9 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -127,14 +127,14 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler pair : ScanIds.decode(entry.request.scanId) ) { + for( Map.Entry pair : entry.request.scanChildIds.decode() ) { scan(fromBuild(entry.request, pair.key, ContainerPlatform.of(pair.value))) } } - else { + else if( entry.request.scanId ) { scan(fromBuild(entry.request)) } } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanIds.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanIds.groovy deleted file mode 100644 index 70d8581c7f..0000000000 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanIds.groovy +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.scan - -import groovy.transform.CompileStatic - -/** - * Helper class for encoding/decoding multi-platform scan IDs. - * - * Single-platform scanId: "sc-abc_1" - * Multi-platform scanId: "sc-abc_1:linux/amd64,sc-def_2:linux/arm64" - * - * @author Paolo Di Tommaso - */ -@CompileStatic -class ScanIds { - - /** - * Check if the scanId represents multiple scans - */ - static boolean isMulti(String value) { - return value != null && value.contains(':') - } - - /** - * Encode a map of scanId-by-platform into a single string. - * @param scanIdByPlatform Map where keys are scanIds and values are platform strings (e.g. "linux/amd64") - * @return Encoded string e.g. "sc-abc_1:linux/amd64,sc-def_2:linux/arm64" - */ - static String encode(Map scanIdByPlatform) { - if( !scanIdByPlatform ) - return null - if( scanIdByPlatform.size() == 1 ) - return scanIdByPlatform.keySet().first() - return scanIdByPlatform - .collect { scanId, platform -> "${scanId}:${platform}" } - .join(',') - } - - /** - * Decode a scanId string into a list of (scanId, platform) pairs. - * For single-platform scanIds, returns a single entry with null platform. - */ - static List> decode(String value) { - if( !value ) - return Collections.>emptyList() - if( !isMulti(value) ) { - final Map.Entry entry = new AbstractMap.SimpleEntry(value, null) - return Collections.>singletonList(entry) - } - final List> result = new ArrayList<>() - for( String part : value.tokenize(',') ) { - final idx = part.indexOf(':') - final scanId = part.substring(0, idx) - final platform = part.substring(idx + 1) - result.add(new AbstractMap.SimpleEntry(scanId, platform)) - } - return result - } - - /** - * Get the first/primary scanId (works for both single and multi) - */ - static String primary(String value) { - if( !value ) - return null - if( !isMulti(value) ) - return value - final idx = value.indexOf(':') - return value.substring(0, idx) - } - - /** - * Get all scanIds from the encoded string - */ - static List allIds(String value) { - return decode(value).collect { it.key } - } - - /** - * Populate scan-related binding keys for view templates and emails. - * Handles both single and multi-platform scan IDs. - * - * Sets: scan_entries (list of maps), scan_url, scan_id - * - * @param binding The binding map to populate - * @param scanId The raw scanId (single or multi-platform encoded) - * @param succeeded Whether the associated build/mirror succeeded - * @param serverUrl The server base URL - */ - static void populateScanBinding(Map binding, String scanId, boolean succeeded, String serverUrl) { - if( scanId && succeeded && isMulti(scanId) ) { - binding.scan_entries = decode(scanId).collect { Map.Entry entry -> - [scan_id: entry.key, scan_platform: entry.value, scan_url: "${serverUrl}/view/scans/${entry.key}"] as Map - } - binding.scan_url = null - binding.scan_id = primary(scanId) - } - else { - binding.scan_entries = null - binding.scan_url = scanId && succeeded ? "${serverUrl}/view/scans/${scanId}" : null - binding.scan_id = scanId - } - } - -} diff --git a/src/main/resources/io/seqera/wave/build-notification.html b/src/main/resources/io/seqera/wave/build-notification.html index 268d2bfbdc..9d3f93540b 100644 --- a/src/main/resources/io/seqera/wave/build-notification.html +++ b/src/main/resources/io/seqera/wave/build-notification.html @@ -343,19 +343,6 @@

Build Summary

${build_exit_status} - <% if (scan_entries) { %> - <% scan_entries.each { entry -> %> - - Security Scan (${entry.scan_platform}) - ${entry.scan_id} - - <% } %> - <% } else if (scan_url) { %> - - Security Scan - ${scan_id} - - <% } %> diff --git a/src/main/resources/io/seqera/wave/build-view.hbs b/src/main/resources/io/seqera/wave/build-view.hbs index 9123eebce9..d4f2e2f161 100644 --- a/src/main/resources/io/seqera/wave/build-view.hbs +++ b/src/main/resources/io/seqera/wave/build-view.hbs @@ -96,6 +96,16 @@ Platform {{build_platform}} + {{#if build_entries}} + {{#each build_entries}} + + Build ({{build_platform}}) + + {{build_id}} + + + {{/each}} + {{/if}} Build template {{build_template}} diff --git a/src/test/groovy/io/seqera/wave/core/ChildEntriesTest.groovy b/src/test/groovy/io/seqera/wave/core/ChildEntriesTest.groovy new file mode 100644 index 0000000000..020a51324e --- /dev/null +++ b/src/test/groovy/io/seqera/wave/core/ChildEntriesTest.groovy @@ -0,0 +1,218 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.core + +import spock.lang.Specification +import spock.lang.Unroll + +/** + * Tests for {@link ChildEntries} + * + * @author Paolo Di Tommaso + */ +class ChildEntriesTest extends Specification { + + def 'should create from map with single entry'() { + expect: + ChildEntries.of(['sc-abc_1': 'linux/amd64']).toString() == 'sc-abc_1:linux/amd64' + } + + def 'should create from map with multiple entries'() { + given: + def map = new LinkedHashMap() + map.put('sc-abc_1', 'linux/amd64') + map.put('sc-def_2', 'linux/arm64') + + expect: + ChildEntries.of(map).toString() == 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64' + } + + def 'should return null from null or empty map'() { + expect: + ChildEntries.of(null) == null + ChildEntries.of([:]) == null + } + + def 'should decode single entry'() { + when: + def result = new ChildEntries('sc-abc_1:linux/amd64').decode() + then: + result.size() == 1 + result[0].key == 'sc-abc_1' + result[0].value == 'linux/amd64' + } + + def 'should decode bare id (no platform)'() { + when: + def result = new ChildEntries('sc-abc_1').decode() + then: + result.size() == 1 + result[0].key == 'sc-abc_1' + result[0].value == null + } + + def 'should decode multi entries'() { + when: + def result = new ChildEntries('sc-abc_1:linux/amd64,sc-def_2:linux/arm64').decode() + then: + result.size() == 2 + result[0].key == 'sc-abc_1' + result[0].value == 'linux/amd64' + result[1].key == 'sc-def_2' + result[1].value == 'linux/arm64' + } + + @Unroll + def 'should get primary id from: #ENCODED'() { + expect: + new ChildEntries(ENCODED).primary() == EXPECTED + + where: + ENCODED | EXPECTED + 'sc-abc_1' | 'sc-abc_1' + 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64' | 'sc-abc_1' + } + + def 'should get all ids'() { + expect: + new ChildEntries('sc-abc_1').allIds() == ['sc-abc_1'] + new ChildEntries('sc-abc_1:linux/amd64,sc-def_2:linux/arm64').allIds() == ['sc-abc_1', 'sc-def_2'] + } + + def 'should roundtrip via of and decode'() { + given: + def map = new LinkedHashMap() + map.put('sc-abc_1', 'linux/amd64') + map.put('sc-def_2', 'linux/arm64') + + when: + def entries = ChildEntries.of(map) + def decoded = entries.decode() + + then: + decoded.size() == 2 + decoded[0].key == 'sc-abc_1' + decoded[0].value == 'linux/amd64' + decoded[1].key == 'sc-def_2' + decoded[1].value == 'linux/arm64' + } + + def 'should serialize to string via toString'() { + given: + def entries = new ChildEntries('sc-abc_1:linux/amd64,sc-def_2:linux/arm64') + expect: + entries.toString() == 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64' + } + + def 'should support groovy truth'() { + expect: + new ChildEntries('sc-abc_1') as boolean == true + new ChildEntries(null) as boolean == false + new ChildEntries('') as boolean == false + } + + def 'should support equality'() { + expect: + new ChildEntries('sc-abc_1:linux/amd64') == new ChildEntries('sc-abc_1:linux/amd64') + !new ChildEntries('sc-abc_1').equals(new ChildEntries('sc-def_2')) + } + + // -- template binding tests -- + + def 'should populate scan binding for single scanId'() { + given: + def binding = new HashMap() + + when: + ChildEntries.populateScanBinding(binding, 'sc-abc_1', null, true, 'https://wave.io') + then: + binding.scan_id == 'sc-abc_1' + binding.scan_url == 'https://wave.io/view/scans/sc-abc_1' + binding.scan_entries == null + } + + def 'should populate scan binding for multi-platform scanChildIds'() { + given: + def binding = new HashMap() + def scanChildIds = new ChildEntries('sc-abc_1:linux/amd64,sc-def_2:linux/arm64') + + when: + ChildEntries.populateScanBinding(binding, null, scanChildIds, true, 'https://wave.io') + then: + binding.scan_id == 'sc-abc_1' + binding.scan_url == null + binding.scan_entries.size() == 2 + binding.scan_entries[0].scan_id == 'sc-abc_1' + binding.scan_entries[0].scan_platform == 'linux/amd64' + binding.scan_entries[0].scan_url == 'https://wave.io/view/scans/sc-abc_1' + binding.scan_entries[1].scan_id == 'sc-def_2' + binding.scan_entries[1].scan_platform == 'linux/arm64' + } + + def 'should populate scan binding when not succeeded'() { + given: + def binding = new HashMap() + def scanChildIds = new ChildEntries('sc-abc_1:linux/amd64,sc-def_2:linux/arm64') + + when: + ChildEntries.populateScanBinding(binding, null, scanChildIds, false, 'https://wave.io') + then: + binding.scan_id == null + binding.scan_url == null + binding.scan_entries == null + } + + def 'should populate scan binding when scanId is null'() { + given: + def binding = new HashMap() + + when: + ChildEntries.populateScanBinding(binding, null, null, true, 'https://wave.io') + then: + binding.scan_id == null + binding.scan_url == null + } + + def 'should populate build binding for child builds'() { + given: + def binding = new HashMap() + def buildChildIds = new ChildEntries('bd-abc_0:linux/amd64,bd-def_0:linux/arm64') + + when: + ChildEntries.populateBuildBinding(binding, buildChildIds, 'https://wave.io') + then: + binding.build_entries.size() == 2 + binding.build_entries[0].build_id == 'bd-abc_0' + binding.build_entries[0].build_platform == 'linux/amd64' + binding.build_entries[0].build_url == 'https://wave.io/view/builds/bd-abc_0' + binding.build_entries[1].build_id == 'bd-def_0' + binding.build_entries[1].build_platform == 'linux/arm64' + binding.build_entries[1].build_url == 'https://wave.io/view/builds/bd-def_0' + } + + def 'should populate build binding when null'() { + given: + def binding = new HashMap() + + when: + ChildEntries.populateBuildBinding(binding, null, 'https://wave.io') + then: + binding.build_entries == null + } +} diff --git a/src/test/groovy/io/seqera/wave/service/request/ContainerRequestTest.groovy b/src/test/groovy/io/seqera/wave/service/request/ContainerRequestTest.groovy index 98852c432a..548cc549e1 100644 --- a/src/test/groovy/io/seqera/wave/service/request/ContainerRequestTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/request/ContainerRequestTest.groovy @@ -67,6 +67,7 @@ class ContainerRequestTest extends Specification { BUILD_NEW, FREEZE, 'scan-1234', + null, // scanChildIds ScanMode.required, List.of(ScanLevel.HIGH), DRY_RUN, diff --git a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy index 57f54d5c3e..e9ed508cbe 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy @@ -29,6 +29,7 @@ import java.time.Instant import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.api.ScanMode import io.seqera.wave.configuration.ScanConfig +import io.seqera.wave.core.ChildEntries import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildFormat @@ -473,7 +474,7 @@ class ContainerScanServiceImplTest extends Specification { def containerId = 'container1234' def workspace = Path.of('/some/workspace') def platform = ContainerPlatform.of('linux/amd64,linux/arm64') - def multiScanId = 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64' + def scanChildIds = new ChildEntries('sc-abc_1:linux/amd64,sc-def_2:linux/arm64') final build = BuildRequest.of( containerId: containerId, @@ -483,7 +484,7 @@ class ContainerScanServiceImplTest extends Specification { identity: PlatformId.NULL, platform: platform, configJson: '{"config":"json"}', - scanId: multiScanId, + scanChildIds: scanChildIds, format: BuildFormat.DOCKER, buildId: "${containerId}_1", ) diff --git a/src/test/groovy/io/seqera/wave/service/scan/ScanIdsTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ScanIdsTest.groovy deleted file mode 100644 index 8fcf936b60..0000000000 --- a/src/test/groovy/io/seqera/wave/service/scan/ScanIdsTest.groovy +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.service.scan - -import spock.lang.Specification -import spock.lang.Unroll - -/** - * Tests for {@link ScanIds} - * - * @author Paolo Di Tommaso - */ -class ScanIdsTest extends Specification { - - @Unroll - def 'should detect multi scanId: #VALUE'() { - expect: - ScanIds.isMulti(VALUE) == EXPECTED - - where: - VALUE | EXPECTED - null | false - 'sc-abc_1' | false - 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64' | true - 'sc-abc_1:linux/amd64' | true - } - - def 'should encode single scanId'() { - expect: - ScanIds.encode(['sc-abc_1': 'linux/amd64']) == 'sc-abc_1' - } - - def 'should encode multiple scanIds'() { - given: - def map = new LinkedHashMap() - map.put('sc-abc_1', 'linux/amd64') - map.put('sc-def_2', 'linux/arm64') - - expect: - ScanIds.encode(map) == 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64' - } - - def 'should encode null or empty'() { - expect: - ScanIds.encode(null) == null - ScanIds.encode([:]) == null - } - - def 'should decode single scanId'() { - when: - def result = ScanIds.decode('sc-abc_1') - then: - result.size() == 1 - result[0].key == 'sc-abc_1' - result[0].value == null - } - - def 'should decode multi scanId'() { - when: - def result = ScanIds.decode('sc-abc_1:linux/amd64,sc-def_2:linux/arm64') - then: - result.size() == 2 - result[0].key == 'sc-abc_1' - result[0].value == 'linux/amd64' - result[1].key == 'sc-def_2' - result[1].value == 'linux/arm64' - } - - def 'should decode null or empty'() { - expect: - ScanIds.decode(null) == [] - ScanIds.decode('') == [] - } - - @Unroll - def 'should get primary scanId from: #VALUE'() { - expect: - ScanIds.primary(VALUE) == EXPECTED - - where: - VALUE | EXPECTED - null | null - 'sc-abc_1' | 'sc-abc_1' - 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64' | 'sc-abc_1' - } - - def 'should get all ids'() { - expect: - ScanIds.allIds('sc-abc_1') == ['sc-abc_1'] - ScanIds.allIds('sc-abc_1:linux/amd64,sc-def_2:linux/arm64') == ['sc-abc_1', 'sc-def_2'] - } - - def 'should roundtrip encode/decode'() { - given: - def map = new LinkedHashMap() - map.put('sc-abc_1', 'linux/amd64') - map.put('sc-def_2', 'linux/arm64') - - when: - def encoded = ScanIds.encode(map) - def decoded = ScanIds.decode(encoded) - - then: - decoded.size() == 2 - decoded[0].key == 'sc-abc_1' - decoded[0].value == 'linux/amd64' - decoded[1].key == 'sc-def_2' - decoded[1].value == 'linux/arm64' - } - - def 'should populate scan binding for single scanId'() { - given: - def binding = new HashMap() - - when: - ScanIds.populateScanBinding(binding, 'sc-abc_1', true, 'https://wave.io') - then: - binding.scan_id == 'sc-abc_1' - binding.scan_url == 'https://wave.io/view/scans/sc-abc_1' - binding.scan_entries == null - } - - def 'should populate scan binding for multi scanId'() { - given: - def binding = new HashMap() - - when: - ScanIds.populateScanBinding(binding, 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64', true, 'https://wave.io') - then: - binding.scan_id == 'sc-abc_1' - binding.scan_url == null - binding.scan_entries.size() == 2 - binding.scan_entries[0].scan_id == 'sc-abc_1' - binding.scan_entries[0].scan_platform == 'linux/amd64' - binding.scan_entries[0].scan_url == 'https://wave.io/view/scans/sc-abc_1' - binding.scan_entries[1].scan_id == 'sc-def_2' - binding.scan_entries[1].scan_platform == 'linux/arm64' - } - - def 'should populate scan binding when not succeeded'() { - given: - def binding = new HashMap() - - when: - ScanIds.populateScanBinding(binding, 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64', false, 'https://wave.io') - then: - binding.scan_id == 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64' - binding.scan_url == null - binding.scan_entries == null - } - - def 'should populate scan binding when scanId is null'() { - given: - def binding = new HashMap() - - when: - ScanIds.populateScanBinding(binding, null, true, 'https://wave.io') - then: - binding.scan_id == null - binding.scan_url == null - } -} From 6747ea85d3c1929d707ceb58b0a54706b460195e Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Mar 2026 19:31:46 +0100 Subject: [PATCH 06/15] Address code review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove dead `withScanId()` method from BuildRequest - Add explicit `= null` initialization for scanChildIds in ContainerController - Fix pre-existing bug: `this.mirror == that.mirror` → `=` in WaveContainerRecord copy constructor - Rename mismatched getter `getChildScanIds()` → `getScanChildIds()` in ContainerRequest - Add Jackson round-trip serialization tests for ChildEntries Co-Authored-By: Claude Opus 4.6 --- .../controller/ContainerController.groovy | 2 +- .../wave/service/builder/BuildRequest.groovy | 28 ----------------- .../persistence/WaveContainerRecord.groovy | 2 +- .../service/request/ContainerRequest.groovy | 2 +- .../seqera/wave/core/ChildEntriesTest.groovy | 31 +++++++++++++++++++ 5 files changed, 34 insertions(+), 31 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index cec74b5d9c..c87876fd18 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -507,7 +507,7 @@ class ContainerController { String buildId boolean buildNew String scanId - ChildEntries scanChildIds + ChildEntries scanChildIds = null Boolean succeeded if( req.containerFile && req.multiPlatform ) { if( !buildService ) throw new UnsupportedBuildServiceException() diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy index 165c0e0c0c..3c4273f335 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy @@ -244,34 +244,6 @@ class BuildRequest { new BuildRequest(opts) } - BuildRequest withScanId(String scanId) { - return BuildRequest.of( - containerId: this.containerId, - containerFile: this.containerFile, - condaFile: this.condaFile, - workspace: this.workspace, - targetImage: this.targetImage, - identity: this.identity, - platform: this.platform, - cacheRepository: this.cacheRepository, - startTime: this.startTime, - ip: this.ip, - configJson: this.configJson, - offsetId: this.offsetId, - containerConfig: this.containerConfig, - scanId: scanId, - buildContext: this.buildContext, - format: this.format, - maxDuration: this.maxDuration, - compression: this.compression, - buildId: this.buildId, - buildTemplate: this.buildTemplate, - noEmail: this.noEmail, - buildChildIds: this.buildChildIds, - scanChildIds: this.scanChildIds - ) - } - BuildRequest withChildScanIds(ChildEntries scanChildIds) { return BuildRequest.of( containerId: this.containerId, diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy index d8c167715e..f685f6f85c 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy @@ -234,7 +234,7 @@ class WaveContainerRecord { this.buildNew = that.buildNew this.freeze = that.freeze this.fusionVersion = that.fusionVersion - this.mirror == that.mirror + this.mirror = that.mirror this.scanId = that.scanId this.scanChildIds = that.scanChildIds // -- digest part diff --git a/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy index 7ab2b8b89f..5536d375e1 100644 --- a/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy @@ -147,7 +147,7 @@ class ContainerRequest { return scanId } - ChildEntries getChildScanIds() { + ChildEntries getScanChildIds() { return scanChildIds } diff --git a/src/test/groovy/io/seqera/wave/core/ChildEntriesTest.groovy b/src/test/groovy/io/seqera/wave/core/ChildEntriesTest.groovy index 020a51324e..61c38858e6 100644 --- a/src/test/groovy/io/seqera/wave/core/ChildEntriesTest.groovy +++ b/src/test/groovy/io/seqera/wave/core/ChildEntriesTest.groovy @@ -18,6 +18,7 @@ package io.seqera.wave.core +import com.fasterxml.jackson.databind.ObjectMapper import spock.lang.Specification import spock.lang.Unroll @@ -206,6 +207,36 @@ class ChildEntriesTest extends Specification { binding.build_entries[1].build_url == 'https://wave.io/view/builds/bd-def_0' } + def 'should roundtrip through Jackson serialization'() { + given: + def mapper = new ObjectMapper() + def original = new ChildEntries('sc-abc_1:linux/amd64,sc-def_2:linux/arm64') + + when: + def json = mapper.writeValueAsString(original) + def restored = mapper.readValue(json, ChildEntries) + + then: + json == '"sc-abc_1:linux/amd64,sc-def_2:linux/arm64"' + restored == original + restored.decode().size() == 2 + } + + def 'should handle null in Jackson serialization'() { + given: + def mapper = new ObjectMapper() + + when: + def json = mapper.writeValueAsString([entries: null]) + then: + json == '{"entries":null}' + + when: + def restored = mapper.readValue('"sc-abc_1:linux/amd64"', ChildEntries) + then: + restored == new ChildEntries('sc-abc_1:linux/amd64') + } + def 'should populate build binding when null'() { given: def binding = new HashMap() From ca0ec8cf1253dd50a9b678d3340c23bf93e492b5 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Mar 2026 19:45:23 +0100 Subject: [PATCH 07/15] Fix failing tests: ContainerPlatform Jackson support and scan mock stubs - Add @JsonCreator/@JsonValue to ContainerPlatform for proper serialization in persistence records (fixes mirror record tests) - Add missing getScanId() stubs to ScanEntry mocks in ContainerStatusServiceTest (scanResult uses scan.scanId not request.scanId for URL) Co-Authored-By: Claude Opus 4.6 --- .../io/seqera/wave/core/ContainerPlatform.groovy | 4 ++++ .../request/ContainerStatusServiceTest.groovy | 13 +++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy b/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy index 4c41629670..0ad8989acf 100644 --- a/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy +++ b/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy @@ -18,6 +18,8 @@ package io.seqera.wave.core +import com.fasterxml.jackson.annotation.JsonCreator +import com.fasterxml.jackson.annotation.JsonValue import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import io.seqera.wave.exception.BadRequestException @@ -67,6 +69,7 @@ class ContainerPlatform { return archs.size() > 1 } + @JsonValue String toString() { if( isMultiArch() ) { return archs.collect { "${os}/${it}" }.join(',') @@ -112,6 +115,7 @@ class ContainerPlatform { return value ? of(value) : defaultPlatform } + @JsonCreator static ContainerPlatform of(String value) { if( !value ) throw new BadRequestException("Missing container platform attribute") diff --git a/src/test/groovy/io/seqera/wave/service/request/ContainerStatusServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/request/ContainerStatusServiceTest.groovy index 2533d9b874..54b0a69002 100644 --- a/src/test/groovy/io/seqera/wave/service/request/ContainerStatusServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/request/ContainerStatusServiceTest.groovy @@ -45,6 +45,7 @@ class ContainerStatusServiceTest extends Specification { def res1 = service.scanResult(request, scan) then: scan.succeeded() >> false + scan.getScanId() >> 'scan-123' request.getScanId() >> 'scan-123' and: res1 == new ContainerStatusServiceImpl.StageResult( @@ -58,6 +59,7 @@ class ContainerStatusServiceTest extends Specification { then: scan.succeeded() >> true scan.summary() >> [MEDIUM: 5] + scan.getScanId() >> 'scan-123' request.getScanId() >> 'scan-123' request.getScanLevels() >> null and: @@ -72,6 +74,7 @@ class ContainerStatusServiceTest extends Specification { then: scan.succeeded() >> true scan.summary() >> [HIGH:2, CRITICAL:1, MEDIUM: 5] + scan.getScanId() >> 'scan-123' request.getScanId() >> 'scan-123' request.getScanLevels() >> [ScanLevel.LOW, ScanLevel.MEDIUM] and: @@ -86,6 +89,7 @@ class ContainerStatusServiceTest extends Specification { then: scan.succeeded() >> true scan.summary() >> [MEDIUM: 5] + scan.getScanId() >> 'scan-123' request.getScanId() >> 'scan-123' request.getScanLevels() >> [ScanLevel.LOW, ScanLevel.MEDIUM] and: @@ -100,6 +104,7 @@ class ContainerStatusServiceTest extends Specification { then: scan.succeeded() >> true scan.summary() >> [:] + scan.getScanId() >> 'scan-123' request.getScanId() >> 'scan-123' request.getScanLevels() >> [ScanLevel.LOW, ScanLevel.MEDIUM] and: @@ -345,7 +350,7 @@ class ContainerStatusServiceTest extends Specification { requestData.scanMode >> ScanMode.required requestData.scanLevels >> List.of() and: - service.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>false } + service.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>false; getScanId()>>'scan-abc' } then: resp.id == requestId resp.status == ContainerStatus.DONE @@ -384,7 +389,7 @@ class ContainerStatusServiceTest extends Specification { requestData.scanMode >> ScanMode.required requestData.scanLevels >> List.of() and: - service.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true; summary()>>[HIGH:1] } + service.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true; summary()>>[HIGH:1]; getScanId()>>'scan-abc' } then: resp.id == requestId resp.status == ContainerStatus.DONE @@ -423,7 +428,7 @@ class ContainerStatusServiceTest extends Specification { requestData.scanMode >> ScanMode.required requestData.scanLevels >> List.of(ScanLevel.MEDIUM,ScanLevel.HIGH) and: - service.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true; summary()>>[HIGH:1] } + service.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true; summary()>>[HIGH:1]; getScanId()>>'scan-abc' } then: resp.id == requestId resp.status == ContainerStatus.DONE @@ -500,7 +505,7 @@ class ContainerStatusServiceTest extends Specification { requestData.scanMode >> ScanMode.required requestData.scanLevels >> List.of() and: - service.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true; summary()>>[HIGH:1]; getStartTime()>>startTime } + service.getScanState('scan-abc') >> Mock(ScanEntry) { getDuration()>>_2min; succeeded()>>true; summary()>>[HIGH:1]; getScanId()>>'scan-abc'; getStartTime()>>startTime } then: resp.id == requestId resp.status == ContainerStatus.DONE From 1beb69b632185666e1a8188620ba482eeb055bc0 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Mar 2026 20:59:34 +0100 Subject: [PATCH 08/15] Refactor ContainerPlatform: introduce inner Platform class Replace separate os/arch/variant/archs fields with a List to properly model multi-platform combinations where each platform can have its own OS. Move parsing logic into Platform.of() static factory method. Co-Authored-By: Claude Opus 4.6 --- .../controller/ContainerController.groovy | 4 +- .../io/seqera/wave/core/ChildEntries.groovy | 127 ++++-------- .../seqera/wave/core/ContainerPlatform.groovy | 180 ++++++++++-------- .../service/builder/ManifestAssembler.groovy | 8 +- .../builder/MultiPlatformBuildService.groovy | 5 +- .../request/ContainerStatusServiceImpl.groovy | 10 +- .../scan/ContainerScanServiceImpl.groovy | 5 +- .../seqera/wave/core/ChildEntriesTest.groovy | 164 +++++++++------- .../wave/core/ContainerPlatformTest.groovy | 8 +- .../service/builder/BuildRequestTest.groovy | 97 ++++++++++ .../scan/ContainerScanServiceImplTest.groovy | 5 +- 11 files changed, 343 insertions(+), 270 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index c87876fd18..0f82619d81 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -411,8 +411,8 @@ class ContainerController { final multiPlatform = ContainerPlatform.MULTI_PLATFORM final scanMode = req.scanMode!=null ? req.scanMode : ScanMode.async final scanIdByPlatform = new LinkedHashMap() - for( String arch : multiPlatform.archs ) { - final platform = "${multiPlatform.os}/${arch}" as String + for( ContainerPlatform.Platform p : multiPlatform.platforms ) { + final platform = p.toString() final id = scanService.getScanId("${build.targetImage}#${platform}", null, scanMode, req.format) if( id ) scanIdByPlatform.put(id, platform) diff --git a/src/main/groovy/io/seqera/wave/core/ChildEntries.groovy b/src/main/groovy/io/seqera/wave/core/ChildEntries.groovy index cc98b2e65c..e67f0f0e70 100644 --- a/src/main/groovy/io/seqera/wave/core/ChildEntries.groovy +++ b/src/main/groovy/io/seqera/wave/core/ChildEntries.groovy @@ -18,70 +18,52 @@ package io.seqera.wave.core -import com.fasterxml.jackson.annotation.JsonCreator -import com.fasterxml.jackson.annotation.JsonValue import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode /** - * Value type representing a set of per-platform child IDs (builds or scans). + * A list of per-platform child IDs (builds or scans). * - * Serialises to/from an encoded string: - * "id1:platform1,id2:platform2" - * - * Example: "sc-abc_1:linux/amd64,sc-def_2:linux/arm64" - * - * Used as the type for both {@code buildChildIds} and {@code scanChildIds}. + * Serialises to/from JSON as: [{"id":"sc-abc_1","platform":"linux/amd64"}, ...] * * @author Paolo Di Tommaso */ @CompileStatic -class ChildEntries { +class ChildEntries extends ArrayList { - /** - * The encoded string representation - */ - private final String encoded + @CompileStatic + @EqualsAndHashCode + static class Entry { + String id + String platform + + Entry() {} - @JsonCreator - ChildEntries(String encoded) { - this.encoded = encoded + Entry(String id, String platform) { + this.id = id + this.platform = platform + } + } + + ChildEntries() { + super() + } + + ChildEntries(List entries) { + super(entries ?: Collections.emptyList()) } /** * Create from a map of id-by-platform. - * @param idByPlatform Map where keys are IDs and values are platform strings (e.g. "linux/amd64") + * @param idByPlatform Map where keys are IDs and values are platform strings * @return A new ChildEntries, or null if the map is null/empty */ static ChildEntries of(Map idByPlatform) { if( !idByPlatform ) return null - if( idByPlatform.size() == 1 ) { - final entry = idByPlatform.entrySet().first() - return new ChildEntries("${entry.key}:${entry.value}") - } - final encoded = idByPlatform - .collect { id, platform -> "${id}:${platform}" } - .join(',') - return new ChildEntries(encoded) - } - - /** - * Decode into a list of (id, platform) pairs. - */ - List> decode() { - if( !encoded ) - return Collections.>emptyList() - final List> result = new ArrayList<>() - for( String part : encoded.tokenize(',') ) { - final idx = part.indexOf(':') - if( idx < 0 ) { - result.add(new AbstractMap.SimpleEntry(part, null)) - } - else { - final id = part.substring(0, idx) - final platform = part.substring(idx + 1) - result.add(new AbstractMap.SimpleEntry(id, platform)) - } + final result = new ChildEntries() + for( Map.Entry it : idByPlatform.entrySet() ) { + result.add(new Entry(it.key, it.value)) } return result } @@ -90,60 +72,22 @@ class ChildEntries { * Get the first/primary ID */ String primary() { - if( !encoded ) - return null - final idx = encoded.indexOf(':') - if( idx < 0 ) - return encoded - return encoded.substring(0, idx) + return this ? this[0].id : null } /** * Get all IDs */ List allIds() { - return decode().collect { it.key } - } - - /** - * Jackson serialisation: serialise as the encoded string - */ - @JsonValue - String toString() { - return encoded - } - - /** - * Groovy truth: non-null and non-empty encoded string - */ - boolean asBoolean() { - return encoded != null && !encoded.isEmpty() - } - - @Override - boolean equals(Object o) { - if( this.is(o) ) return true - if( o == null || getClass() != o.getClass() ) return false - return encoded == ((ChildEntries) o).encoded - } - - @Override - int hashCode() { - return encoded != null ? encoded.hashCode() : 0 + return this.collect { it.id } } // -- template binding helpers -- - /** - * Populate scan-related binding keys for view templates and emails. - * Handles both single-platform (scanId) and multi-platform (scanChildIds) scan IDs. - * - * Sets: scan_entries (list of maps), scan_url, scan_id - */ static void populateScanBinding(Map binding, String scanId, ChildEntries scanChildIds, boolean succeeded, String serverUrl) { if( scanChildIds && succeeded ) { - binding.scan_entries = scanChildIds.decode().collect { Map.Entry entry -> - [scan_id: entry.key, scan_platform: entry.value, scan_url: "${serverUrl}/view/scans/${entry.key}"] as Map + binding.scan_entries = scanChildIds.collect { Entry entry -> + [scan_id: entry.id, scan_platform: entry.platform, scan_url: "${serverUrl}/view/scans/${entry.id}"] as Map } binding.scan_url = null binding.scan_id = scanChildIds.primary() @@ -155,15 +99,10 @@ class ChildEntries { } } - /** - * Populate build-related binding keys for child (per-arch) build entries. - * - * Sets: build_entries (list of maps with build_id, build_platform, build_url) - */ static void populateBuildBinding(Map binding, ChildEntries buildChildIds, String serverUrl) { if( buildChildIds ) { - binding.build_entries = buildChildIds.decode().collect { Map.Entry entry -> - [build_id: entry.key, build_platform: entry.value, build_url: "${serverUrl}/view/builds/${entry.key}"] as Map + binding.build_entries = buildChildIds.collect { Entry entry -> + [build_id: entry.id, build_platform: entry.platform, build_url: "${serverUrl}/view/builds/${entry.id}"] as Map } } else { diff --git a/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy b/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy index 0ad8989acf..fde462f78b 100644 --- a/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy +++ b/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy @@ -27,7 +27,7 @@ import io.seqera.wave.exception.BadRequestException * Model a container platform * @author Paolo Di Tommaso */ -@EqualsAndHashCode +@EqualsAndHashCode(includes = 'platforms') @CompileStatic class ContainerPlatform { @@ -38,46 +38,111 @@ class ContainerPlatform { public static final String DEFAULT_ARCH = 'amd64' public static final String DEFAULT_OS = 'linux' - public static final ContainerPlatform DEFAULT = new ContainerPlatform(DEFAULT_OS, DEFAULT_ARCH) + @EqualsAndHashCode + @CompileStatic + static class Platform implements Serializable { + final String os + final String arch + final String variant + + Platform(String os, String arch, String variant=null) { + this.os = os + this.arch = arch + this.variant = variant + } + + static Platform of(String value) { + if( !value ) + throw new BadRequestException("Missing container platform attribute") - /** - * A composite platform representing linux/amd64 + linux/arm64 multi-arch builds - */ - public static final ContainerPlatform MULTI_PLATFORM = new ContainerPlatform('linux', ['amd64', 'arm64']) + final items = value.tokenize('/') + if( items.size()==1 ) + items.add(0, DEFAULT_OS) - final String os - final String arch - final String variant - final List archs + if( items.size()==2 || items.size()==3 ) { + final os = os0(items[0]) + final arch = arch0(items[1]) + // variant v8 for amd64 is normalised to empty + // see https://github.com/containerd/containerd/blob/v1.4.3/platforms/database.go#L96 + final variant = variant0(arch, items[2]) + return new Platform(os, arch, variant) + } + + throw new BadRequestException("Invalid container platform: $value -- offending value: $value") + } + + @Override + String toString() { + def result = os + "/" + arch + if( variant ) + result += "/" + variant + return result + } + + static private String arch0(String value) { + if( !value ) + return DEFAULT_ARCH + if( value !in ALLOWED_ARCH) throw new BadRequestException("Unsupported container platform: $value") + // see + // https://github.com/containerd/containerd/blob/v1.4.3/platforms/database.go#L89 + if( value in AMD64 ) + return AMD64.get(0) + if( value in ARM64 ) + return ARM64.get(0) + return value + } + + static private String os0(String value) { + return value ?: DEFAULT_OS + } + + static private String variant0(String arch, String variant) { + if( arch in ARM64 && variant in V8 ) { + // this also address this issue + // https://github.com/GoogleContainerTools/kaniko/issues/1995#issuecomment-1327706161 + return null + } + if( arch == 'arm' ) { + // arm defaults to variant v7 + // https://github.com/containerd/containerd/blob/v1.4.3/platforms/database.go#L89 + if( (!variant || variant=='7') ) return 'v7' + if( variant in ['5','6','8']) return 'v'+variant + } + return variant + } + } + + public static final ContainerPlatform DEFAULT = new ContainerPlatform([new Platform(DEFAULT_OS, DEFAULT_ARCH)]) + + public static final ContainerPlatform MULTI_PLATFORM = new ContainerPlatform([ + new Platform('linux', 'amd64'), + new Platform('linux', 'arm64') + ]) + + final List platforms ContainerPlatform(String os, String arch, String variant=null) { - this.os = os - this.arch = arch - this.variant = variant - this.archs = List.of(arch) + this.platforms = List.of(new Platform(os, arch, variant)) } - private ContainerPlatform(String os, List archs) { - assert archs.size() >= 2, "Multi-arch platform requires at least 2 architectures" - this.os = os - this.arch = archs[0] - this.variant = null - this.archs = List.copyOf(archs) + private ContainerPlatform(List platforms) { + assert platforms.size() >= 1, "Platform list must not be empty" + this.platforms = List.copyOf(platforms) } + String getOs() { platforms[0].os } + + String getArch() { platforms[0].arch } + + String getVariant() { platforms[0].variant } + boolean isMultiArch() { - return archs.size() > 1 + return platforms.size() > 1 } @JsonValue String toString() { - if( isMultiArch() ) { - return archs.collect { "${os}/${it}" }.join(',') - } - def result = os + "/" + arch - if( variant ) - result += "/" + variant - return result + return platforms.collect { it.toString() }.join(',') } boolean matches(Map record) { @@ -120,61 +185,8 @@ class ContainerPlatform { if( !value ) throw new BadRequestException("Missing container platform attribute") - // handle comma-separated multi-platform values e.g. "linux/amd64,linux/arm64" - if( value.contains(',') ) { - final parts = value.tokenize(',').collect { it.trim() } - final platforms = parts.collect { of(it) } - // all platforms must share the same OS - final os = platforms[0].os - final archs = platforms.collect { it.arch } - return new ContainerPlatform(os, archs) - } - - final items= value.tokenize('/') - if( items.size()==1 ) - items.add(0, DEFAULT_OS) - - if( items.size()==2 || items.size()==3 ) { - final os = os0(items[0]) - final arch = arch0(items[1]) - // variant v8 for amd64 is normalised to empty - // see https://github.com/containerd/containerd/blob/v1.4.3/platforms/database.go#L96 - final variant = variant0(arch,items[2]) - return new ContainerPlatform(os, arch, variant) - } - - throw new BadRequestException("Invalid container platform: $value -- offending value: $value") - } - - static private String arch0(String value) { - if( !value ) - return DEFAULT_ARCH - if( value !in ALLOWED_ARCH) throw new BadRequestException("Unsupported container platform: $value") - // see - // https://github.com/containerd/containerd/blob/v1.4.3/platforms/database.go#L89 - if( value in AMD64 ) - return AMD64.get(0) - if( value in ARM64 ) - return ARM64.get(0) - return value - } - - static private String os0(String value) { - return value ?: DEFAULT_OS - } - - static private String variant0(String arch, String variant) { - if( arch in ARM64 && variant in V8 ) { - // this also address this issue - // https://github.com/GoogleContainerTools/kaniko/issues/1995#issuecomment-1327706161 - return null - } - if( arch == 'arm' ) { - // arm defaults to variant v7 - // https://github.com/containerd/containerd/blob/v1.4.3/platforms/database.go#L89 - if( (!variant || variant=='7') ) return 'v7' - if( variant in ['5','6','8']) return 'v'+variant - } - return variant + return new ContainerPlatform(value + .tokenize(',') + .collect(it-> Platform.of(it.trim()))) } } diff --git a/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy b/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy index d8ca6cf072..9d5891de8a 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy @@ -50,16 +50,16 @@ import static io.seqera.wave.model.ContentType.OCI_IMAGE_MANIFEST_V1 class ManifestAssembler { @Inject - RegistryLookupService registryLookup + private RegistryLookupService registryLookup @Inject - RegistryCredentialsProvider credentialsProvider + private RegistryCredentialsProvider credentialsProvider @Inject - RegistryAuthService loginService + private RegistryAuthService loginService @Inject - HttpClientConfig httpConfig + private HttpClientConfig httpConfig /** * Create and push an OCI Image Index (manifest list) to the registry. diff --git a/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy index 3abe7ebb6a..3b1d4bf02a 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy @@ -107,7 +107,10 @@ class MultiPlatformBuildService implements JobHandler { final startTime = Instant.now() // Encode child build IDs for the parent build view - final buildChildIds = new ChildEntries("${amd64Track.id}:${PLATFORM_AMD64},${arm64Track.id}:${PLATFORM_ARM64}") + final buildChildIds = new ChildEntries([ + new ChildEntries.Entry(amd64Track.id, PLATFORM_AMD64.toString()), + new ChildEntries.Entry(arm64Track.id, PLATFORM_ARM64.toString()) + ]) // Store an in-progress build entry so status polling can find it final syntheticRequest = BuildRequest.of( diff --git a/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy index a48f21da68..66d634bbd0 100644 --- a/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy @@ -18,6 +18,7 @@ package io.seqera.wave.service.request +import io.seqera.wave.core.ChildEntries import io.seqera.wave.exception.UnsupportedBuildServiceException import io.seqera.wave.exception.UnsupportedMirrorServiceException import io.seqera.wave.exception.UnsupportedScanServiceException @@ -181,15 +182,14 @@ class ContainerStatusServiceImpl implements ContainerStatusService { } protected ContainerStatusResponse handleMultiScanStatus(ContainerRequest request, ContainerState state) { - final entries = request.scanChildIds.decode() - final List scans = new ArrayList<>(entries.size()) + final List scans = new ArrayList<>(request.scanChildIds.size()) boolean allDone = true boolean allSucceeded = true Duration maxScanDuration = Duration.ZERO - for( Map.Entry pair : entries ) { - final scan = getScanState(pair.key) + for( ChildEntries.Entry pair : request.scanChildIds ) { + final scan = getScanState(pair.id) if( !scan ) - throw new NotFoundException("Missing container scan record with id: ${pair.key}") + throw new NotFoundException("Missing container scan record with id: ${pair.id}") scans.add(scan) if( !scan.duration ) { allDone = false diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index 5cf8fb5ec9..fd868e45fd 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -27,6 +27,7 @@ import java.util.concurrent.ExecutorService import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import io.seqera.wave.core.ChildEntries import io.micronaut.context.annotation.Requires import io.micronaut.core.annotation.Nullable import io.micronaut.http.server.types.files.StreamedFile @@ -130,8 +131,8 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler pair : entry.request.scanChildIds.decode() ) { - scan(fromBuild(entry.request, pair.key, ContainerPlatform.of(pair.value))) + for( ChildEntries.Entry pair : entry.request.scanChildIds ) { + scan(fromBuild(entry.request, pair.id, ContainerPlatform.of(pair.platform))) } } else if( entry.request.scanId ) { diff --git a/src/test/groovy/io/seqera/wave/core/ChildEntriesTest.groovy b/src/test/groovy/io/seqera/wave/core/ChildEntriesTest.groovy index 61c38858e6..5b451a0b08 100644 --- a/src/test/groovy/io/seqera/wave/core/ChildEntriesTest.groovy +++ b/src/test/groovy/io/seqera/wave/core/ChildEntriesTest.groovy @@ -20,7 +20,6 @@ package io.seqera.wave.core import com.fasterxml.jackson.databind.ObjectMapper import spock.lang.Specification -import spock.lang.Unroll /** * Tests for {@link ChildEntries} @@ -30,8 +29,12 @@ import spock.lang.Unroll class ChildEntriesTest extends Specification { def 'should create from map with single entry'() { - expect: - ChildEntries.of(['sc-abc_1': 'linux/amd64']).toString() == 'sc-abc_1:linux/amd64' + when: + def result = ChildEntries.of(['sc-abc_1': 'linux/amd64']) + then: + result.size() == 1 + result[0].id == 'sc-abc_1' + result[0].platform == 'linux/amd64' } def 'should create from map with multiple entries'() { @@ -40,8 +43,14 @@ class ChildEntriesTest extends Specification { map.put('sc-abc_1', 'linux/amd64') map.put('sc-def_2', 'linux/arm64') - expect: - ChildEntries.of(map).toString() == 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64' + when: + def result = ChildEntries.of(map) + then: + result.size() == 2 + result[0].id == 'sc-abc_1' + result[0].platform == 'linux/amd64' + result[1].id == 'sc-def_2' + result[1].platform == 'linux/arm64' } def 'should return null from null or empty map'() { @@ -50,88 +59,70 @@ class ChildEntriesTest extends Specification { ChildEntries.of([:]) == null } - def 'should decode single entry'() { - when: - def result = new ChildEntries('sc-abc_1:linux/amd64').decode() - then: - result.size() == 1 - result[0].key == 'sc-abc_1' - result[0].value == 'linux/amd64' - } - - def 'should decode bare id (no platform)'() { - when: - def result = new ChildEntries('sc-abc_1').decode() - then: - result.size() == 1 - result[0].key == 'sc-abc_1' - result[0].value == null - } - - def 'should decode multi entries'() { + def 'should create from list of entries'() { when: - def result = new ChildEntries('sc-abc_1:linux/amd64,sc-def_2:linux/arm64').decode() + def result = new ChildEntries([ + new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), + new ChildEntries.Entry('sc-def_2', 'linux/arm64') + ]) then: result.size() == 2 - result[0].key == 'sc-abc_1' - result[0].value == 'linux/amd64' - result[1].key == 'sc-def_2' - result[1].value == 'linux/arm64' + result[0].id == 'sc-abc_1' + result[0].platform == 'linux/amd64' + result[1].id == 'sc-def_2' + result[1].platform == 'linux/arm64' } - @Unroll - def 'should get primary id from: #ENCODED'() { + def 'should get primary id'() { expect: - new ChildEntries(ENCODED).primary() == EXPECTED - - where: - ENCODED | EXPECTED - 'sc-abc_1' | 'sc-abc_1' - 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64' | 'sc-abc_1' + new ChildEntries([new ChildEntries.Entry('sc-abc_1', null)]).primary() == 'sc-abc_1' + and: + new ChildEntries([ + new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), + new ChildEntries.Entry('sc-def_2', 'linux/arm64') + ]).primary() == 'sc-abc_1' + and: + new ChildEntries([]).primary() == null } def 'should get all ids'() { expect: - new ChildEntries('sc-abc_1').allIds() == ['sc-abc_1'] - new ChildEntries('sc-abc_1:linux/amd64,sc-def_2:linux/arm64').allIds() == ['sc-abc_1', 'sc-def_2'] + new ChildEntries([new ChildEntries.Entry('sc-abc_1', null)]).allIds() == ['sc-abc_1'] + and: + new ChildEntries([ + new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), + new ChildEntries.Entry('sc-def_2', 'linux/arm64') + ]).allIds() == ['sc-abc_1', 'sc-def_2'] } - def 'should roundtrip via of and decode'() { + def 'should roundtrip via of'() { given: def map = new LinkedHashMap() map.put('sc-abc_1', 'linux/amd64') map.put('sc-def_2', 'linux/arm64') when: - def entries = ChildEntries.of(map) - def decoded = entries.decode() + def result = ChildEntries.of(map) then: - decoded.size() == 2 - decoded[0].key == 'sc-abc_1' - decoded[0].value == 'linux/amd64' - decoded[1].key == 'sc-def_2' - decoded[1].value == 'linux/arm64' - } - - def 'should serialize to string via toString'() { - given: - def entries = new ChildEntries('sc-abc_1:linux/amd64,sc-def_2:linux/arm64') - expect: - entries.toString() == 'sc-abc_1:linux/amd64,sc-def_2:linux/arm64' + result.size() == 2 + result[0].id == 'sc-abc_1' + result[0].platform == 'linux/amd64' + result[1].id == 'sc-def_2' + result[1].platform == 'linux/arm64' } def 'should support groovy truth'() { expect: - new ChildEntries('sc-abc_1') as boolean == true + new ChildEntries([new ChildEntries.Entry('sc-abc_1', null)]) as boolean == true new ChildEntries(null) as boolean == false - new ChildEntries('') as boolean == false + new ChildEntries([]) as boolean == false } def 'should support equality'() { expect: - new ChildEntries('sc-abc_1:linux/amd64') == new ChildEntries('sc-abc_1:linux/amd64') - !new ChildEntries('sc-abc_1').equals(new ChildEntries('sc-def_2')) + new ChildEntries([new ChildEntries.Entry('sc-abc_1', 'linux/amd64')]) == new ChildEntries([new ChildEntries.Entry('sc-abc_1', 'linux/amd64')]) + new ChildEntries([new ChildEntries.Entry('sc-abc_1', null)]) != new ChildEntries([new ChildEntries.Entry('sc-def_2', null)]) } // -- template binding tests -- @@ -151,7 +142,10 @@ class ChildEntriesTest extends Specification { def 'should populate scan binding for multi-platform scanChildIds'() { given: def binding = new HashMap() - def scanChildIds = new ChildEntries('sc-abc_1:linux/amd64,sc-def_2:linux/arm64') + def scanChildIds = new ChildEntries([ + new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), + new ChildEntries.Entry('sc-def_2', 'linux/arm64') + ]) when: ChildEntries.populateScanBinding(binding, null, scanChildIds, true, 'https://wave.io') @@ -169,7 +163,10 @@ class ChildEntriesTest extends Specification { def 'should populate scan binding when not succeeded'() { given: def binding = new HashMap() - def scanChildIds = new ChildEntries('sc-abc_1:linux/amd64,sc-def_2:linux/arm64') + def scanChildIds = new ChildEntries([ + new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), + new ChildEntries.Entry('sc-def_2', 'linux/arm64') + ]) when: ChildEntries.populateScanBinding(binding, null, scanChildIds, false, 'https://wave.io') @@ -193,7 +190,10 @@ class ChildEntriesTest extends Specification { def 'should populate build binding for child builds'() { given: def binding = new HashMap() - def buildChildIds = new ChildEntries('bd-abc_0:linux/amd64,bd-def_0:linux/arm64') + def buildChildIds = new ChildEntries([ + new ChildEntries.Entry('bd-abc_0', 'linux/amd64'), + new ChildEntries.Entry('bd-def_0', 'linux/arm64') + ]) when: ChildEntries.populateBuildBinding(binding, buildChildIds, 'https://wave.io') @@ -210,31 +210,49 @@ class ChildEntriesTest extends Specification { def 'should roundtrip through Jackson serialization'() { given: def mapper = new ObjectMapper() - def original = new ChildEntries('sc-abc_1:linux/amd64,sc-def_2:linux/arm64') + def original = new ChildEntries([ + new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), + new ChildEntries.Entry('sc-def_2', 'linux/arm64') + ]) when: def json = mapper.writeValueAsString(original) def restored = mapper.readValue(json, ChildEntries) then: - json == '"sc-abc_1:linux/amd64,sc-def_2:linux/arm64"' + json == '[{"id":"sc-abc_1","platform":"linux/amd64"},{"id":"sc-def_2","platform":"linux/arm64"}]' restored == original - restored.decode().size() == 2 + restored.size() == 2 } - def 'should handle null in Jackson serialization'() { + def 'should deserialise from json string'() { given: def mapper = new ObjectMapper() - when: - def json = mapper.writeValueAsString([entries: null]) - then: - json == '{"entries":null}' + expect: + mapper.readValue(JSON, ChildEntries) == EXPECTED - when: - def restored = mapper.readValue('"sc-abc_1:linux/amd64"', ChildEntries) - then: - restored == new ChildEntries('sc-abc_1:linux/amd64') + where: + JSON | EXPECTED + '[]' | new ChildEntries([]) + '[{"id":"sc-1","platform":"linux/amd64"}]' | new ChildEntries([new ChildEntries.Entry('sc-1', 'linux/amd64')]) + '[{"id":"sc-1","platform":"linux/amd64"},{"id":"sc-2","platform":"linux/arm64"}]' | new ChildEntries([new ChildEntries.Entry('sc-1', 'linux/amd64'), new ChildEntries.Entry('sc-2', 'linux/arm64')]) + '[{"id":"sc-1","platform":null}]' | new ChildEntries([new ChildEntries.Entry('sc-1', null)]) + } + + def 'should serialise to json string'() { + given: + def mapper = new ObjectMapper() + + expect: + mapper.writeValueAsString(INPUT) == EXPECTED + + where: + INPUT | EXPECTED + new ChildEntries([]) | '[]' + new ChildEntries([new ChildEntries.Entry('sc-1', 'linux/amd64')]) | '[{"id":"sc-1","platform":"linux/amd64"}]' + new ChildEntries([new ChildEntries.Entry('sc-1', 'linux/amd64'), new ChildEntries.Entry('sc-2', 'linux/arm64')]) | '[{"id":"sc-1","platform":"linux/amd64"},{"id":"sc-2","platform":"linux/arm64"}]' + new ChildEntries([new ChildEntries.Entry('sc-1', null)]) | '[{"id":"sc-1","platform":null}]' } def 'should populate build binding when null'() { diff --git a/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy b/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy index 1e7f87bacd..81623cdb72 100644 --- a/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy +++ b/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy @@ -124,7 +124,7 @@ class ContainerPlatformTest extends Specification { then: platform.os == 'linux' platform.arch == 'amd64' - platform.archs == ['amd64', 'arm64'] + platform.platforms == [new ContainerPlatform.Platform('linux','amd64'), new ContainerPlatform.Platform('linux','arm64')] platform.isMultiArch() platform.toString() == 'linux/amd64,linux/arm64' } @@ -135,7 +135,7 @@ class ContainerPlatformTest extends Specification { then: platform.os == 'linux' platform.arch == 'amd64' - platform.archs == ['amd64'] + platform.platforms == [new ContainerPlatform.Platform('linux','amd64')] !platform.isMultiArch() platform.toString() == 'linux/amd64' } @@ -147,7 +147,7 @@ class ContainerPlatformTest extends Specification { then: roundTripped == original roundTripped.isMultiArch() - roundTripped.archs == ['amd64', 'arm64'] + roundTripped.platforms == [new ContainerPlatform.Platform('linux','amd64'), new ContainerPlatform.Platform('linux','arm64')] } def 'should have MULTI_PLATFORM constant' () { @@ -155,7 +155,7 @@ class ContainerPlatformTest extends Specification { ContainerPlatform.MULTI_PLATFORM.isMultiArch() ContainerPlatform.MULTI_PLATFORM.os == 'linux' ContainerPlatform.MULTI_PLATFORM.arch == 'amd64' - ContainerPlatform.MULTI_PLATFORM.archs == ['amd64', 'arm64'] + ContainerPlatform.MULTI_PLATFORM.platforms == [new ContainerPlatform.Platform('linux','amd64'), new ContainerPlatform.Platform('linux','arm64')] ContainerPlatform.MULTI_PLATFORM.toString() == 'linux/amd64,linux/arm64' } diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy index e72ae904d9..6b93d7813d 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy @@ -22,15 +22,19 @@ import spock.lang.Specification import java.nio.file.Path import java.time.Duration +import java.time.Instant import java.time.OffsetDateTime import io.seqera.wave.api.BuildCompression import io.seqera.wave.api.BuildContext import io.seqera.wave.api.ContainerConfig +import io.seqera.wave.core.ChildEntries import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User import io.seqera.wave.util.ContainerHelper +import io.seqera.wave.util.JacksonHelper +import groovy.json.JsonOutput /** * * @author Paolo Di Tommaso @@ -270,6 +274,99 @@ class BuildRequestTest extends Specification { req7.offsetId == 'UTC+2' } + def 'should serialise and deserialise via Jackson'() { + given: + def buildChildIds = new ChildEntries([ + new ChildEntries.Entry('bd-abc_0', 'linux/amd64'), + new ChildEntries.Entry('bd-def_0', 'linux/arm64') + ]) + def scanChildIds = new ChildEntries([ + new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), + new ChildEntries.Entry('sc-def_2', 'linux/arm64') + ]) + def request = BuildRequest.of( + containerId: 'abc123', + containerFile: 'FROM ubuntu', + condaFile: 'samtools=1.0', + workspace: Path.of('/some/workspace'), + targetImage: 'docker.io/wave:abc123', + identity: new PlatformId(new User(id: 1, email: 'foo@user.com')), + platform: ContainerPlatform.of('linux/amd64'), + cacheRepository: 'docker.io/cache', + startTime: Instant.parse('2024-01-15T10:30:00Z'), + ip: '10.20.30.40', + configJson: '{"config":"json"}', + offsetId: '+02:00', + scanId: 'sc-main', + format: BuildFormat.DOCKER, + maxDuration: Duration.ofMinutes(10), + compression: BuildCompression.gzip, + buildId: 'bd-abc123_0', + buildTemplate: 'some-template', + noEmail: true, + buildChildIds: buildChildIds, + scanChildIds: scanChildIds + ) + + when: + def json = JsonOutput.prettyPrint(JacksonHelper.toJson(request)) + + then: + json == '''\ + { + "containerId": "abc123", + "containerFile": "FROM ubuntu", + "condaFile": "samtools=1.0", + "workspace": "file:///some/workspace", + "targetImage": "docker.io/wave:abc123", + "identity": { + "user": { + "id": 1, + "email": "foo@user.com" + }, + "userId": 1, + "userEmail": "foo@user.com" + }, + "platform": "linux/amd64", + "cacheRepository": "docker.io/cache", + "startTime": "2024-01-15T10:30:00Z", + "ip": "10.20.30.40", + "configJson": "{\\"config\\":\\"json\\"}", + "offsetId": "+02:00", + "scanId": "sc-main", + "format": "DOCKER", + "maxDuration": 600.000000000, + "compression": { + "mode": "gzip" + }, + "buildId": "bd-abc123_0", + "buildTemplate": "some-template", + "noEmail": true, + "buildChildIds": [ + { + "id": "bd-abc_0", + "platform": "linux/amd64" + }, + { + "id": "bd-def_0", + "platform": "linux/arm64" + } + ], + "scanChildIds": [ + { + "id": "sc-abc_1", + "platform": "linux/amd64" + }, + { + "id": "sc-def_2", + "platform": "linux/arm64" + } + ], + "dockerFile": "FROM ubuntu", + "workDir": "file:///some/workspace/bd-abc123_0" + }'''.stripIndent() + } + def 'should parse legacy id' () { expect: BuildRequest.legacyBuildId(BUILD_ID) == EXPECTED diff --git a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy index e9ed508cbe..4838bf2de0 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy @@ -474,7 +474,10 @@ class ContainerScanServiceImplTest extends Specification { def containerId = 'container1234' def workspace = Path.of('/some/workspace') def platform = ContainerPlatform.of('linux/amd64,linux/arm64') - def scanChildIds = new ChildEntries('sc-abc_1:linux/amd64,sc-def_2:linux/arm64') + def scanChildIds = new ChildEntries([ + new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), + new ChildEntries.Entry('sc-def_2', 'linux/arm64') + ]) final build = BuildRequest.of( containerId: containerId, From 059e5d55fb81249b581af54892948a48f928d7d3 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Mar 2026 21:19:05 +0100 Subject: [PATCH 09/15] Replace multiPlatform flag with multi-arch containerPlatform detection Remove the boolean multiPlatform field from SubmitContainerTokenRequest. Multi-platform builds are now triggered by specifying a multi-arch containerPlatform value (e.g. "linux/amd64,linux/arm64"). Add validation that only the linux/amd64+arm64 pair is currently allowed. Add @JsonPropertyOrder to BuildRequest to fix CI field ordering. Co-Authored-By: Claude Opus 4.6 --- .../controller/ContainerController.groovy | 29 ++++++++++--------- .../wave/service/builder/BuildRequest.groovy | 2 ++ .../wave/api/SubmitContainerTokenRequest.java | 12 -------- 3 files changed, 18 insertions(+), 25 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index 0f82619d81..74ed764005 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -260,8 +260,20 @@ class ContainerController { if( v2 && req.packages && req.freeze && !validationService.isCustomRepo(req.buildRepository) && !buildConfig.defaultPublicRepository ) throw new BadRequestException("Attribute `buildRepository` must be specified when using freeze mode [3]") + if( v2 && req.packages ) { + // generate the container file required to assemble the container + final generated = containerFileFromRequest(req) + req = req.copyWith(containerFile: generated.bytes.encodeBase64().toString()) + } + // make sure container platform is defined + if( !req.containerPlatform ) + req.containerPlatform = ContainerPlatform.DEFAULT.toString() + // multi-platform validation - if( req.multiPlatform ) { + final parsedPlatform = ContainerPlatform.of(req.containerPlatform) + if( parsedPlatform.isMultiArch() ) { + if( parsedPlatform != ContainerPlatform.MULTI_PLATFORM ) + throw new BadRequestException("Only linux/amd64,linux/arm64 multi-platform combination is currently supported") if( !req.containerFile && !req.packages ) throw new BadRequestException("Multi-platform builds require either 'containerFile' or 'packages' attribute") if( req.formatSingularity() ) @@ -270,15 +282,6 @@ class ContainerController { throw new UnsupportedBuildServiceException() } - if( v2 && req.packages ) { - // generate the container file required to assemble the container - final generated = containerFileFromRequest(req) - req = req.copyWith(containerFile: generated.bytes.encodeBase64().toString()) - } - // make sure container platform is defined - if( !req.containerPlatform ) - req.containerPlatform = ContainerPlatform.DEFAULT.toString() - final ip = addressResolver.resolve(httpRequest) // check the rate limit before continuing if( rateLimiterService ) @@ -406,9 +409,9 @@ class ContainerController { } protected ChildEntries makeChildScanIds(BuildRequest build, SubmitContainerTokenRequest req) { - if( !scanService || !req.multiPlatform ) + if( !scanService || !build.platform.isMultiArch() ) return null - final multiPlatform = ContainerPlatform.MULTI_PLATFORM + final multiPlatform = build.platform final scanMode = req.scanMode!=null ? req.scanMode : ScanMode.async final scanIdByPlatform = new LinkedHashMap() for( ContainerPlatform.Platform p : multiPlatform.platforms ) { @@ -509,7 +512,7 @@ class ContainerController { String scanId ChildEntries scanChildIds = null Boolean succeeded - if( req.containerFile && req.multiPlatform ) { + if( req.containerFile && ContainerPlatform.of(req.containerPlatform).isMultiArch() ) { if( !buildService ) throw new UnsupportedBuildServiceException() final build0 = makeBuildRequest(req, identity, ip) // create per-platform scan IDs for multi-arch builds diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy index 3c4273f335..952e9b0816 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy @@ -24,6 +24,7 @@ import java.time.Instant import java.time.OffsetDateTime import java.time.ZoneId +import com.fasterxml.jackson.annotation.JsonPropertyOrder import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import io.seqera.wave.api.BuildCompression @@ -40,6 +41,7 @@ import static io.seqera.wave.util.StringUtils.trunc * * @author Paolo Di Tommaso */ +@JsonPropertyOrder(['containerId','containerFile','condaFile','workspace','targetImage','identity','platform','cacheRepository','startTime','ip','configJson','offsetId','scanId','format','maxDuration','compression','buildId','buildTemplate','noEmail','buildChildIds','scanChildIds']) @EqualsAndHashCode(includes = 'containerId,targetImage,buildId') @CompileStatic class BuildRequest { diff --git a/wave-api/src/main/java/io/seqera/wave/api/SubmitContainerTokenRequest.java b/wave-api/src/main/java/io/seqera/wave/api/SubmitContainerTokenRequest.java index 05c29ce42a..99d85d4eb2 100644 --- a/wave-api/src/main/java/io/seqera/wave/api/SubmitContainerTokenRequest.java +++ b/wave-api/src/main/java/io/seqera/wave/api/SubmitContainerTokenRequest.java @@ -165,10 +165,6 @@ public class SubmitContainerTokenRequest implements Cloneable { */ public String buildTemplate; - /** - * When {@code true}, build a multi-platform (linux/amd64 + linux/arm64) container image - */ - public boolean multiPlatform; public SubmitContainerTokenRequest copyWith(Map opts) { try { @@ -225,8 +221,6 @@ public SubmitContainerTokenRequest copyWith(Map opts) { copy.buildCompression = (BuildCompression) opts.get("buildCompression"); if( opts.containsKey("buildTemplate")) copy.buildTemplate = (String) opts.get("buildTemplate"); - if( opts.containsKey("multiPlatform")) - copy.multiPlatform = (boolean) opts.get("multiPlatform"); // done return copy; } @@ -372,11 +366,6 @@ public SubmitContainerTokenRequest withBuildTemplate(String template) { return this; } - public SubmitContainerTokenRequest withMultiPlatform(boolean value) { - this.multiPlatform = value; - return this; - } - public boolean formatSingularity() { return "sif".equals(format); } @@ -410,7 +399,6 @@ public String toString() { ", scanLevels=" + scanLevels + ", buildCompression=" + buildCompression + ", buildTemplate=" + buildTemplate + - ", multiPlatform=" + multiPlatform + '}'; } } From 8ae134f4b95c4fee4ea46291e80b6f9d5138428a Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Mar 2026 22:13:16 +0100 Subject: [PATCH 10/15] Rewrite ChildEntries as plain POJO for Moshi serialization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ChildEntries no longer extends ArrayList — it wraps a List field that Moshi can serialize/deserialize correctly through BuildStateStore. Replace Jackson-based tests with Moshi roundtrip tests using the same MoshiEncodeStrategy as production. Remove unused @JsonPropertyOrder from BuildRequest. Co-Authored-By: Claude Opus 4.6 --- .../io/seqera/wave/core/ChildEntries.groovy | 42 +++++++-- .../wave/service/builder/BuildRequest.groovy | 2 - .../seqera/wave/core/ChildEntriesTest.groovy | 46 ++++----- .../service/builder/BuildRequestTest.groovy | 93 +++++++------------ .../persistence/WaveBuildRecordTest.groovy | 25 ++++- 5 files changed, 119 insertions(+), 89 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/core/ChildEntries.groovy b/src/main/groovy/io/seqera/wave/core/ChildEntries.groovy index e67f0f0e70..095ca46ee6 100644 --- a/src/main/groovy/io/seqera/wave/core/ChildEntries.groovy +++ b/src/main/groovy/io/seqera/wave/core/ChildEntries.groovy @@ -24,12 +24,13 @@ import groovy.transform.EqualsAndHashCode /** * A list of per-platform child IDs (builds or scans). * - * Serialises to/from JSON as: [{"id":"sc-abc_1","platform":"linux/amd64"}, ...] + * Serialises to/from JSON via Moshi as: {"entries":[{"id":"sc-abc_1","platform":"linux/amd64"}, ...]} * * @author Paolo Di Tommaso */ @CompileStatic -class ChildEntries extends ArrayList { +@EqualsAndHashCode(includes = 'entries') +class ChildEntries implements Iterable { @CompileStatic @EqualsAndHashCode @@ -43,14 +44,21 @@ class ChildEntries extends ArrayList { this.id = id this.platform = platform } + + @Override + String toString() { + return "${id}:${platform}" + } } + List entries + ChildEntries() { - super() + this.entries = [] } ChildEntries(List entries) { - super(entries ?: Collections.emptyList()) + this.entries = entries != null ? new ArrayList<>(entries) : [] } /** @@ -68,18 +76,40 @@ class ChildEntries extends ArrayList { return result } + // -- delegate methods -- + + int size() { entries.size() } + + boolean isEmpty() { entries.isEmpty() } + + Entry getAt(int index) { entries[index] } + + void add(Entry entry) { entries.add(entry) } + + boolean asBoolean() { entries != null && !entries.isEmpty() } + + @Override + Iterator iterator() { entries.iterator() } + + def List collect(Closure closure) { entries.collect(closure) } + /** * Get the first/primary ID */ String primary() { - return this ? this[0].id : null + return entries ? entries[0].id : null } /** * Get all IDs */ List allIds() { - return this.collect { it.id } + return entries.collect { it.id } + } + + @Override + String toString() { + return entries.collect { it.toString() }.toString() } // -- template binding helpers -- diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy index 952e9b0816..3c4273f335 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy @@ -24,7 +24,6 @@ import java.time.Instant import java.time.OffsetDateTime import java.time.ZoneId -import com.fasterxml.jackson.annotation.JsonPropertyOrder import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import io.seqera.wave.api.BuildCompression @@ -41,7 +40,6 @@ import static io.seqera.wave.util.StringUtils.trunc * * @author Paolo Di Tommaso */ -@JsonPropertyOrder(['containerId','containerFile','condaFile','workspace','targetImage','identity','platform','cacheRepository','startTime','ip','configJson','offsetId','scanId','format','maxDuration','compression','buildId','buildTemplate','noEmail','buildChildIds','scanChildIds']) @EqualsAndHashCode(includes = 'containerId,targetImage,buildId') @CompileStatic class BuildRequest { diff --git a/src/test/groovy/io/seqera/wave/core/ChildEntriesTest.groovy b/src/test/groovy/io/seqera/wave/core/ChildEntriesTest.groovy index 5b451a0b08..6616ffdbb5 100644 --- a/src/test/groovy/io/seqera/wave/core/ChildEntriesTest.groovy +++ b/src/test/groovy/io/seqera/wave/core/ChildEntriesTest.groovy @@ -18,7 +18,7 @@ package io.seqera.wave.core -import com.fasterxml.jackson.databind.ObjectMapper +import com.squareup.moshi.Moshi import spock.lang.Specification /** @@ -207,52 +207,56 @@ class ChildEntriesTest extends Specification { binding.build_entries[1].build_url == 'https://wave.io/view/builds/bd-def_0' } - def 'should roundtrip through Jackson serialization'() { + def 'should roundtrip through Moshi serialization'() { given: - def mapper = new ObjectMapper() + def moshi = new Moshi.Builder().build() + def adapter = moshi.adapter(ChildEntries) def original = new ChildEntries([ new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), new ChildEntries.Entry('sc-def_2', 'linux/arm64') ]) when: - def json = mapper.writeValueAsString(original) - def restored = mapper.readValue(json, ChildEntries) + def json = adapter.toJson(original) + def restored = adapter.fromJson(json) then: - json == '[{"id":"sc-abc_1","platform":"linux/amd64"},{"id":"sc-def_2","platform":"linux/arm64"}]' restored == original restored.size() == 2 + restored[0].id == 'sc-abc_1' + restored[0].platform == 'linux/amd64' + restored[1].id == 'sc-def_2' + restored[1].platform == 'linux/arm64' } - def 'should deserialise from json string'() { + def 'should deserialise from Moshi json string'() { given: - def mapper = new ObjectMapper() + def moshi = new Moshi.Builder().build() + def adapter = moshi.adapter(ChildEntries) expect: - mapper.readValue(JSON, ChildEntries) == EXPECTED + adapter.fromJson(JSON) == EXPECTED where: - JSON | EXPECTED - '[]' | new ChildEntries([]) - '[{"id":"sc-1","platform":"linux/amd64"}]' | new ChildEntries([new ChildEntries.Entry('sc-1', 'linux/amd64')]) - '[{"id":"sc-1","platform":"linux/amd64"},{"id":"sc-2","platform":"linux/arm64"}]' | new ChildEntries([new ChildEntries.Entry('sc-1', 'linux/amd64'), new ChildEntries.Entry('sc-2', 'linux/arm64')]) - '[{"id":"sc-1","platform":null}]' | new ChildEntries([new ChildEntries.Entry('sc-1', null)]) + JSON | EXPECTED + '{"entries":[]}' | new ChildEntries([]) + '{"entries":[{"id":"sc-1","platform":"linux/amd64"}]}' | new ChildEntries([new ChildEntries.Entry('sc-1', 'linux/amd64')]) + '{"entries":[{"id":"sc-1","platform":"linux/amd64"},{"id":"sc-2","platform":"linux/arm64"}]}' | new ChildEntries([new ChildEntries.Entry('sc-1', 'linux/amd64'), new ChildEntries.Entry('sc-2', 'linux/arm64')]) } - def 'should serialise to json string'() { + def 'should serialise to Moshi json string'() { given: - def mapper = new ObjectMapper() + def moshi = new Moshi.Builder().build() + def adapter = moshi.adapter(ChildEntries) expect: - mapper.writeValueAsString(INPUT) == EXPECTED + adapter.toJson(INPUT) == EXPECTED where: INPUT | EXPECTED - new ChildEntries([]) | '[]' - new ChildEntries([new ChildEntries.Entry('sc-1', 'linux/amd64')]) | '[{"id":"sc-1","platform":"linux/amd64"}]' - new ChildEntries([new ChildEntries.Entry('sc-1', 'linux/amd64'), new ChildEntries.Entry('sc-2', 'linux/arm64')]) | '[{"id":"sc-1","platform":"linux/amd64"},{"id":"sc-2","platform":"linux/arm64"}]' - new ChildEntries([new ChildEntries.Entry('sc-1', null)]) | '[{"id":"sc-1","platform":null}]' + new ChildEntries([]) | '{"entries":[]}' + new ChildEntries([new ChildEntries.Entry('sc-1', 'linux/amd64')]) | '{"entries":[{"id":"sc-1","platform":"linux/amd64"}]}' + new ChildEntries([new ChildEntries.Entry('sc-1', 'linux/amd64'), new ChildEntries.Entry('sc-2', 'linux/arm64')]) | '{"entries":[{"id":"sc-1","platform":"linux/amd64"},{"id":"sc-2","platform":"linux/arm64"}]}' } def 'should populate build binding when null'() { diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy index 6b93d7813d..6acecedd02 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy @@ -28,13 +28,13 @@ import java.time.OffsetDateTime import io.seqera.wave.api.BuildCompression import io.seqera.wave.api.BuildContext import io.seqera.wave.api.ContainerConfig -import io.seqera.wave.core.ChildEntries + import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User +import io.seqera.wave.core.ChildEntries import io.seqera.wave.util.ContainerHelper -import io.seqera.wave.util.JacksonHelper -import groovy.json.JsonOutput +import io.seqera.serde.moshi.MoshiEncodeStrategy /** * * @author Paolo Di Tommaso @@ -274,7 +274,7 @@ class BuildRequestTest extends Specification { req7.offsetId == 'UTC+2' } - def 'should serialise and deserialise via Jackson'() { + def 'should serialise and deserialise via Moshi'() { given: def buildChildIds = new ChildEntries([ new ChildEntries.Entry('bd-abc_0', 'linux/amd64'), @@ -307,64 +307,41 @@ class BuildRequestTest extends Specification { buildChildIds: buildChildIds, scanChildIds: scanChildIds ) + // use the same encode strategy as BuildStateStoreImpl + def encoder = new MoshiEncodeStrategy() {} + def entry = BuildEntry.create(request) when: - def json = JsonOutput.prettyPrint(JacksonHelper.toJson(request)) + def json = encoder.encode(entry) + def restored = encoder.decode(json) then: - json == '''\ - { - "containerId": "abc123", - "containerFile": "FROM ubuntu", - "condaFile": "samtools=1.0", - "workspace": "file:///some/workspace", - "targetImage": "docker.io/wave:abc123", - "identity": { - "user": { - "id": 1, - "email": "foo@user.com" - }, - "userId": 1, - "userEmail": "foo@user.com" - }, - "platform": "linux/amd64", - "cacheRepository": "docker.io/cache", - "startTime": "2024-01-15T10:30:00Z", - "ip": "10.20.30.40", - "configJson": "{\\"config\\":\\"json\\"}", - "offsetId": "+02:00", - "scanId": "sc-main", - "format": "DOCKER", - "maxDuration": 600.000000000, - "compression": { - "mode": "gzip" - }, - "buildId": "bd-abc123_0", - "buildTemplate": "some-template", - "noEmail": true, - "buildChildIds": [ - { - "id": "bd-abc_0", - "platform": "linux/amd64" - }, - { - "id": "bd-def_0", - "platform": "linux/arm64" - } - ], - "scanChildIds": [ - { - "id": "sc-abc_1", - "platform": "linux/amd64" - }, - { - "id": "sc-def_2", - "platform": "linux/arm64" - } - ], - "dockerFile": "FROM ubuntu", - "workDir": "file:///some/workspace/bd-abc123_0" - }'''.stripIndent() + restored.request.containerId == 'abc123' + restored.request.containerFile == 'FROM ubuntu' + restored.request.condaFile == 'samtools=1.0' + restored.request.targetImage == 'docker.io/wave:abc123' + restored.request.platform == ContainerPlatform.of('linux/amd64') + restored.request.cacheRepository == 'docker.io/cache' + restored.request.ip == '10.20.30.40' + restored.request.configJson == '{"config":"json"}' + restored.request.offsetId == '+02:00' + restored.request.scanId == 'sc-main' + restored.request.format == BuildFormat.DOCKER + restored.request.buildId == 'bd-abc123_0' + restored.request.buildTemplate == 'some-template' + restored.request.noEmail == true + and: + restored.request.buildChildIds.size() == 2 + restored.request.buildChildIds[0].id == 'bd-abc_0' + restored.request.buildChildIds[0].platform == 'linux/amd64' + restored.request.buildChildIds[1].id == 'bd-def_0' + restored.request.buildChildIds[1].platform == 'linux/arm64' + and: + restored.request.scanChildIds.size() == 2 + restored.request.scanChildIds[0].id == 'sc-abc_1' + restored.request.scanChildIds[0].platform == 'linux/amd64' + restored.request.scanChildIds[1].id == 'sc-def_2' + restored.request.scanChildIds[1].platform == 'linux/arm64' } def 'should parse legacy id' () { diff --git a/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy index cc0069ecc1..310c21bd1b 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy @@ -27,6 +27,7 @@ import java.time.Instant import io.seqera.wave.api.BuildCompression import io.seqera.wave.api.BuildStatusResponse +import io.seqera.wave.core.ChildEntries import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildFormat @@ -56,7 +57,15 @@ class WaveBuildRecordTest extends Specification { scanId: 'scan12345', format: BuildFormat.DOCKER, maxDuration: Duration.ofMinutes(1), - compression: BuildCompression.gzip + compression: BuildCompression.gzip, + buildChildIds: new ChildEntries([ + new ChildEntries.Entry('bd-abc_0', 'linux/amd64'), + new ChildEntries.Entry('bd-def_0', 'linux/arm64') + ]), + scanChildIds: new ChildEntries([ + new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), + new ChildEntries.Entry('sc-def_2', 'linux/arm64') + ]) ) final result = new BuildResult(request.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) final event = new BuildEvent(request, result) @@ -65,7 +74,19 @@ class WaveBuildRecordTest extends Specification { when: def json = JacksonHelper.toJson(record) then: - JacksonHelper.fromJson(json, WaveBuildRecord) == record + def restored = JacksonHelper.fromJson(json, WaveBuildRecord) + restored.buildId == record.buildId + restored.targetImage == record.targetImage + restored.buildChildIds.size() == 2 + restored.buildChildIds[0].id == 'bd-abc_0' + restored.buildChildIds[0].platform == 'linux/amd64' + restored.buildChildIds[1].id == 'bd-def_0' + restored.buildChildIds[1].platform == 'linux/arm64' + restored.scanChildIds.size() == 2 + restored.scanChildIds[0].id == 'sc-abc_1' + restored.scanChildIds[0].platform == 'linux/amd64' + restored.scanChildIds[1].id == 'sc-def_2' + restored.scanChildIds[1].platform == 'linux/arm64' } From 9a0318e8ffd55b458a8682dee6b94a9d6e532c29 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 3 Mar 2026 22:26:25 +0100 Subject: [PATCH 11/15] =?UTF-8?q?Rename=20ChildEntries=20to=20ChildRefs=20?= =?UTF-8?q?with=20Entry=E2=86=92Ref=20and=20platform=E2=86=92value?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 --- .../controller/ContainerController.groovy | 8 +- .../wave/controller/ViewController.groovy | 10 +- .../{ChildEntries.groovy => ChildRefs.groovy} | 68 +++++----- .../wave/service/builder/BuildRequest.groovy | 12 +- .../builder/MultiPlatformBuildService.groovy | 8 +- .../persistence/WaveBuildRecord.groovy | 6 +- .../persistence/WaveContainerRecord.groovy | 4 +- .../service/request/ContainerRequest.groovy | 10 +- .../request/ContainerStatusServiceImpl.groovy | 4 +- .../scan/ContainerScanServiceImpl.groovy | 6 +- ...ntriesTest.groovy => ChildRefsTest.groovy} | 120 +++++++++--------- .../service/builder/BuildRequestTest.groovy | 22 ++-- .../persistence/WaveBuildRecordTest.groovy | 22 ++-- .../scan/ContainerScanServiceImplTest.groovy | 8 +- 14 files changed, 154 insertions(+), 154 deletions(-) rename src/main/groovy/io/seqera/wave/core/{ChildEntries.groovy => ChildRefs.groovy} (55%) rename src/test/groovy/io/seqera/wave/core/{ChildEntriesTest.groovy => ChildRefsTest.groovy} (56%) diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index 74ed764005..6f1a06c120 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -77,7 +77,7 @@ import io.seqera.wave.service.request.ContainerRequestService import io.seqera.wave.service.request.ContainerStatusService import io.seqera.wave.service.request.TokenData import io.seqera.wave.service.scan.ContainerScanService -import io.seqera.wave.core.ChildEntries +import io.seqera.wave.core.ChildRefs import io.seqera.wave.service.validation.ValidationService import io.seqera.wave.service.validation.ValidationServiceImpl import io.seqera.wave.tower.PlatformId @@ -408,7 +408,7 @@ class ContainerController { ) } - protected ChildEntries makeChildScanIds(BuildRequest build, SubmitContainerTokenRequest req) { + protected ChildRefs makeChildScanIds(BuildRequest build, SubmitContainerTokenRequest req) { if( !scanService || !build.platform.isMultiArch() ) return null final multiPlatform = build.platform @@ -420,7 +420,7 @@ class ContainerController { if( id ) scanIdByPlatform.put(id, platform) } - return ChildEntries.of(scanIdByPlatform) + return ChildRefs.of(scanIdByPlatform) } protected BuildTrack checkBuild(BuildRequest build, boolean dryRun) { @@ -510,7 +510,7 @@ class ContainerController { String buildId boolean buildNew String scanId - ChildEntries scanChildIds = null + ChildRefs scanChildIds = null Boolean succeeded if( req.containerFile && ContainerPlatform.of(req.containerPlatform).isMultiArch() ) { if( !buildService ) throw new UnsupportedBuildServiceException() diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy index e0fe778763..5ea4bf581e 100644 --- a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy @@ -56,7 +56,7 @@ import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveScanRecord import io.seqera.wave.service.scan.ContainerScanService import io.seqera.wave.service.scan.ScanEntry -import io.seqera.wave.core.ChildEntries +import io.seqera.wave.core.ChildRefs import io.seqera.wave.service.scan.ScanType import io.seqera.wave.service.scan.ScanVulnerability import io.seqera.wave.util.JacksonHelper @@ -138,7 +138,7 @@ class ViewController { binding.mirror_digest = result.digest ?: '-' binding.mirror_user = result.userName ?: '-' binding.put('server_url', serverUrl) - ChildEntries.populateScanBinding(binding, result.scanId, null, result.succeeded(), serverUrl) + ChildRefs.populateScanBinding(binding, result.scanId, null, result.succeeded(), serverUrl) return binding } @@ -254,8 +254,8 @@ class ViewController { binding.build_condafile = result.condaFile binding.build_digest = result.digest ?: '-' binding.put('server_url', serverUrl) - ChildEntries.populateScanBinding(binding, result.scanId, result.scanChildIds, result.succeeded(), serverUrl) - ChildEntries.populateBuildBinding(binding, result.buildChildIds, serverUrl) + ChildRefs.populateScanBinding(binding, result.scanId, result.scanChildIds, result.succeeded(), serverUrl) + ChildRefs.populateBuildBinding(binding, result.buildChildIds, serverUrl) // inspect uri binding.inspect_url = result.succeeded() ? "$serverUrl/view/inspect?image=${result.targetImage}&platform=${result.platform}" : null // configure build logs when available @@ -314,7 +314,7 @@ class ViewController { binding.build_url = data.buildId ? "$serverUrl/view/builds/${data.buildId}" : null binding.fusion_version = data.fusionVersion ?: '-' - ChildEntries.populateScanBinding(binding, data.scanId, data.scanChildIds, true, serverUrl) + ChildRefs.populateScanBinding(binding, data.scanId, data.scanChildIds, true, serverUrl) binding.mirror_id = data.mirror ? data.buildId : null binding.mirror_url = data.mirror ? "$serverUrl/view/mirrors/${data.buildId}" : null diff --git a/src/main/groovy/io/seqera/wave/core/ChildEntries.groovy b/src/main/groovy/io/seqera/wave/core/ChildRefs.groovy similarity index 55% rename from src/main/groovy/io/seqera/wave/core/ChildEntries.groovy rename to src/main/groovy/io/seqera/wave/core/ChildRefs.groovy index 095ca46ee6..eb3c065038 100644 --- a/src/main/groovy/io/seqera/wave/core/ChildEntries.groovy +++ b/src/main/groovy/io/seqera/wave/core/ChildRefs.groovy @@ -24,100 +24,100 @@ import groovy.transform.EqualsAndHashCode /** * A list of per-platform child IDs (builds or scans). * - * Serialises to/from JSON via Moshi as: {"entries":[{"id":"sc-abc_1","platform":"linux/amd64"}, ...]} + * Serialises to/from JSON via Moshi as: {"refs":[{"id":"sc-abc_1","value":"linux/amd64"}, ...]} * * @author Paolo Di Tommaso */ @CompileStatic -@EqualsAndHashCode(includes = 'entries') -class ChildEntries implements Iterable { +@EqualsAndHashCode(includes = 'refs') +class ChildRefs implements Iterable { @CompileStatic @EqualsAndHashCode - static class Entry { + static class Ref { String id - String platform + String value - Entry() {} + Ref() {} - Entry(String id, String platform) { + Ref(String id, String value) { this.id = id - this.platform = platform + this.value = value } @Override String toString() { - return "${id}:${platform}" + return "${id}:${value}" } } - List entries + List refs - ChildEntries() { - this.entries = [] + ChildRefs() { + this.refs = [] } - ChildEntries(List entries) { - this.entries = entries != null ? new ArrayList<>(entries) : [] + ChildRefs(List refs) { + this.refs = refs != null ? new ArrayList<>(refs) : [] } /** * Create from a map of id-by-platform. * @param idByPlatform Map where keys are IDs and values are platform strings - * @return A new ChildEntries, or null if the map is null/empty + * @return A new ChildRefs, or null if the map is null/empty */ - static ChildEntries of(Map idByPlatform) { + static ChildRefs of(Map idByPlatform) { if( !idByPlatform ) return null - final result = new ChildEntries() + final result = new ChildRefs() for( Map.Entry it : idByPlatform.entrySet() ) { - result.add(new Entry(it.key, it.value)) + result.add(new Ref(it.key, it.value)) } return result } // -- delegate methods -- - int size() { entries.size() } + int size() { refs.size() } - boolean isEmpty() { entries.isEmpty() } + boolean isEmpty() { refs.isEmpty() } - Entry getAt(int index) { entries[index] } + Ref getAt(int index) { refs[index] } - void add(Entry entry) { entries.add(entry) } + void add(Ref entry) { refs.add(entry) } - boolean asBoolean() { entries != null && !entries.isEmpty() } + boolean asBoolean() { refs != null && !refs.isEmpty() } @Override - Iterator iterator() { entries.iterator() } + Iterator iterator() { refs.iterator() } - def List collect(Closure closure) { entries.collect(closure) } + def List collect(Closure closure) { refs.collect(closure) } /** * Get the first/primary ID */ String primary() { - return entries ? entries[0].id : null + return refs ? refs[0].id : null } /** * Get all IDs */ List allIds() { - return entries.collect { it.id } + return refs.collect { it.id } } @Override String toString() { - return entries.collect { it.toString() }.toString() + return refs.collect { it.toString() }.toString() } // -- template binding helpers -- - static void populateScanBinding(Map binding, String scanId, ChildEntries scanChildIds, boolean succeeded, String serverUrl) { + static void populateScanBinding(Map binding, String scanId, ChildRefs scanChildIds, boolean succeeded, String serverUrl) { if( scanChildIds && succeeded ) { - binding.scan_entries = scanChildIds.collect { Entry entry -> - [scan_id: entry.id, scan_platform: entry.platform, scan_url: "${serverUrl}/view/scans/${entry.id}"] as Map + binding.scan_entries = scanChildIds.collect { Ref entry -> + [scan_id: entry.id, scan_platform: entry.value, scan_url: "${serverUrl}/view/scans/${entry.id}"] as Map } binding.scan_url = null binding.scan_id = scanChildIds.primary() @@ -129,10 +129,10 @@ class ChildEntries implements Iterable { } } - static void populateBuildBinding(Map binding, ChildEntries buildChildIds, String serverUrl) { + static void populateBuildBinding(Map binding, ChildRefs buildChildIds, String serverUrl) { if( buildChildIds ) { - binding.build_entries = buildChildIds.collect { Entry entry -> - [build_id: entry.id, build_platform: entry.platform, build_url: "${serverUrl}/view/builds/${entry.id}"] as Map + binding.build_entries = buildChildIds.collect { Ref entry -> + [build_id: entry.id, build_platform: entry.value, build_url: "${serverUrl}/view/builds/${entry.id}"] as Map } } else { diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy index 3c4273f335..df28bb4bd2 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy @@ -29,7 +29,7 @@ import groovy.transform.EqualsAndHashCode import io.seqera.wave.api.BuildCompression import io.seqera.wave.api.BuildContext import io.seqera.wave.api.ContainerConfig -import io.seqera.wave.core.ChildEntries +import io.seqera.wave.core.ChildRefs import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.tower.PlatformId import static io.seqera.wave.service.builder.BuildFormat.DOCKER @@ -158,12 +158,12 @@ class BuildRequest { /** * Child build IDs for multi-platform builds */ - final ChildEntries buildChildIds + final ChildRefs buildChildIds /** * Child scan IDs for multi-platform builds */ - final ChildEntries scanChildIds + final ChildRefs scanChildIds BuildRequest( String containerId, @@ -236,15 +236,15 @@ class BuildRequest { this.buildId = opts.buildId ?: computeBuildId(containerId) this.buildTemplate = opts.buildTemplate this.noEmail = opts.noEmail as boolean - this.buildChildIds = opts.buildChildIds as ChildEntries - this.scanChildIds = opts.scanChildIds as ChildEntries + this.buildChildIds = opts.buildChildIds as ChildRefs + this.scanChildIds = opts.scanChildIds as ChildRefs } static BuildRequest of(Map opts) { new BuildRequest(opts) } - BuildRequest withChildScanIds(ChildEntries scanChildIds) { + BuildRequest withChildScanIds(ChildRefs scanChildIds) { return BuildRequest.of( containerId: this.containerId, containerFile: this.containerFile, diff --git a/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy index 3b1d4bf02a..9e6239fb60 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy @@ -23,7 +23,7 @@ import java.time.Instant import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.event.ApplicationEventPublisher -import io.seqera.wave.core.ChildEntries +import io.seqera.wave.core.ChildRefs import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.model.ContainerCoordinates import io.seqera.wave.service.job.JobHandler @@ -107,9 +107,9 @@ class MultiPlatformBuildService implements JobHandler { final startTime = Instant.now() // Encode child build IDs for the parent build view - final buildChildIds = new ChildEntries([ - new ChildEntries.Entry(amd64Track.id, PLATFORM_AMD64.toString()), - new ChildEntries.Entry(arm64Track.id, PLATFORM_ARM64.toString()) + final buildChildIds = new ChildRefs([ + new ChildRefs.Ref(amd64Track.id, PLATFORM_AMD64.toString()), + new ChildRefs.Ref(arm64Track.id, PLATFORM_ARM64.toString()) ]) // Store an in-progress build entry so status polling can find it diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy index 0d3c316ece..9d05426618 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveBuildRecord.groovy @@ -26,7 +26,7 @@ import groovy.transform.EqualsAndHashCode import groovy.transform.ToString import io.seqera.wave.api.BuildCompression import io.seqera.wave.api.BuildStatusResponse -import io.seqera.wave.core.ChildEntries +import io.seqera.wave.core.ChildRefs import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildFormat @@ -62,8 +62,8 @@ class WaveBuildRecord { String digest BuildCompression compression String buildTemplate - ChildEntries buildChildIds - ChildEntries scanChildIds + ChildRefs buildChildIds + ChildRefs scanChildIds Boolean succeeded() { return duration != null ? (exitStatus==0) : null diff --git a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy index f685f6f85c..e591d9c72f 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/WaveContainerRecord.groovy @@ -28,7 +28,7 @@ import groovy.transform.ToString import groovy.util.logging.Slf4j import io.seqera.wave.api.ContainerConfig import io.seqera.wave.api.SubmitContainerTokenRequest -import io.seqera.wave.core.ChildEntries +import io.seqera.wave.core.ChildRefs import io.seqera.wave.service.request.ContainerRequest import io.seqera.wave.tower.User import io.seqera.wave.util.FusionVersionStringDeserializer @@ -181,7 +181,7 @@ class WaveContainerRecord { /** * Child scan IDs for multi-platform builds */ - final ChildEntries scanChildIds + final ChildRefs scanChildIds WaveContainerRecord(SubmitContainerTokenRequest request, ContainerRequest data, String waveImage, String addr, Instant expiration) { this.id = data.requestId diff --git a/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy index 5536d375e1..1420ff36bd 100644 --- a/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerRequest.groovy @@ -25,7 +25,7 @@ import groovy.transform.CompileStatic import io.seqera.wave.api.ContainerConfig import io.seqera.wave.api.ScanLevel import io.seqera.wave.api.ScanMode -import io.seqera.wave.core.ChildEntries +import io.seqera.wave.core.ChildRefs import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.model.ContainerCoordinates import io.seqera.wave.tower.PlatformId @@ -54,7 +54,7 @@ class ContainerRequest { Boolean buildNew Boolean freeze String scanId - ChildEntries scanChildIds + ChildRefs scanChildIds ScanMode scanMode List scanLevels boolean dryRun @@ -147,7 +147,7 @@ class ContainerRequest { return scanId } - ChildEntries getScanChildIds() { + ChildRefs getScanChildIds() { return scanChildIds } @@ -183,7 +183,7 @@ class ContainerRequest { Boolean buildNew, Boolean freeze, String scanId, - ChildEntries scanChildIds, + ChildRefs scanChildIds, ScanMode scanMode, List scanLevels, boolean dryRun, @@ -231,7 +231,7 @@ class ContainerRequest { (Boolean) data.buildNew, (Boolean) data.freeze, data.scanId as String, - data.scanChildIds as ChildEntries, + data.scanChildIds as ChildRefs, data.scanMode as ScanMode, data.scanLevels as List, data.dryRun as boolean, diff --git a/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy index 66d634bbd0..9edb5e87a6 100644 --- a/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/request/ContainerStatusServiceImpl.groovy @@ -18,7 +18,7 @@ package io.seqera.wave.service.request -import io.seqera.wave.core.ChildEntries +import io.seqera.wave.core.ChildRefs import io.seqera.wave.exception.UnsupportedBuildServiceException import io.seqera.wave.exception.UnsupportedMirrorServiceException import io.seqera.wave.exception.UnsupportedScanServiceException @@ -186,7 +186,7 @@ class ContainerStatusServiceImpl implements ContainerStatusService { boolean allDone = true boolean allSucceeded = true Duration maxScanDuration = Duration.ZERO - for( ChildEntries.Entry pair : request.scanChildIds ) { + for( ChildRefs.Ref pair : request.scanChildIds ) { final scan = getScanState(pair.id) if( !scan ) throw new NotFoundException("Missing container scan record with id: ${pair.id}") diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index fd868e45fd..29833c090c 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -27,7 +27,7 @@ import java.util.concurrent.ExecutorService import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import io.seqera.wave.core.ChildEntries +import io.seqera.wave.core.ChildRefs import io.micronaut.context.annotation.Requires import io.micronaut.core.annotation.Nullable import io.micronaut.http.server.types.files.StreamedFile @@ -131,8 +131,8 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler */ -class ChildEntriesTest extends Specification { +class ChildRefsTest extends Specification { def 'should create from map with single entry'() { when: - def result = ChildEntries.of(['sc-abc_1': 'linux/amd64']) + def result = ChildRefs.of(['sc-abc_1': 'linux/amd64']) then: result.size() == 1 result[0].id == 'sc-abc_1' - result[0].platform == 'linux/amd64' + result[0].value == 'linux/amd64' } def 'should create from map with multiple entries'() { @@ -44,54 +44,54 @@ class ChildEntriesTest extends Specification { map.put('sc-def_2', 'linux/arm64') when: - def result = ChildEntries.of(map) + def result = ChildRefs.of(map) then: result.size() == 2 result[0].id == 'sc-abc_1' - result[0].platform == 'linux/amd64' + result[0].value == 'linux/amd64' result[1].id == 'sc-def_2' - result[1].platform == 'linux/arm64' + result[1].value == 'linux/arm64' } def 'should return null from null or empty map'() { expect: - ChildEntries.of(null) == null - ChildEntries.of([:]) == null + ChildRefs.of(null) == null + ChildRefs.of([:]) == null } def 'should create from list of entries'() { when: - def result = new ChildEntries([ - new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), - new ChildEntries.Entry('sc-def_2', 'linux/arm64') + def result = new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') ]) then: result.size() == 2 result[0].id == 'sc-abc_1' - result[0].platform == 'linux/amd64' + result[0].value == 'linux/amd64' result[1].id == 'sc-def_2' - result[1].platform == 'linux/arm64' + result[1].value == 'linux/arm64' } def 'should get primary id'() { expect: - new ChildEntries([new ChildEntries.Entry('sc-abc_1', null)]).primary() == 'sc-abc_1' + new ChildRefs([new ChildRefs.Ref('sc-abc_1', null)]).primary() == 'sc-abc_1' and: - new ChildEntries([ - new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), - new ChildEntries.Entry('sc-def_2', 'linux/arm64') + new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') ]).primary() == 'sc-abc_1' and: - new ChildEntries([]).primary() == null + new ChildRefs([]).primary() == null } def 'should get all ids'() { expect: - new ChildEntries([new ChildEntries.Entry('sc-abc_1', null)]).allIds() == ['sc-abc_1'] + new ChildRefs([new ChildRefs.Ref('sc-abc_1', null)]).allIds() == ['sc-abc_1'] and: - new ChildEntries([ - new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), - new ChildEntries.Entry('sc-def_2', 'linux/arm64') + new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') ]).allIds() == ['sc-abc_1', 'sc-def_2'] } @@ -102,27 +102,27 @@ class ChildEntriesTest extends Specification { map.put('sc-def_2', 'linux/arm64') when: - def result = ChildEntries.of(map) + def result = ChildRefs.of(map) then: result.size() == 2 result[0].id == 'sc-abc_1' - result[0].platform == 'linux/amd64' + result[0].value == 'linux/amd64' result[1].id == 'sc-def_2' - result[1].platform == 'linux/arm64' + result[1].value == 'linux/arm64' } def 'should support groovy truth'() { expect: - new ChildEntries([new ChildEntries.Entry('sc-abc_1', null)]) as boolean == true - new ChildEntries(null) as boolean == false - new ChildEntries([]) as boolean == false + new ChildRefs([new ChildRefs.Ref('sc-abc_1', null)]) as boolean == true + new ChildRefs(null) as boolean == false + new ChildRefs([]) as boolean == false } def 'should support equality'() { expect: - new ChildEntries([new ChildEntries.Entry('sc-abc_1', 'linux/amd64')]) == new ChildEntries([new ChildEntries.Entry('sc-abc_1', 'linux/amd64')]) - new ChildEntries([new ChildEntries.Entry('sc-abc_1', null)]) != new ChildEntries([new ChildEntries.Entry('sc-def_2', null)]) + new ChildRefs([new ChildRefs.Ref('sc-abc_1', 'linux/amd64')]) == new ChildRefs([new ChildRefs.Ref('sc-abc_1', 'linux/amd64')]) + new ChildRefs([new ChildRefs.Ref('sc-abc_1', null)]) != new ChildRefs([new ChildRefs.Ref('sc-def_2', null)]) } // -- template binding tests -- @@ -132,7 +132,7 @@ class ChildEntriesTest extends Specification { def binding = new HashMap() when: - ChildEntries.populateScanBinding(binding, 'sc-abc_1', null, true, 'https://wave.io') + ChildRefs.populateScanBinding(binding, 'sc-abc_1', null, true, 'https://wave.io') then: binding.scan_id == 'sc-abc_1' binding.scan_url == 'https://wave.io/view/scans/sc-abc_1' @@ -142,13 +142,13 @@ class ChildEntriesTest extends Specification { def 'should populate scan binding for multi-platform scanChildIds'() { given: def binding = new HashMap() - def scanChildIds = new ChildEntries([ - new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), - new ChildEntries.Entry('sc-def_2', 'linux/arm64') + def scanChildIds = new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') ]) when: - ChildEntries.populateScanBinding(binding, null, scanChildIds, true, 'https://wave.io') + ChildRefs.populateScanBinding(binding, null, scanChildIds, true, 'https://wave.io') then: binding.scan_id == 'sc-abc_1' binding.scan_url == null @@ -163,13 +163,13 @@ class ChildEntriesTest extends Specification { def 'should populate scan binding when not succeeded'() { given: def binding = new HashMap() - def scanChildIds = new ChildEntries([ - new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), - new ChildEntries.Entry('sc-def_2', 'linux/arm64') + def scanChildIds = new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') ]) when: - ChildEntries.populateScanBinding(binding, null, scanChildIds, false, 'https://wave.io') + ChildRefs.populateScanBinding(binding, null, scanChildIds, false, 'https://wave.io') then: binding.scan_id == null binding.scan_url == null @@ -181,7 +181,7 @@ class ChildEntriesTest extends Specification { def binding = new HashMap() when: - ChildEntries.populateScanBinding(binding, null, null, true, 'https://wave.io') + ChildRefs.populateScanBinding(binding, null, null, true, 'https://wave.io') then: binding.scan_id == null binding.scan_url == null @@ -190,13 +190,13 @@ class ChildEntriesTest extends Specification { def 'should populate build binding for child builds'() { given: def binding = new HashMap() - def buildChildIds = new ChildEntries([ - new ChildEntries.Entry('bd-abc_0', 'linux/amd64'), - new ChildEntries.Entry('bd-def_0', 'linux/arm64') + def buildChildIds = new ChildRefs([ + new ChildRefs.Ref('bd-abc_0', 'linux/amd64'), + new ChildRefs.Ref('bd-def_0', 'linux/arm64') ]) when: - ChildEntries.populateBuildBinding(binding, buildChildIds, 'https://wave.io') + ChildRefs.populateBuildBinding(binding, buildChildIds, 'https://wave.io') then: binding.build_entries.size() == 2 binding.build_entries[0].build_id == 'bd-abc_0' @@ -210,10 +210,10 @@ class ChildEntriesTest extends Specification { def 'should roundtrip through Moshi serialization'() { given: def moshi = new Moshi.Builder().build() - def adapter = moshi.adapter(ChildEntries) - def original = new ChildEntries([ - new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), - new ChildEntries.Entry('sc-def_2', 'linux/arm64') + def adapter = moshi.adapter(ChildRefs) + def original = new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') ]) when: @@ -224,39 +224,39 @@ class ChildEntriesTest extends Specification { restored == original restored.size() == 2 restored[0].id == 'sc-abc_1' - restored[0].platform == 'linux/amd64' + restored[0].value == 'linux/amd64' restored[1].id == 'sc-def_2' - restored[1].platform == 'linux/arm64' + restored[1].value == 'linux/arm64' } def 'should deserialise from Moshi json string'() { given: def moshi = new Moshi.Builder().build() - def adapter = moshi.adapter(ChildEntries) + def adapter = moshi.adapter(ChildRefs) expect: adapter.fromJson(JSON) == EXPECTED where: JSON | EXPECTED - '{"entries":[]}' | new ChildEntries([]) - '{"entries":[{"id":"sc-1","platform":"linux/amd64"}]}' | new ChildEntries([new ChildEntries.Entry('sc-1', 'linux/amd64')]) - '{"entries":[{"id":"sc-1","platform":"linux/amd64"},{"id":"sc-2","platform":"linux/arm64"}]}' | new ChildEntries([new ChildEntries.Entry('sc-1', 'linux/amd64'), new ChildEntries.Entry('sc-2', 'linux/arm64')]) + '{"refs":[]}' | new ChildRefs([]) + '{"refs":[{"id":"sc-1","value":"linux/amd64"}]}' | new ChildRefs([new ChildRefs.Ref('sc-1', 'linux/amd64')]) + '{"refs":[{"id":"sc-1","value":"linux/amd64"},{"id":"sc-2","value":"linux/arm64"}]}' | new ChildRefs([new ChildRefs.Ref('sc-1', 'linux/amd64'), new ChildRefs.Ref('sc-2', 'linux/arm64')]) } def 'should serialise to Moshi json string'() { given: def moshi = new Moshi.Builder().build() - def adapter = moshi.adapter(ChildEntries) + def adapter = moshi.adapter(ChildRefs) expect: adapter.toJson(INPUT) == EXPECTED where: INPUT | EXPECTED - new ChildEntries([]) | '{"entries":[]}' - new ChildEntries([new ChildEntries.Entry('sc-1', 'linux/amd64')]) | '{"entries":[{"id":"sc-1","platform":"linux/amd64"}]}' - new ChildEntries([new ChildEntries.Entry('sc-1', 'linux/amd64'), new ChildEntries.Entry('sc-2', 'linux/arm64')]) | '{"entries":[{"id":"sc-1","platform":"linux/amd64"},{"id":"sc-2","platform":"linux/arm64"}]}' + new ChildRefs([]) | '{"refs":[]}' + new ChildRefs([new ChildRefs.Ref('sc-1', 'linux/amd64')]) | '{"refs":[{"id":"sc-1","value":"linux/amd64"}]}' + new ChildRefs([new ChildRefs.Ref('sc-1', 'linux/amd64'), new ChildRefs.Ref('sc-2', 'linux/arm64')]) | '{"refs":[{"id":"sc-1","value":"linux/amd64"},{"id":"sc-2","value":"linux/arm64"}]}' } def 'should populate build binding when null'() { @@ -264,7 +264,7 @@ class ChildEntriesTest extends Specification { def binding = new HashMap() when: - ChildEntries.populateBuildBinding(binding, null, 'https://wave.io') + ChildRefs.populateBuildBinding(binding, null, 'https://wave.io') then: binding.build_entries == null } diff --git a/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy index 6acecedd02..19a4b2872e 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/BuildRequestTest.groovy @@ -32,7 +32,7 @@ import io.seqera.wave.api.ContainerConfig import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.tower.PlatformId import io.seqera.wave.tower.User -import io.seqera.wave.core.ChildEntries +import io.seqera.wave.core.ChildRefs import io.seqera.wave.util.ContainerHelper import io.seqera.serde.moshi.MoshiEncodeStrategy /** @@ -276,13 +276,13 @@ class BuildRequestTest extends Specification { def 'should serialise and deserialise via Moshi'() { given: - def buildChildIds = new ChildEntries([ - new ChildEntries.Entry('bd-abc_0', 'linux/amd64'), - new ChildEntries.Entry('bd-def_0', 'linux/arm64') + def buildChildIds = new ChildRefs([ + new ChildRefs.Ref('bd-abc_0', 'linux/amd64'), + new ChildRefs.Ref('bd-def_0', 'linux/arm64') ]) - def scanChildIds = new ChildEntries([ - new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), - new ChildEntries.Entry('sc-def_2', 'linux/arm64') + def scanChildIds = new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') ]) def request = BuildRequest.of( containerId: 'abc123', @@ -333,15 +333,15 @@ class BuildRequestTest extends Specification { and: restored.request.buildChildIds.size() == 2 restored.request.buildChildIds[0].id == 'bd-abc_0' - restored.request.buildChildIds[0].platform == 'linux/amd64' + restored.request.buildChildIds[0].value == 'linux/amd64' restored.request.buildChildIds[1].id == 'bd-def_0' - restored.request.buildChildIds[1].platform == 'linux/arm64' + restored.request.buildChildIds[1].value == 'linux/arm64' and: restored.request.scanChildIds.size() == 2 restored.request.scanChildIds[0].id == 'sc-abc_1' - restored.request.scanChildIds[0].platform == 'linux/amd64' + restored.request.scanChildIds[0].value == 'linux/amd64' restored.request.scanChildIds[1].id == 'sc-def_2' - restored.request.scanChildIds[1].platform == 'linux/arm64' + restored.request.scanChildIds[1].value == 'linux/arm64' } def 'should parse legacy id' () { diff --git a/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy index 310c21bd1b..36310712bc 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/WaveBuildRecordTest.groovy @@ -27,7 +27,7 @@ import java.time.Instant import io.seqera.wave.api.BuildCompression import io.seqera.wave.api.BuildStatusResponse -import io.seqera.wave.core.ChildEntries +import io.seqera.wave.core.ChildRefs import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.builder.BuildEvent import io.seqera.wave.service.builder.BuildFormat @@ -58,13 +58,13 @@ class WaveBuildRecordTest extends Specification { format: BuildFormat.DOCKER, maxDuration: Duration.ofMinutes(1), compression: BuildCompression.gzip, - buildChildIds: new ChildEntries([ - new ChildEntries.Entry('bd-abc_0', 'linux/amd64'), - new ChildEntries.Entry('bd-def_0', 'linux/arm64') + buildChildIds: new ChildRefs([ + new ChildRefs.Ref('bd-abc_0', 'linux/amd64'), + new ChildRefs.Ref('bd-def_0', 'linux/arm64') ]), - scanChildIds: new ChildEntries([ - new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), - new ChildEntries.Entry('sc-def_2', 'linux/arm64') + scanChildIds: new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') ]) ) final result = new BuildResult(request.buildId, -1, "ok", Instant.now(), Duration.ofSeconds(3), null) @@ -79,14 +79,14 @@ class WaveBuildRecordTest extends Specification { restored.targetImage == record.targetImage restored.buildChildIds.size() == 2 restored.buildChildIds[0].id == 'bd-abc_0' - restored.buildChildIds[0].platform == 'linux/amd64' + restored.buildChildIds[0].value == 'linux/amd64' restored.buildChildIds[1].id == 'bd-def_0' - restored.buildChildIds[1].platform == 'linux/arm64' + restored.buildChildIds[1].value == 'linux/arm64' restored.scanChildIds.size() == 2 restored.scanChildIds[0].id == 'sc-abc_1' - restored.scanChildIds[0].platform == 'linux/amd64' + restored.scanChildIds[0].value == 'linux/amd64' restored.scanChildIds[1].id == 'sc-def_2' - restored.scanChildIds[1].platform == 'linux/arm64' + restored.scanChildIds[1].value == 'linux/arm64' } diff --git a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy index 4838bf2de0..b08dc3931e 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy @@ -29,7 +29,7 @@ import java.time.Instant import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.api.ScanMode import io.seqera.wave.configuration.ScanConfig -import io.seqera.wave.core.ChildEntries +import io.seqera.wave.core.ChildRefs import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildFormat @@ -474,9 +474,9 @@ class ContainerScanServiceImplTest extends Specification { def containerId = 'container1234' def workspace = Path.of('/some/workspace') def platform = ContainerPlatform.of('linux/amd64,linux/arm64') - def scanChildIds = new ChildEntries([ - new ChildEntries.Entry('sc-abc_1', 'linux/amd64'), - new ChildEntries.Entry('sc-def_2', 'linux/arm64') + def scanChildIds = new ChildRefs([ + new ChildRefs.Ref('sc-abc_1', 'linux/amd64'), + new ChildRefs.Ref('sc-def_2', 'linux/arm64') ]) final build = BuildRequest.of( From bd32c39976c6d188f0e7718cda202f1c78dee688 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 4 Mar 2026 10:02:34 +0100 Subject: [PATCH 12/15] Add validation to reject multi-platform values in non-build endpoints Add ContainerPlatform.validateSinglePlatform() helper and use it in InspectController to prevent comma-separated platform values (e.g. linux/amd64,linux/arm64) which should only be allowed for container build requests. Co-Authored-By: Claude Opus 4.6 --- .../wave/controller/InspectController.groovy | 4 ++++ .../seqera/wave/core/ContainerPlatform.groovy | 11 +++++++++ .../wave/core/ContainerPlatformTest.groovy | 23 +++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/src/main/groovy/io/seqera/wave/controller/InspectController.groovy b/src/main/groovy/io/seqera/wave/controller/InspectController.groovy index 506b0c6c98..e99957f863 100644 --- a/src/main/groovy/io/seqera/wave/controller/InspectController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/InspectController.groovy @@ -32,6 +32,7 @@ import io.micronaut.scheduling.TaskExecutors import io.micronaut.scheduling.annotation.ExecuteOn import io.seqera.wave.api.ContainerInspectRequest import io.seqera.wave.api.ContainerInspectResponse +import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.exception.BadRequestException import io.seqera.wave.service.UserService import io.seqera.wave.service.inspect.ContainerInspectService @@ -74,6 +75,9 @@ class InspectController { if( !req.containerImage ) throw new BadRequestException("Missing 'containerImage' attribute") + // multi-platform values are not allowed for inspect requests + ContainerPlatform.validateSinglePlatform(platform) + // this is needed for backward compatibility with old clients if( !req.towerEndpoint ) { req.towerEndpoint = towerEndpointUrl diff --git a/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy b/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy index fde462f78b..2b8c3b710c 100644 --- a/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy +++ b/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy @@ -180,6 +180,17 @@ class ContainerPlatform { return value ? of(value) : defaultPlatform } + /** + * Validate that the given platform string is a single platform value (not multi-arch). + * Throws {@link BadRequestException} if the value contains comma-separated platforms. + * + * @param value The platform string to validate + */ + static void validateSinglePlatform(String value) { + if( value && value.contains(',') ) + throw new BadRequestException("Container multi-platform architecture not allowed - offending value: $value") + } + @JsonCreator static ContainerPlatform of(String value) { if( !value ) diff --git a/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy b/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy index 81623cdb72..48ee54edaf 100644 --- a/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy +++ b/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy @@ -165,4 +165,27 @@ class ContainerPlatformTest extends Specification { ContainerPlatform.of('linux/amd64,linux/arm64') == ContainerPlatform.of('linux/amd64,linux/arm64') ContainerPlatform.of('linux/amd64') != ContainerPlatform.MULTI_PLATFORM } + + def 'should validate single platform allows valid values' () { + when: + ContainerPlatform.validateSinglePlatform(VALUE) + then: + noExceptionThrown() + + where: + VALUE | _ + null | _ + '' | _ + 'linux/amd64' | _ + 'linux/arm64' | _ + 'amd64' | _ + } + + def 'should validate single platform rejects multi-platform values' () { + when: + ContainerPlatform.validateSinglePlatform('linux/amd64,linux/arm64') + then: + def e = thrown(BadRequestException) + e.message.contains('Container multi-platform architecture not allowed') + } } From 971eb5f34995978918968248f26a23b582764d09 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 5 Mar 2026 17:21:15 +0100 Subject: [PATCH 13/15] [release] bump version Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 4a76bfdc14..d8ab306cc7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.32.5 \ No newline at end of file +1.33.0-B0 \ No newline at end of file From aa971fa91a38ce11325e3eb1d5f137c2ca428991 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 6 Mar 2026 11:30:54 +0100 Subject: [PATCH 14/15] Augment all matching platform manifests for multi-arch containers When containerPlatform is multi-arch, the resolution process now augments each matching platform manifest in the image index with correctly filtered arch-specific fusion layers (amd64 gets fusion-amd64.tar.gz, arm64 gets fusion-arm64.tar.gz). Key changes: - Add resolveImageIndex() and augmentManifest() to handle per-platform augmentation with arch-specific layer filtering - Add AugmentedManifest class replacing Tuple2 - Move matches() from ContainerPlatform to Platform inner class - Guard ContainerPlatform.os/arch/variant with exception on multi-arch - Move skip-cache setting from debug flag to BuildConfig (default false) - Fix body.bytes double computation in ManifestAssembler Co-Authored-By: Claude Opus 4.6 --- .../wave/configuration/BuildConfig.groovy | 3 + .../controller/ContainerController.groovy | 12 +- .../wave/core/ContainerAugmenter.groovy | 277 +++++++++++++++--- .../seqera/wave/core/ContainerPlatform.groovy | 83 +++--- .../filter/PullMetricsRequestsFilter.groovy | 2 +- .../service/builder/ManifestAssembler.groovy | 6 +- .../builder/MultiPlatformBuildService.groovy | 30 +- .../wave/core/ContainerAugmenterTest.groovy | 2 +- .../wave/core/ContainerPlatformTest.groovy | 41 ++- .../MultiPlatformBuildServiceTest.groovy | 66 +++++ 10 files changed, 427 insertions(+), 95 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy index 80c12b2ae0..4be2cb112f 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy @@ -152,6 +152,9 @@ class BuildConfig { @Value('${wave.build.logs.maxLength:100000}') long maxLength + @Value('${wave.build.skip-cache:false}') + boolean skipCache + @PostConstruct private void init() { log.info("Builder config: " + diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index 6f1a06c120..70a8e76cef 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -461,11 +461,13 @@ class ContainerController { } // check if the multi-platform image already exists - final digest = registryProxyService.getImageDigest(targetImage, identity) - if( digest ) { - log.debug "== Found cached multi-platform build for $targetImage" - final cache = persistenceService.loadBuildSucceed(targetImage, digest) - return new BuildTrack(cache?.buildId, targetImage, true, true) + if( !buildConfig.skipCache ) { + final digest = registryProxyService.getImageDigest(targetImage, identity) + if( digest ) { + log.debug "== Found cached multi-platform build for $targetImage" + final cache = persistenceService.loadBuildSucceed(targetImage, digest) + return new BuildTrack(cache?.buildId, targetImage, true, true) + } } // delegate to multi-platform build service diff --git a/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy b/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy index 3b73782add..22b9225df6 100644 --- a/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy +++ b/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy @@ -39,6 +39,7 @@ import io.seqera.wave.model.ContainerOrIndexSpec import io.seqera.wave.proxy.ProxyClient import io.seqera.wave.storage.Storage import io.seqera.wave.util.JacksonHelper +import io.seqera.wave.service.builder.MultiPlatformBuildService import io.seqera.wave.util.RegHelper import static io.seqera.wave.model.ContentType.DOCKER_IMAGE_CONFIG_V1 import static io.seqera.wave.model.ContentType.DOCKER_IMAGE_INDEX_V2 @@ -66,6 +67,16 @@ class ContainerAugmenter { final Boolean oci final ManifestSpec manifestSpec } + + /** + * Holds the digest and size of an augmented manifest, used to update + * entries in a manifest index after per-platform augmentation. + */ + @Canonical + static class AugmentedManifest { + final String digest + final int size + } private ProxyClient client private ContainerConfig containerConfig @@ -110,8 +121,14 @@ class ContainerAugmenter { this.platform = route.request?.platform ?: ContainerPlatform.DEFAULT // note: do not propagate container config when "freeze" mode is enabled, because it has been already // applied during the container build phase, and therefore it should be ignored by the augmenter - if( route.request?.containerConfig && !route.request.freeze ) - this.containerConfig = route.request.containerConfig + if( route.request?.containerConfig && !route.request.freeze ) { + log.debug "Augmenter resolve: platform=${this.platform}; layers=${route.request.containerConfig?.layers?.collect { it.location }}" + // for single-arch: filters fusion layers to keep only the matching arch + // for multi-arch: this is a no-op (returns all layers); per-platform filtering + // happens later in augmentManifest() when the actual image architecture is known + this.containerConfig = MultiPlatformBuildService.filterLayersForPlatform(route.request.containerConfig, this.platform) + log.debug "Augmenter resolve: filtered layers=${this.containerConfig?.layers?.collect { it.location }}" + } return resolve(route.image, route.reference, headers) } @@ -185,46 +202,211 @@ class ContainerAugmenter { return new ContainerDigestPair(digest, v1Digest) } - final manifestResult = findImageManifestAndDigest(manifestsList, imageName, tag, headers) + // Parse the manifest body to inspect its mediaType field. + // This determines whether we're dealing with a manifest index (multi-platform list) + // or a single platform manifest. + final parsedManifest = new JsonSlurper().parseText(manifestsList) as Map + final media = parsedManifest.mediaType as String + + // Manifest index (Docker v2 or OCI): + // Contains a list of platform-specific manifests. Delegate to resolveImageIndex + // which iterates over matching platform entries, augments each one separately + // with the correct arch-specific fusion layers, and updates the index. + // This handles both single-arch (one match) and multi-arch (multiple matches). + if( media==DOCKER_IMAGE_INDEX_V2 || media==OCI_IMAGE_INDEX_V1 ) { + return resolveImageIndex(imageName, tag, headers, manifestsList, digest, parsedManifest, media) + } - // fetch the image config - final resp5 = client.getString("/v2/$imageName/blobs/$manifestResult.configDigest", headers) - checkResponseCode(resp5, client.route, true) - final imageConfig = resp5.body() + // Single platform manifest (Docker v2 or OCI): + // No index wrapping — augment the manifest directly with fusion layers. + if( media==DOCKER_MANIFEST_V2_TYPE || media==OCI_IMAGE_MANIFEST_V1 ) { + final ref = ManifestSpec.of(parsedManifest) + // targetDigest is null because there's no parent index entry to update + final manifestInfo = new ManifestInfo(manifestsList, ref.config.digest, null, media==OCI_IMAGE_MANIFEST_V1, ref) + final result = augmentManifest(imageName, tag, headers, manifestInfo) + return new ContainerDigestPair(digest, result.digest) + } + + throw new IllegalArgumentException("Unexpected media type for request '$imageName:$tag' - offending value: $media") + } + + /** + * Resolve an image index (manifest list) by augmenting each matching platform manifest. + * + * An image index is a JSON document listing platform-specific manifests (e.g. linux/amd64, + * linux/arm64). This method iterates over those entries, finds the ones matching our target + * platform(s), and augments each one separately with the correct arch-specific fusion layers. + * + * For single-arch requests (e.g. platform=linux/amd64): only one entry matches, so we + * augment one manifest and update one index entry. + * + * For multi-arch requests (e.g. platform=linux/amd64,linux/arm64): multiple entries match, + * each gets augmented with its own filtered fusion layers (amd64 gets fusion-amd64.tar.gz, + * arm64 gets fusion-arm64.tar.gz), and all index entries are updated. + * + * @param imageName the repository name (e.g. "library/ubuntu") + * @param tag the image tag or reference + * @param headers HTTP headers for upstream registry requests + * @param manifestsList the raw JSON string of the manifest index + * @param originalDigest the original digest of the unmodified index + * @param indexJson the already-parsed index JSON (avoids double parsing) + * @param indexMedia the mediaType of the index (OCI or Docker) + * @return a pair of (original digest, new augmented index digest) + */ + protected ContainerDigestPair resolveImageIndex(String imageName, String tag, Map> headers, String manifestsList, String originalDigest, Map indexJson, String indexMedia) { + final oci = indexMedia == OCI_IMAGE_INDEX_V1 + // each entry in this list represents a platform-specific manifest (os/arch/variant) + final manifests = indexJson.manifests as List + // save the original container config (with ALL fusion layers for all platforms) + // because augmentManifest mutates this.containerConfig via filterLayersForPlatform, + // and we need to reset it before processing each platform + final originalConfig = this.containerConfig + // collect old digest → (new digest, new size) for each augmented platform + final digestUpdates = new LinkedHashMap() + + try { + for( Map indexEntry : manifests ) { + // each index entry has a 'platform' object with os/architecture/variant fields + final entryPlatform = indexEntry.platform as Map + if( !entryPlatform ) + continue + + // verify the entry's mediaType matches what we expect for this index type: + // OCI indexes reference OCI manifests, Docker indexes reference Docker v2 manifests + final entryMedia = indexEntry.mediaType as String + final expectedMedia = oci ? OCI_IMAGE_MANIFEST_V1 : DOCKER_MANIFEST_V2_TYPE + if( entryMedia != expectedMedia ) + continue + + // check if this entry's platform (e.g. linux/amd64) matches any of our + // target platforms — for single-arch this matches one, for multi-arch it + // matches each platform in turn + if( !matchesPlatformEntry(entryPlatform) ) + continue + + final targetDigest = indexEntry.digest as String + if( !targetDigest ) + continue + + // fetch the actual platform-specific manifest (not the index — the real manifest + // that contains the layer list and config reference for this architecture) + final resp = client.getString("/v2/$imageName/manifests/$targetDigest", headers) + checkResponseCode(resp, client.route, false) + final platformManifest = resp.body() as String + // parse the manifest to extract config digest, layer info, and mediaType + final manifestInfo = parseManifest(platformManifest, targetDigest) + if( !manifestInfo ) + continue + + // reset to the original unfiltered container config before each platform, + // because augmentManifest will filter it down to only the layers matching + // this specific architecture (e.g. keep fusion-amd64, discard fusion-arm64) + this.containerConfig = originalConfig + // augment this platform's manifest: fetch config, filter layers, inject layers + final result = augmentManifest(imageName, tag, headers, manifestInfo) + // record the mapping from old digest to new digest+size for the index update + digestUpdates.put(targetDigest, result) + } + } + finally { + // always restore the original container config, even if an exception occurs, + // to avoid leaving the augmenter in a partially-filtered state + this.containerConfig = originalConfig + } + + if( digestUpdates.isEmpty() ) { + throw new BadRequestException("Cannot find matching platform '${platform}' in the image index for '$imageName:$tag'") + } + + // rewrite the image index: replace old digests with new augmented digests, + // recompute the index's own digest, and store it in the cache + final newListDigest = updateImageIndex(imageName, manifestsList, digestUpdates, oci) + if( log.isTraceEnabled() ) + log.trace "resolveImageIndex: new index digest: $newListDigest (updated ${digestUpdates.size()} platform entries)" + // return: original unmodified index digest + new augmented index digest + return new ContainerDigestPair(originalDigest, newListDigest) + } + + /** + * Check if a manifest index entry's platform matches any of our target platforms. + * For single-arch (e.g. linux/amd64), the platforms list has one entry. + * For multi-arch (e.g. linux/amd64,linux/arm64), it has multiple entries. + * Delegates to Platform.matches() which handles arch aliases (x86_64→amd64) + * and variant normalization (arm64/v8→arm64). + */ + protected boolean matchesPlatformEntry(Map entryPlatform) { + return platform.platforms.any { it.matches(entryPlatform) } + } + + /** + * Augment a single platform manifest by injecting fusion layers and config changes. + * + * This is the core augmentation logic shared by both single-manifest and index-based + * resolution. It performs three steps: + * 1. Fetch the image config blob to determine the actual architecture + * 2. Filter fusion layers to keep only those matching the resolved architecture + * (e.g. for amd64: keep fusion-amd64.tar.gz, discard fusion-arm64.tar.gz) + * 3. Inject the filtered layers into both the image config and manifest + * + * NOTE: This method mutates this.containerConfig via filterLayersForPlatform. + * When called from resolveImageIndex, the caller must reset this.containerConfig + * before each invocation to ensure correct filtering per platform. + * + * @return a Tuple2 of (newManifestDigest, newManifestSize) for updating the parent index + */ + protected AugmentedManifest augmentManifest(String imageName, String tag, Map> headers, ManifestInfo manifestInfo) { + // fetch the image config blob — this JSON contains the architecture field, + // rootfs layer digests, and container config (env, entrypoint, etc.) + final resp = client.getString("/v2/$imageName/blobs/$manifestInfo.configDigest", headers) + checkResponseCode(resp, client.route, true) + final imageConfig = resp.body() as String if( log.isTraceEnabled() ) - log.trace "Resolve (5): image $imageName:$tag => image config=\n${JsonOutput.prettyPrint(imageConfig)}" + log.trace "Augment: image $imageName:$tag => image config=\n${JsonOutput.prettyPrint(imageConfig)}" + + // read the architecture from the image config to determine which fusion layers to keep. + // this is the authoritative source of the image's architecture — more reliable than + // the platform field in the manifest index, which may be missing or incorrect. + final configJson = new JsonSlurper().parseText(imageConfig) as Map + final resolvedArch = configJson.architecture as String + if( resolvedArch ) { + // filter this.containerConfig.layers to only include: + // - non-fusion layers (always kept, e.g. the wave launcher data layer) + // - fusion layers matching this architecture (e.g. fusion-amd64.tar.gz for amd64) + // fusion layers for other architectures are discarded + final resolvedPlatform = ContainerPlatform.of("linux/${resolvedArch}") + this.containerConfig = MultiPlatformBuildService.filterLayersForPlatform(this.containerConfig, resolvedPlatform) + log.debug "Augment: filtered layers for arch=${resolvedArch}; remaining=${this.containerConfig?.layers?.size()}" + } - // update the image config adding the new layer - final newConfigResult = updateImageConfig(imageName, imageConfig, manifestResult.oci) + // update the image config: add layer tar digests to rootfs.diff_ids, + // apply container config changes (env, entrypoint, cmd, workingDir), + // then store the new config blob and return its digest + final newConfigResult = updateImageConfig(imageName, imageConfig, manifestInfo.oci) final newConfigDigest = newConfigResult[0] final newConfigJson = newConfigResult[1] if( log.isTraceEnabled() ) - log.trace "Resolve (6) ==> new config digest: $newConfigDigest => new config=\n${JsonOutput.prettyPrint(newConfigJson)} " + log.trace "Augment: new config digest: $newConfigDigest" - // update the image manifest adding a new layer - // returns the updated image manifest digest - final newManifestResult = updateImageManifest(imageName, manifestResult.imageManifest, newConfigDigest, newConfigJson.size(), manifestResult.oci) - final newManifestDigest = newManifestResult.v1 - final newManifestSize = newManifestResult.v2 - if( log.isTraceEnabled() ) - log.trace "Resolve (7) ==> new image digest: $newManifestDigest" + // update the image manifest: append new layer blobs to the layers array, + // update the config reference to point to the new config digest, + // then store the new manifest and return its digest + size + final newManifestResult = updateImageManifest(imageName, manifestInfo.imageManifest, newConfigDigest, newConfigJson.size(), manifestInfo.oci) + return new AugmentedManifest(newManifestResult.v1, newManifestResult.v2) + } - if( !manifestResult.targetDigest ) { - return new ContainerDigestPair(digest, newManifestDigest) - } - else { - // update the manifests list with the new digest - // returns the manifests list digest - final newListDigest = updateImageIndex(imageName, manifestsList, manifestResult.targetDigest, newManifestDigest, newManifestSize, manifestResult.oci) - if( log.isTraceEnabled() ) - log.trace "Resolve (8) ==> new list digest: $newListDigest" - return new ContainerDigestPair(digest, newListDigest) - } + protected ManifestInfo parseManifest(String manifest, String targetDigest) { + final json = new JsonSlurper().parseText(manifest) as Map + final media = json.mediaType as String + return parseManifest(media, manifest, json, targetDigest) } protected ManifestInfo parseManifest(String media, String manifest, String targetDigest) { + final json = new JsonSlurper().parseText(manifest) as Map + return parseManifest(media, manifest, json, targetDigest) + } + + protected ManifestInfo parseManifest(String media, String manifest, Map json, String targetDigest) { if( media==DOCKER_MANIFEST_V2_TYPE || media==OCI_IMAGE_MANIFEST_V1 ) { - final json = new JsonSlurper().parseText(manifest) as Map final ref = ManifestSpec.of(json) return new ManifestInfo(manifest, ref.config.digest, targetDigest, media==OCI_IMAGE_MANIFEST_V1, ref) } @@ -264,26 +446,31 @@ class ContainerAugmenter { } - protected String updateImageIndex(String imageName, String manifestsList, String targetDigest, String newDigest, Integer newSize, boolean oci) { + /** + * Rewrite the image index by replacing old platform manifest digests with new augmented ones. + * Handles one or more digest replacements in a single pass, then stores the updated index. + */ + protected String updateImageIndex(String imageName, String manifestsList, Map digestUpdates, boolean oci) { + // parse the original index JSON final json = new JsonSlurper().parseText(manifestsList) as Map final list = json.manifests as List - final entry = list.find( it -> it.digest==targetDigest ) - if( !entry ) - throw new IllegalStateException("Missing manifest entry for digest: $targetDigest") - // update the target entry digest and size - entry.digest = newDigest - entry.size = newSize - // serialize to json again + + // replace each old digest with the new augmented digest and size + for( Map.Entry update : digestUpdates.entrySet() ) { + final entry = list.find( it -> it.digest == update.key ) + if( !entry ) + throw new IllegalStateException("Missing manifest entry for digest: ${update.key}") + entry.digest = update.value.digest + entry.size = update.value.size + } + final updated = JsonOutput.toJson(json) final result = RegHelper.digest(updated) final type = oci ? OCI_IMAGE_INDEX_V1 : DOCKER_IMAGE_INDEX_V2 - // make sure the manifest was updated - if( manifestsList==updated ) - throw new IllegalArgumentException("Unable to find target digest '$targetDigest' into image index") - // store in the cache + if( manifestsList == updated ) + throw new IllegalArgumentException("Unable to update image index - no entries were modified") final target = "$client.registry.name/v2/$imageName/manifests/$result" storage.saveManifest(target, updated, type, result) - // return the updated manifests list digest return result } @@ -420,11 +607,11 @@ class ContainerAugmenter { } protected boolean matchesDockerManifest(Map record) { - return record.mediaType == DOCKER_MANIFEST_V2_TYPE && platform.matches(record.platform as Map) + return record.mediaType == DOCKER_MANIFEST_V2_TYPE && matchesPlatformEntry(record.platform as Map) } protected boolean matchesOciManifest(Map record) { - return record.mediaType == OCI_IMAGE_MANIFEST_V1 && platform.matches(record.platform as Map) + return record.mediaType == OCI_IMAGE_MANIFEST_V1 && matchesPlatformEntry(record.platform as Map) } protected void rewriteHistoryV1( List history ){ diff --git a/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy b/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy index 2b8c3b710c..a6938f9280 100644 --- a/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy +++ b/src/main/groovy/io/seqera/wave/core/ContainerPlatform.groovy @@ -71,6 +71,45 @@ class ContainerPlatform { throw new BadRequestException("Invalid container platform: $value -- offending value: $value") } + /** + * Check if this platform matches a manifest index entry's platform record. + * Handles architecture aliases (x86_64→amd64, aarch64→arm64) and + * variant normalization (arm64/v8 matches arm64 with no variant). + * + * @param record a map with 'os', 'architecture', and optionally 'variant' keys + * as found in a manifest index entry's platform field + */ + boolean matches(Map record) { + return sameOs(record) && sameArch(record) && sameVariant(record) + } + + private boolean sameOs(Map record) { + this.os == record.os + } + + private boolean sameArch(Map record) { + if( this.arch==record.architecture ) + return true + if( this.arch=='amd64' && record.architecture in AMD64 ) + return true + if( this.arch=='arm64' && record.architecture in ARM64 ) + return true + else + return false + } + + private boolean sameVariant(Map record) { + if( this.variant == record.variant ) + return true + if( this.arch=='arm64' ) { + if( !this.variant && (!record.variant || record.variant in V8)) + return true + if( this.variant && record.variant==this.variant ) + return true + } + return false + } + @Override String toString() { def result = os + "/" + arch @@ -130,11 +169,19 @@ class ContainerPlatform { this.platforms = List.copyOf(platforms) } - String getOs() { platforms[0].os } + @Deprecated + String getOs() { requireSinglePlatform(); platforms[0].os } - String getArch() { platforms[0].arch } + @Deprecated + String getArch() { requireSinglePlatform(); platforms[0].arch } - String getVariant() { platforms[0].variant } + @Deprecated + String getVariant() { requireSinglePlatform(); platforms[0].variant } + + private void requireSinglePlatform() { + if( isMultiArch() ) + throw new IllegalStateException("Cannot access single-platform property on multi-arch ContainerPlatform: ${toString()}") + } boolean isMultiArch() { return platforms.size() > 1 @@ -145,36 +192,6 @@ class ContainerPlatform { return platforms.collect { it.toString() }.join(',') } - boolean matches(Map record) { - return sameOs(record) && sameArch(record) && sameVariant(record) - } - - private boolean sameOs(Map record) { - this.os == record.os - } - - private boolean sameArch(Map record) { - if( this.arch==record.architecture ) - return true - if( this.arch=='amd64' && record.architecture in AMD64 ) - return true - if( this.arch=='arm64' && record.architecture in ARM64 ) - return true - else - return false - } - - private boolean sameVariant(Map record) { - if( this.variant == record.variant ) - return true - if( this.arch=='arm64' ) { - if( !this.variant && (!record.variant || record.variant in V8)) - return true - if( this.variant && record.variant==this.variant ) - return true - } - return false - } static ContainerPlatform parseOrDefault(String value, ContainerPlatform defaultPlatform=DEFAULT) { return value ? of(value) : defaultPlatform diff --git a/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy b/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy index 4a7541cccc..d6c2b8895a 100644 --- a/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy +++ b/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy @@ -83,7 +83,7 @@ class PullMetricsRequestsFilter implements HttpServerFilter { final contentType = response.headers.get(HttpHeaders.CONTENT_TYPE) if( contentType && contentType in MANIFEST_TYPES ) { final route = routeHelper.parse(request.path) - final arch = route.request.platform.arch + final arch = route.request.platform?.platforms?.collect(it->it.arch)?.join(',') CompletableFuture.runAsync(() -> metricsService.incrementPullsCounter(route.identity, arch), executor) final version = route.request?.containerConfig?.fusionVersion() if (version) { diff --git a/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy b/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy index 9d5891de8a..2eff6a74fe 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/ManifestAssembler.groovy @@ -26,6 +26,7 @@ import io.seqera.wave.auth.RegistryAuthService import io.seqera.wave.auth.RegistryCredentialsProvider import io.seqera.wave.auth.RegistryLookupService import io.seqera.wave.configuration.HttpClientConfig +import io.seqera.wave.util.RegHelper import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.core.RegistryProxyService import io.seqera.wave.core.RoutePath @@ -93,9 +94,10 @@ class ManifestAssembler { throw new IllegalStateException("Failed to GET manifest for '$image' — status: ${resp.statusCode()}, body: ${resp.body()}") final body = resp.body() + final bodyBytes = body.bytes final contentType = resp.headers().firstValue('content-type').orElse(OCI_IMAGE_MANIFEST_V1) - final digest = resp.headers().firstValue('docker-content-digest').orElse(null) - final size = body.bytes.length + final digest = resp.headers().firstValue('docker-content-digest').orElse(RegHelper.digest(bodyBytes)) + final size = bodyBytes.length return [ mediaType: contentType, diff --git a/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy index 9e6239fb60..c79efdeadd 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/MultiPlatformBuildService.groovy @@ -19,10 +19,14 @@ package io.seqera.wave.service.builder import java.time.Instant +import java.util.regex.Pattern import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.event.ApplicationEventPublisher + +import io.seqera.wave.api.ContainerConfig +import io.seqera.wave.api.ContainerLayer import io.seqera.wave.core.ChildRefs import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.model.ContainerCoordinates @@ -250,15 +254,37 @@ class MultiPlatformBuildService implements JobHandler { } } + static final private Pattern FUSION_LAYER_ARCH = Pattern.compile('.*/fusion-(\\w+)\\.tar\\.gz$') + + /** + * Filter container config layers to only include fusion layers matching the target platform arch. + * Non-fusion layers are kept as-is. + */ + static ContainerConfig filterLayersForPlatform(ContainerConfig config, ContainerPlatform platform) { + if( !config?.layers || !platform || platform.isMultiArch() ) + return config + final filtered = config.layers.findAll { ContainerLayer layer -> + final matcher = layer.location ? FUSION_LAYER_ARCH.matcher(layer.location) : null + final keep = !matcher?.matches() || matcher.group(1) == platform.arch + if( matcher?.matches() ) { + log.debug "Layer filter: platform=${platform}; layer=${layer.location}; arch=${matcher.group(1)}; keep=${keep}" + } + return keep + } + log.debug "Layer filter: platform=${platform}; input=${config.layers.size()} layers; output=${filtered.size()} layers" + return new ContainerConfig(config.entrypoint, config.cmd, config.env, config.workingDir, filtered) + } + protected BuildRequest createPlatformRequest(BuildRequest template, ContainerPlatform platform, String suffix) { final repo = ContainerCoordinates.parse(template.targetImage).repository + final containerConfig = filterLayersForPlatform(template.containerConfig, platform) final platformId = makeContainerId( template.containerFile, template.condaFile, platform, repo, template.buildContext, - template.containerConfig + containerConfig ) final platformImage = makeTargetImage(template.format, repo, platformId, template.condaFile, null) @@ -274,7 +300,7 @@ class MultiPlatformBuildService implements JobHandler { template.ip, template.configJson, template.offsetId, - template.containerConfig, + containerConfig, null, // scanId - suppress per-platform scans; scan runs on the composite image template.buildContext, template.format, diff --git a/src/test/groovy/io/seqera/wave/core/ContainerAugmenterTest.groovy b/src/test/groovy/io/seqera/wave/core/ContainerAugmenterTest.groovy index 6dfbe04e53..f1abdb9301 100644 --- a/src/test/groovy/io/seqera/wave/core/ContainerAugmenterTest.groovy +++ b/src/test/groovy/io/seqera/wave/core/ContainerAugmenterTest.groovy @@ -301,7 +301,7 @@ class ContainerAugmenterTest extends Specification { .withContainerConfig(config) when: - def digest = scanner.updateImageIndex(IMAGE, MANIFEST, DIGEST, NEW_DIGEST, NEW_SIZE, false) + def digest = scanner.updateImageIndex(IMAGE, MANIFEST, [(DIGEST): new ContainerAugmenter.AugmentedManifest(NEW_DIGEST, NEW_SIZE)], false) then: def entry = storage.getManifest("$REGISTRY/v2/$IMAGE/manifests/$digest").get() diff --git a/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy b/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy index 48ee54edaf..da82d44c47 100644 --- a/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy +++ b/src/test/groovy/io/seqera/wave/core/ContainerPlatformTest.groovy @@ -83,7 +83,7 @@ class ContainerPlatformTest extends Specification { @Unroll def 'should match' () { expect: - ContainerPlatform.of(PLATFORM).matches(RECORD) + ContainerPlatform.of(PLATFORM).platforms[0].matches(RECORD) where: PLATFORM | RECORD 'amd64' | [os:'linux', architecture:'amd64'] @@ -106,7 +106,7 @@ class ContainerPlatformTest extends Specification { @Unroll def 'should not match' () { expect: - !ContainerPlatform.of(PLATFORM).matches(RECORD) + !ContainerPlatform.of(PLATFORM).platforms[0].matches(RECORD) where: PLATFORM | RECORD 'amd64' | [os:'linux', architecture:'arm64'] @@ -122,8 +122,8 @@ class ContainerPlatformTest extends Specification { when: def platform = ContainerPlatform.of('linux/amd64,linux/arm64') then: - platform.os == 'linux' - platform.arch == 'amd64' + platform.platforms[0].os == 'linux' + platform.platforms[0].arch == 'amd64' platform.platforms == [new ContainerPlatform.Platform('linux','amd64'), new ContainerPlatform.Platform('linux','arm64')] platform.isMultiArch() platform.toString() == 'linux/amd64,linux/arm64' @@ -153,8 +153,8 @@ class ContainerPlatformTest extends Specification { def 'should have MULTI_PLATFORM constant' () { expect: ContainerPlatform.MULTI_PLATFORM.isMultiArch() - ContainerPlatform.MULTI_PLATFORM.os == 'linux' - ContainerPlatform.MULTI_PLATFORM.arch == 'amd64' + ContainerPlatform.MULTI_PLATFORM.platforms[0].os == 'linux' + ContainerPlatform.MULTI_PLATFORM.platforms[0].arch == 'amd64' ContainerPlatform.MULTI_PLATFORM.platforms == [new ContainerPlatform.Platform('linux','amd64'), new ContainerPlatform.Platform('linux','arm64')] ContainerPlatform.MULTI_PLATFORM.toString() == 'linux/amd64,linux/arm64' } @@ -188,4 +188,33 @@ class ContainerPlatformTest extends Specification { def e = thrown(BadRequestException) e.message.contains('Container multi-platform architecture not allowed') } + + def 'should throw when accessing os/arch/variant on multi-arch platform' () { + given: + def platform = ContainerPlatform.of('linux/amd64,linux/arm64') + + when: + platform.os + then: + thrown(IllegalStateException) + + when: + platform.arch + then: + thrown(IllegalStateException) + + when: + platform.variant + then: + thrown(IllegalStateException) + } + + def 'should allow os/arch/variant on single-arch platform' () { + given: + def platform = ContainerPlatform.of('linux/amd64') + expect: + platform.os == 'linux' + platform.arch == 'amd64' + platform.variant == null + } } diff --git a/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy index 24f9dd4996..ee42644f9b 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/MultiPlatformBuildServiceTest.groovy @@ -25,6 +25,8 @@ import java.time.Duration import java.time.Instant import io.micronaut.context.event.ApplicationEventPublisher +import io.seqera.wave.api.ContainerConfig +import io.seqera.wave.api.ContainerLayer import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.job.JobService import io.seqera.wave.service.job.JobSpec @@ -310,6 +312,70 @@ class MultiPlatformBuildServiceTest extends Specification { 1 * eventPublisher.publishEvent(_ as BuildEvent) } + def 'should filter fusion layers for target platform'() { + given: + def amd64Layer = new ContainerLayer( + location: 'https://fusionfs.seqera.io/releases/pkg/2/4/20/fusion-amd64.tar.gz', + gzipDigest: 'sha256:aaa', + gzipSize: 100, + tarDigest: 'sha256:bbb' + ) + def arm64Layer = new ContainerLayer( + location: 'https://fusionfs.seqera.io/releases/pkg/2/4/20/fusion-arm64.tar.gz', + gzipDigest: 'sha256:ccc', + gzipSize: 200, + tarDigest: 'sha256:ddd' + ) + def dataLayer = new ContainerLayer( + location: 'data:H4sIAAAAAAAA/+2STQ6C', + gzipDigest: 'sha256:eee', + gzipSize: 50, + tarDigest: 'sha256:fff' + ) + def config = new ContainerConfig( + ['/usr/bin/fusion'] as List, + null, + ['FUSION_CONFIG_PROFILE=nextflow'] as List, + null, + [dataLayer, amd64Layer, arm64Layer] + ) + + when: 'filtering for amd64' + def amd64Config = MultiPlatformBuildService.filterLayersForPlatform(config, ContainerPlatform.of('linux/amd64')) + then: + amd64Config.layers.size() == 2 + amd64Config.layers[0].location == dataLayer.location + amd64Config.layers[1].location == amd64Layer.location + amd64Config.entrypoint == ['/usr/bin/fusion'] + amd64Config.env == ['FUSION_CONFIG_PROFILE=nextflow'] + + when: 'filtering for arm64' + def arm64Config = MultiPlatformBuildService.filterLayersForPlatform(config, ContainerPlatform.of('linux/arm64')) + then: + arm64Config.layers.size() == 2 + arm64Config.layers[0].location == dataLayer.location + arm64Config.layers[1].location == arm64Layer.location + } + + def 'should return config as-is when no fusion layers'() { + given: + def dataLayer = new ContainerLayer(location: 'data:H4sIAAAAAAAA/+2STQ6C') + def config = new ContainerConfig(null, null, null, null, [dataLayer]) + + when: + def result = MultiPlatformBuildService.filterLayersForPlatform(config, ContainerPlatform.of('linux/amd64')) + + then: + result.layers.size() == 1 + result.layers[0].location == dataLayer.location + } + + def 'should handle null and empty config'() { + expect: + MultiPlatformBuildService.filterLayersForPlatform(null, ContainerPlatform.of('linux/amd64')) == null + MultiPlatformBuildService.filterLayersForPlatform(new ContainerConfig(), ContainerPlatform.of('linux/amd64')).layers == [] + } + def 'launchJob should be a no-op returning job with launch time'() { given: def service = new MultiPlatformBuildService() From 9af8edd92bc0e950d93f27546aa4367aafa2825f Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 6 Mar 2026 11:32:03 +0100 Subject: [PATCH 15/15] [release] bump version 1.33.0-B1 Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d8ab306cc7..133872a354 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.33.0-B0 \ No newline at end of file +1.33.0-B1 \ No newline at end of file