diff --git a/src/spec-configuration/containerCollectionsOCI.ts b/src/spec-configuration/containerCollectionsOCI.ts index 7cd17976a..bff2bbd74 100644 --- a/src/spec-configuration/containerCollectionsOCI.ts +++ b/src/spec-configuration/containerCollectionsOCI.ts @@ -278,7 +278,8 @@ export async function fetchOCIManifestIfExists(params: CommonParams, ref: OCIRef } const manifestUrl = `https://${ref.registry}/v2/${ref.path}/manifests/${reference}`; output.write(`manifest url: ${manifestUrl}`, LogLevel.Trace); - const manifestContainer = await getManifest(params, manifestUrl, ref); + const expectedDigest = manifestDigest || ('digest' in ref ? ref.digest : undefined); + const manifestContainer = await getManifest(params, manifestUrl, ref, undefined, expectedDigest); if (!manifestContainer || !manifestContainer.manifestObj) { return; @@ -294,7 +295,7 @@ export async function fetchOCIManifestIfExists(params: CommonParams, ref: OCIRef return manifestContainer; } -export async function getManifest(params: CommonParams, url: string, ref: OCIRef | OCICollectionRef, mimeType?: string): Promise { +export async function getManifest(params: CommonParams, url: string, ref: OCIRef | OCICollectionRef, mimeType?: string, expectedDigest?: string): Promise { const { output } = params; const res = await getBufferWithMimeType(params, url, ref, mimeType || 'application/vnd.oci.image.manifest.v1+json'); if (!res) { @@ -309,11 +310,17 @@ export async function getManifest(params: CommonParams, url: string, ref: OCIRef // That is useful to have, so if the server doesn't provide it, recalculate it outselves. // Headers are always automatically downcased by node. let contentDigest = headers['docker-content-digest']; - if (!contentDigest) { - output.write('Registry did not send a \'docker-content-digest\' header. Recalculating...', LogLevel.Trace); + if (!contentDigest || expectedDigest) { + if (!contentDigest) { + output.write('Registry did not send a \'docker-content-digest\' header. Recalculating...', LogLevel.Trace); + } contentDigest = `sha256:${crypto.createHash('sha256').update(body).digest('hex')}`; } + if (expectedDigest && contentDigest !== expectedDigest) { + throw new Error(`Digest did not match for ${url}.`); + } + return { contentDigest, manifestObj: JSON.parse(body.toString()), @@ -479,7 +486,7 @@ export async function getPublishedVersions(params: CommonParams, ref: OCIRef, so } } -export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, ignoredFilesDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> { +export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, expectedDigest: string, ignoredFilesDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> { // TODO: Parallelize if multiple layers (not likely). // TODO: Seeking might be needed if the size is too large. @@ -510,6 +517,11 @@ export async function getBlob(params: CommonParams, url: string, ociCacheDir: st return; } + const actualDigest = `sha256:${crypto.createHash('sha256').update(resBody).digest('hex')}`; + if (actualDigest !== expectedDigest) { + throw new Error(`Digest did not match for ${url}.`); + } + await mkdirpLocal(destCachePath); await writeLocalFile(tempTarballPath, resBody); diff --git a/src/spec-configuration/containerFeaturesConfiguration.ts b/src/spec-configuration/containerFeaturesConfiguration.ts index 2d2d1c6e3..c517a58d5 100644 --- a/src/spec-configuration/containerFeaturesConfiguration.ts +++ b/src/spec-configuration/containerFeaturesConfiguration.ts @@ -7,6 +7,8 @@ import * as jsonc from 'jsonc-parser'; import * as path from 'path'; import * as URL from 'url'; import * as tar from 'tar'; +import * as crypto from 'crypto'; + import { DevContainerConfig, DevContainerFeature, VSCodeCustomizations } from './configuration'; import { mkdirpLocal, readLocalFile, rmLocal, writeLocalFile, cpDirectoryLocal, isLocalFile } from '../spec-utils/pfs'; import { Log, LogLevel } from '../spec-utils/log'; @@ -15,6 +17,7 @@ import { computeFeatureInstallationOrder } from './containerFeaturesOrder'; import { fetchOCIFeature, tryGetOCIFeatureSet, fetchOCIFeatureManifestIfExistsFromUserIdentifier } from './containerFeaturesOCI'; import { uriToFsPath } from './configurationCommonUtils'; import { CommonParams, OCIManifest, OCIRef } from './containerCollectionsOCI'; +import { Lockfile, readLockfile, writeLockfile } from './lockfile'; // v1 const V1_ASSET_NAME = 'devcontainer-features.tgz'; @@ -124,6 +127,7 @@ export interface OCISourceInformation extends BaseSourceInformation { type: 'oci'; featureRef: OCIRef; manifest: OCIManifest; + manifestDigest: string; userFeatureIdWithoutVersion: string; } @@ -164,6 +168,7 @@ export interface FeatureSet { features: Feature[]; internalVersion?: string; sourceInformation: SourceInformation; + computedDigest?: string; } export interface FeaturesConfig { @@ -200,6 +205,8 @@ export interface ContainerFeatureInternalParams { env: NodeJS.ProcessEnv; skipFeatureAutoMapping: boolean; platform: NodeJS.Platform; + experimentalLockfile?: boolean; + experimentalFrozenLockfile?: boolean; } export const multiStageBuildExploration = false; @@ -561,7 +568,8 @@ export async function generateFeaturesConfig(params: ContainerFeatureInternalPar // Read features and get the type. output.write('--- Processing User Features ----', LogLevel.Trace); - featuresConfig = await processUserFeatures(params, config, workspaceRoot, userFeatures, featuresConfig); + const lockfile = await readLockfile(params, config); + featuresConfig = await processUserFeatures(params, config, workspaceRoot, userFeatures, featuresConfig, lockfile); output.write(JSON.stringify(featuresConfig, null, 4), LogLevel.Trace); const ociCacheDir = await prepareOCICache(dstFolder); @@ -570,6 +578,8 @@ export async function generateFeaturesConfig(params: ContainerFeatureInternalPar output.write('--- Fetching User Features ----', LogLevel.Trace); await fetchFeatures(params, featuresConfig, locallyCachedFeatureSet, dstFolder, localFeaturesFolder, ociCacheDir); + await writeLockfile(params, config, featuresConfig); + const orderedFeatures = computeFeatureInstallationOrder(config, featuresConfig.featureSets); output.write('--- Computed order ----', LogLevel.Trace); @@ -628,7 +638,7 @@ function featuresToArray(config: DevContainerConfig, additionalFeatures: Record< // Process features contained in devcontainer.json // Creates one feature set per feature to aid in support of the previous structure. -async function processUserFeatures(params: ContainerFeatureInternalParams, config: DevContainerConfig, workspaceRoot: string, userFeatures: DevContainerFeature[], featuresConfig: FeaturesConfig): Promise { +async function processUserFeatures(params: ContainerFeatureInternalParams, config: DevContainerConfig, workspaceRoot: string, userFeatures: DevContainerFeature[], featuresConfig: FeaturesConfig, lockfile: Lockfile | undefined): Promise { const { platform, output } = params; let configPath = config.configFilePath && uriToFsPath(config.configFilePath, platform); @@ -636,7 +646,7 @@ async function processUserFeatures(params: ContainerFeatureInternalParams, confi const updatedUserFeatures = updateDeprecatedFeaturesIntoOptions(userFeatures, output); for (const userFeature of updatedUserFeatures) { - const newFeatureSet = await processFeatureIdentifier(params, configPath, workspaceRoot, userFeature); + const newFeatureSet = await processFeatureIdentifier(params, configPath, workspaceRoot, userFeature, lockfile); if (!newFeatureSet) { throw new Error(`Failed to process feature ${userFeature.id}`); @@ -696,7 +706,7 @@ export function updateDeprecatedFeaturesIntoOptions(userFeatures: DevContainerFe return updatedUserFeatures; } -export async function getFeatureIdType(params: CommonParams, userFeatureId: string) { +export async function getFeatureIdType(params: CommonParams, userFeatureId: string, lockfile: Lockfile | undefined) { const { output } = params; // See the specification for valid feature identifiers: // > https://github.com/devcontainers/spec/blob/main/proposals/devcontainer-features.md#referencing-a-feature @@ -724,7 +734,7 @@ export async function getFeatureIdType(params: CommonParams, userFeatureId: stri return { type: 'file-path', manifest: undefined }; } - const manifest = await fetchOCIFeatureManifestIfExistsFromUserIdentifier(params, userFeatureId); + const manifest = await fetchOCIFeatureManifestIfExistsFromUserIdentifier(params, userFeatureId, lockfile?.features[userFeatureId]?.integrity); if (manifest) { return { type: 'oci', manifest: manifest }; } else { @@ -776,7 +786,7 @@ export function getBackwardCompatibleFeatureId(output: Log, id: string) { // Strictly processes the user provided feature identifier to determine sourceInformation type. // Returns a featureSet per feature. -export async function processFeatureIdentifier(params: CommonParams, configPath: string | undefined, _workspaceRoot: string, userFeature: DevContainerFeature, skipFeatureAutoMapping?: boolean): Promise { +export async function processFeatureIdentifier(params: CommonParams, configPath: string | undefined, _workspaceRoot: string, userFeature: DevContainerFeature, lockfile?: Lockfile, skipFeatureAutoMapping?: boolean): Promise { const { output } = params; output.write(`* Processing feature: ${userFeature.id}`); @@ -788,7 +798,7 @@ export async function processFeatureIdentifier(params: CommonParams, configPath: userFeature.id = getBackwardCompatibleFeatureId(output, userFeature.id); } - const { type, manifest } = await getFeatureIdType(params, userFeature.id); + const { type, manifest } = await getFeatureIdType(params, userFeature.id, lockfile); // cached feature // Resolves deprecated features (fish, maven, gradle, homebrew, jupyterlab) @@ -910,7 +920,7 @@ export async function processFeatureIdentifier(params: CommonParams, configPath: // (6) Oci Identifier if (type === 'oci' && manifest) { - return tryGetOCIFeatureSet(output, userFeature.id, userFeature.options, manifest.manifestObj, originalUserFeatureId); + return tryGetOCIFeatureSet(output, userFeature.id, userFeature.options, manifest, originalUserFeatureId); } output.write(`Github feature.`); @@ -1019,13 +1029,13 @@ async function fetchFeatures(params: { extensionPath: string; cwd: string; outpu if (sourceInfoType === 'oci') { output.write(`Fetching from OCI`, LogLevel.Trace); await mkdirpLocal(featCachePath); - const success = await fetchOCIFeature(params, featureSet, ociCacheDir, featCachePath); - if (!success) { + const res = await fetchOCIFeature(params, featureSet, ociCacheDir, featCachePath); + if (!res) { const err = `Could not download OCI feature: ${featureSet.sourceInformation.featureRef.id}`; throw new Error(err); } - if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath))) { + if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath, featureSet.sourceInformation.manifestDigest))) { const err = `Failed to parse feature '${featureDebugId}'. Please check your devcontainer.json 'features' attribute.`; throw new Error(err); } @@ -1038,7 +1048,7 @@ async function fetchFeatures(params: { extensionPath: string; cwd: string; outpu await mkdirpLocal(featCachePath); await cpDirectoryLocal(localFeaturesFolder, featCachePath); - if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath))) { + if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath, undefined))) { const err = `Failed to parse feature '${featureDebugId}'. Please check your devcontainer.json 'features' attribute.`; throw new Error(err); } @@ -1051,7 +1061,7 @@ async function fetchFeatures(params: { extensionPath: string; cwd: string; outpu const executionPath = featureSet.sourceInformation.resolvedFilePath; await cpDirectoryLocal(executionPath, featCachePath); - if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath))) { + if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath, undefined))) { const err = `Failed to parse feature '${featureDebugId}'. Please check your devcontainer.json 'features' attribute.`; throw new Error(err); } @@ -1089,13 +1099,13 @@ async function fetchFeatures(params: { extensionPath: string; cwd: string; outpu } // Attempt to fetch from 'tarballUris' in order, until one succeeds. - let didSucceed: boolean = false; + let res: { computedDigest: string } | undefined; for (const tarballUri of tarballUris) { - didSucceed = await fetchContentsAtTarballUri(tarballUri, featCachePath, headers, dstFolder, output); + res = await fetchContentsAtTarballUri(tarballUri, featCachePath, headers, dstFolder, output); - if (didSucceed) { + if (res) { output.write(`Succeeded fetching ${tarballUri}`, LogLevel.Trace); - if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath))) { + if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath, res.computedDigest))) { const err = `Failed to parse feature '${featureDebugId}'. Please check your devcontainer.json 'features' attribute.`; throw new Error(err); } @@ -1103,7 +1113,7 @@ async function fetchFeatures(params: { extensionPath: string; cwd: string; outpu } } - if (!didSucceed) { + if (!res) { const msg = `(!) Failed to fetch tarball for ${featureDebugId} after attempting ${tarballUris.length} possibilities.`; throw new Error(msg); } @@ -1115,7 +1125,7 @@ async function fetchFeatures(params: { extensionPath: string; cwd: string; outpu } } -async function fetchContentsAtTarballUri(tarballUri: string, featCachePath: string, headers: { 'user-agent': string; 'Authorization'?: string; 'Accept'?: string }, dstFolder: string, output: Log): Promise { +async function fetchContentsAtTarballUri(tarballUri: string, featCachePath: string, headers: { 'user-agent': string; 'Authorization'?: string; 'Accept'?: string }, dstFolder: string, output: Log): Promise<{ computedDigest: string } | undefined> { const tempTarballPath = path.join(dstFolder, 'temp.tgz'); try { const options = { @@ -1129,9 +1139,11 @@ async function fetchContentsAtTarballUri(tarballUri: string, featCachePath: stri if (!tarball || tarball.length === 0) { output.write(`Did not receive a response from tarball download URI: ${tarballUri}`, LogLevel.Trace); - return false; + return undefined; } + const computedDigest = `sha256:${crypto.createHash('sha256').update(tarball).digest('hex')}`; + // Filter what gets emitted from the tar.extract(). const filter = (file: string, _: tar.FileStat) => { // Don't include .dotfiles or the archive itself. @@ -1155,11 +1167,11 @@ async function fetchContentsAtTarballUri(tarballUri: string, featCachePath: stri await cleanupIterationFetchAndMerge(tempTarballPath, output); - return true; + return { computedDigest }; } catch (e) { output.write(`Caught failure when fetching from URI '${tarballUri}': ${e}`, LogLevel.Trace); await cleanupIterationFetchAndMerge(tempTarballPath, output); - return false; + return undefined; } } @@ -1168,7 +1180,7 @@ async function fetchContentsAtTarballUri(tarballUri: string, featCachePath: stri // Implements the latest ('internalVersion' = '2') parsing logic, // Falls back to earlier implementation(s) if requirements not present. // Returns a boolean indicating whether the feature was successfully parsed. -async function applyFeatureConfigToFeature(output: Log, featureSet: FeatureSet, feature: Feature, featCachePath: string): Promise { +async function applyFeatureConfigToFeature(output: Log, featureSet: FeatureSet, feature: Feature, featCachePath: string, computedDigest: string | undefined): Promise { const innerJsonPath = path.join(featCachePath, DEVCONTAINER_FEATURE_FILE_NAME); if (!(await isLocalFile(innerJsonPath))) { @@ -1178,6 +1190,7 @@ async function applyFeatureConfigToFeature(output: Log, featureSet: FeatureSet, } featureSet.internalVersion = '2'; + featureSet.computedDigest = computedDigest; feature.cachePath = featCachePath; const jsonString: Buffer = await readLocalFile(innerJsonPath); const featureJson = jsonc.parse(jsonString.toString()); diff --git a/src/spec-configuration/containerFeaturesOCI.ts b/src/spec-configuration/containerFeaturesOCI.ts index 57adf5312..f127baa84 100644 --- a/src/spec-configuration/containerFeaturesOCI.ts +++ b/src/spec-configuration/containerFeaturesOCI.ts @@ -1,8 +1,8 @@ import { Log, LogLevel } from '../spec-utils/log'; import { Feature, FeatureSet } from './containerFeaturesConfiguration'; -import { CommonParams, fetchOCIManifestIfExists, getBlob, getRef, ManifestContainer, OCIManifest } from './containerCollectionsOCI'; +import { CommonParams, fetchOCIManifestIfExists, getBlob, getRef, ManifestContainer } from './containerCollectionsOCI'; -export function tryGetOCIFeatureSet(output: Log, identifier: string, options: boolean | string | Record, manifest: OCIManifest, originalUserFeatureId: string): FeatureSet | undefined { +export function tryGetOCIFeatureSet(output: Log, identifier: string, options: boolean | string | Record, manifest: ManifestContainer, originalUserFeatureId: string): FeatureSet | undefined { const featureRef = getRef(output, identifier); if (!featureRef) { output.write(`Unable to parse '${identifier}'`, LogLevel.Error); @@ -19,7 +19,8 @@ export function tryGetOCIFeatureSet(output: Log, identifier: string, options: bo let featureSet: FeatureSet = { sourceInformation: { type: 'oci', - manifest: manifest, + manifest: manifest.manifestObj, + manifestDigest: manifest.contentDigest, featureRef: featureRef, userFeatureId: originalUserFeatureId, userFeatureIdWithoutVersion @@ -43,7 +44,7 @@ export async function fetchOCIFeatureManifestIfExistsFromUserIdentifier(params: // Download a feature from which a manifest was previously downloaded. // Specification: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-blobs -export async function fetchOCIFeature(params: CommonParams, featureSet: FeatureSet, ociCacheDir: string, featCachePath: string): Promise { +export async function fetchOCIFeature(params: CommonParams, featureSet: FeatureSet, ociCacheDir: string, featCachePath: string) { const { output } = params; if (featureSet.sourceInformation.type !== 'oci') { @@ -53,14 +54,15 @@ export async function fetchOCIFeature(params: CommonParams, featureSet: FeatureS const { featureRef } = featureSet.sourceInformation; - const blobUrl = `https://${featureSet.sourceInformation.featureRef.registry}/v2/${featureSet.sourceInformation.featureRef.path}/blobs/${featureSet.sourceInformation.manifest?.layers[0].digest}`; + const expectedDigest = featureSet.sourceInformation.manifest?.layers[0].digest; + const blobUrl = `https://${featureSet.sourceInformation.featureRef.registry}/v2/${featureSet.sourceInformation.featureRef.path}/blobs/${expectedDigest}`; output.write(`blob url: ${blobUrl}`, LogLevel.Trace); - const blobResult = await getBlob(params, blobUrl, ociCacheDir, featCachePath, featureRef); + const blobResult = await getBlob(params, blobUrl, ociCacheDir, featCachePath, featureRef, expectedDigest); if (!blobResult) { throw new Error(`Failed to download package for ${featureSet.sourceInformation.featureRef.resource}`); } - return true; + return blobResult; } diff --git a/src/spec-configuration/containerTemplatesOCI.ts b/src/spec-configuration/containerTemplatesOCI.ts index 1d4f99ee1..acd40a702 100644 --- a/src/spec-configuration/containerTemplatesOCI.ts +++ b/src/spec-configuration/containerTemplatesOCI.ts @@ -46,7 +46,7 @@ export async function fetchTemplate(params: CommonParams, selectedTemplate: Sele output.write(`blob url: ${blobUrl}`, LogLevel.Trace); const tmpDir = userProvidedTmpDir || path.join(os.tmpdir(), 'vsch-template-temp', `${Date.now()}`); - const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, ['devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json'); + const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, blobDigest, ['devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json'); if (!blobResult) { throw new Error(`Failed to download package for ${templateRef.resource}`); diff --git a/src/spec-configuration/lockfile.ts b/src/spec-configuration/lockfile.ts new file mode 100644 index 000000000..10e366c12 --- /dev/null +++ b/src/spec-configuration/lockfile.ts @@ -0,0 +1,78 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as path from 'path'; +import { DevContainerConfig } from './configuration'; +import { readLocalFile, writeLocalFile } from '../spec-utils/pfs'; +import { ContainerFeatureInternalParams, FeatureSet, FeaturesConfig, OCISourceInformation } from './containerFeaturesConfiguration'; + + +export interface Lockfile { + features: Record; +} + +export async function writeLockfile(params: ContainerFeatureInternalParams, config: DevContainerConfig, featuresConfig: FeaturesConfig) { + if (!params.experimentalLockfile && !params.experimentalFrozenLockfile) { + return; + } + + const lockfile: Lockfile = featuresConfig.featureSets + .map(f => [f, f.sourceInformation] as const) + .filter((tup): tup is [FeatureSet, OCISourceInformation] => tup[1].type === 'oci') + .map(([set, source]) => ({ + id: source.userFeatureId, + version: set.features[0].version!, + resolved: `${source.featureRef.registry}/${source.featureRef.path}@${set.computedDigest}`, + integrity: set.computedDigest!, + })) + .sort((a, b) => a.id.localeCompare(b.id)) + .reduce((acc, cur) => { + const feature = { ...cur }; + delete (feature as any).id; + acc.features[cur.id] = feature; + return acc; + }, { + features: {} as Record, + }); + + const lockfilePath = getLockfilePath(config); + const newLockfileContent = Buffer.from(JSON.stringify(lockfile, null, 2)); + const oldLockfileContent = await readLocalFile(lockfilePath) + .catch(err => { + if (err?.code !== 'ENOENT') { + throw err; + } + }); + if (params.experimentalFrozenLockfile && !oldLockfileContent) { + throw new Error('Lockfile does not exist.'); + } + if (!oldLockfileContent || !newLockfileContent.equals(oldLockfileContent)) { + if (params.experimentalFrozenLockfile) { + throw new Error('Lockfile does not match.'); + } + await writeLocalFile(lockfilePath, newLockfileContent); + } +} + +export async function readLockfile(params: ContainerFeatureInternalParams, config: DevContainerConfig): Promise { + if (!params.experimentalLockfile && !params.experimentalFrozenLockfile) { + return undefined; + } + + try { + const content = await readLocalFile(getLockfilePath(config)); + return JSON.parse(content.toString()) as Lockfile; + } catch (err) { + if (err?.code === 'ENOENT') { + return undefined; + } + throw err; + } +} + +export function getLockfilePath(configOrPath: DevContainerConfig | string) { + const configPath = typeof configOrPath === 'string' ? configOrPath : configOrPath.configFilePath!.fsPath; + return path.join(path.dirname(configPath), path.basename(configPath).startsWith('.') ? '.devcontainer-lock.json' : 'devcontainer-lock.json'); +} diff --git a/src/spec-node/containerFeatures.ts b/src/spec-node/containerFeatures.ts index 02d78db1d..111834f64 100644 --- a/src/spec-node/containerFeatures.ts +++ b/src/spec-node/containerFeatures.ts @@ -126,7 +126,8 @@ export async function getExtendImageBuildInfo(params: DockerResolverParameters, // Processes the user's configuration. const platform = params.common.cliHost.platform; - const featuresConfig = await generateFeaturesConfig({ ...params.common, platform }, dstFolder, config.config, getContainerFeaturesFolder, additionalFeatures); + const { experimentalLockfile, experimentalFrozenLockfile } = params; + const featuresConfig = await generateFeaturesConfig({ ...params.common, platform, experimentalLockfile, experimentalFrozenLockfile }, dstFolder, config.config, getContainerFeaturesFolder, additionalFeatures); if (!featuresConfig) { if (canAddLabelsToContainer && !imageBuildInfo.dockerfile) { return { diff --git a/src/spec-node/devContainers.ts b/src/spec-node/devContainers.ts index cff48346e..9253be7e7 100644 --- a/src/spec-node/devContainers.ts +++ b/src/spec-node/devContainers.ts @@ -63,6 +63,8 @@ export interface ProvisionOptions { installCommand?: string; targetPath?: string; }; + experimentalLockfile?: boolean; + experimentalFrozenLockfile?: boolean; } export async function launch(options: ProvisionOptions, providedIdLabels: string[] | undefined, disposables: (() => Promise | undefined)[]) { @@ -90,7 +92,7 @@ export async function launch(options: ProvisionOptions, providedIdLabels: string } export async function createDockerParams(options: ProvisionOptions, disposables: (() => Promise | undefined)[]): Promise { - const { persistedFolder, additionalMounts, updateRemoteUserUIDDefault, containerDataFolder, containerSystemDataFolder, workspaceMountConsistency, mountWorkspaceGitRoot, remoteEnv } = options; + const { persistedFolder, additionalMounts, updateRemoteUserUIDDefault, containerDataFolder, containerSystemDataFolder, workspaceMountConsistency, mountWorkspaceGitRoot, remoteEnv, experimentalLockfile, experimentalFrozenLockfile } = options; let parsedAuthority: DevContainerAuthority | undefined; if (options.workspaceFolder) { parsedAuthority = { hostPath: options.workspaceFolder } as DevContainerAuthority; @@ -179,6 +181,8 @@ export async function createDockerParams(options: ProvisionOptions, disposables: additionalCacheFroms: options.additionalCacheFroms, buildKitVersion, isTTY: process.stdout.isTTY || options.logFormat === 'json', + experimentalLockfile, + experimentalFrozenLockfile, buildxPlatform: common.buildxPlatform, buildxPush: common.buildxPush, buildxOutput: common.buildxOutput, diff --git a/src/spec-node/devContainersSpecCLI.ts b/src/spec-node/devContainersSpecCLI.ts index b76cee8f9..e29ea7163 100644 --- a/src/spec-node/devContainersSpecCLI.ts +++ b/src/spec-node/devContainersSpecCLI.ts @@ -118,6 +118,8 @@ function provisionOptions(y: Argv) { 'dotfiles-target-path': { type: 'string', default: '~/dotfiles', description: 'The path to clone the dotfiles repository to. Defaults to `~/dotfiles`.' }, 'container-session-data-folder': { type: 'string', description: 'Folder to cache CLI data, for example userEnvProbe results' }, 'omit-config-remote-env-from-metadata': { type: 'boolean', default: false, hidden: true, description: 'Omit remoteEnv from devcontainer.json for container metadata label' }, + 'experimental-lockfile': { type: 'boolean', default: false, hidden: true, description: 'Write lockfile' }, + 'experimental-frozen-lockfile': { type: 'boolean', default: false, hidden: true, description: 'Ensure lockfile remains unchanged' }, }) .check(argv => { const idLabels = (argv['id-label'] && (Array.isArray(argv['id-label']) ? argv['id-label'] : [argv['id-label']])) as string[] | undefined; @@ -184,6 +186,8 @@ async function provision({ 'dotfiles-target-path': dotfilesTargetPath, 'container-session-data-folder': containerSessionDataFolder, 'omit-config-remote-env-from-metadata': omitConfigRemotEnvFromMetadata, + 'experimental-lockfile': experimentalLockfile, + 'experimental-frozen-lockfile': experimentalFrozenLockfile, }: ProvisionArgs) { const workspaceFolder = workspaceFolderArg ? path.resolve(process.cwd(), workspaceFolderArg) : undefined; @@ -240,6 +244,8 @@ async function provision({ containerSessionDataFolder, skipPersistingCustomizationsFromFeatures: false, omitConfigRemotEnvFromMetadata: omitConfigRemotEnvFromMetadata, + experimentalLockfile, + experimentalFrozenLockfile, }; const result = await doProvision(options, providedIdLabels); @@ -461,6 +467,8 @@ function buildOptions(y: Argv) { 'additional-features': { type: 'string', description: 'Additional features to apply to the dev container (JSON as per "features" section in devcontainer.json)' }, 'skip-feature-auto-mapping': { type: 'boolean', default: false, hidden: true, description: 'Temporary option for testing.' }, 'skip-persisting-customizations-from-features': { type: 'boolean', default: false, hidden: true, description: 'Do not save customizations from referenced Features as image metadata' }, + 'experimental-lockfile': { type: 'boolean', default: false, hidden: true, description: 'Write lockfile' }, + 'experimental-frozen-lockfile': { type: 'boolean', default: false, hidden: true, description: 'Ensure lockfile remains unchanged' }, }); } @@ -496,6 +504,8 @@ async function doBuild({ 'additional-features': additionalFeaturesJson, 'skip-feature-auto-mapping': skipFeatureAutoMapping, 'skip-persisting-customizations-from-features': skipPersistingCustomizationsFromFeatures, + 'experimental-lockfile': experimentalLockfile, + 'experimental-frozen-lockfile': experimentalFrozenLockfile, }: BuildArgs) { const disposables: (() => Promise | undefined)[] = []; const dispose = async () => { @@ -539,7 +549,9 @@ async function doBuild({ skipFeatureAutoMapping, skipPostAttach: true, skipPersistingCustomizationsFromFeatures: skipPersistingCustomizationsFromFeatures, - dotfiles: {} + dotfiles: {}, + experimentalLockfile, + experimentalFrozenLockfile, }, disposables); const { common, dockerCLI, dockerComposeCLI } = params; diff --git a/src/spec-node/utils.ts b/src/spec-node/utils.ts index e9e148fc5..0302d8ee0 100644 --- a/src/spec-node/utils.ts +++ b/src/spec-node/utils.ts @@ -112,6 +112,8 @@ export interface DockerResolverParameters { additionalCacheFroms: string[]; buildKitVersion: string | null; isTTY: boolean; + experimentalLockfile?: boolean; + experimentalFrozenLockfile?: boolean; buildxPlatform: string | undefined; buildxPush: boolean; buildxOutput: string | undefined; diff --git a/src/test/container-features/configs/lockfile-frozen/.devcontainer-lock.json b/src/test/container-features/configs/lockfile-frozen/.devcontainer-lock.json new file mode 100644 index 000000000..4d9bc604e --- /dev/null +++ b/src/test/container-features/configs/lockfile-frozen/.devcontainer-lock.json @@ -0,0 +1,14 @@ +{ + "features": { + "ghcr.io/codspace/features/color:1": { + "version": "1.0.4", + "resolved": "ghcr.io/codspace/features/color@sha256:6e9d07c7f488fabc981e7508d8c25eea4b102ebe4b87f9fc6f233efa1d325908", + "integrity": "sha256:6e9d07c7f488fabc981e7508d8c25eea4b102ebe4b87f9fc6f233efa1d325908" + }, + "ghcr.io/codspace/features/flower:1": { + "version": "1.0.0", + "resolved": "ghcr.io/codspace/features/flower@sha256:c9cc1ac636b9ef595512b5ca7ecb3a35b7d3499cb6f86372edec76ae0cd71d43", + "integrity": "sha256:c9cc1ac636b9ef595512b5ca7ecb3a35b7d3499cb6f86372edec76ae0cd71d43" + } + } +} \ No newline at end of file diff --git a/src/test/container-features/configs/lockfile-frozen/.devcontainer.json b/src/test/container-features/configs/lockfile-frozen/.devcontainer.json new file mode 100644 index 000000000..d98d20705 --- /dev/null +++ b/src/test/container-features/configs/lockfile-frozen/.devcontainer.json @@ -0,0 +1,7 @@ +{ + "image": "mcr.microsoft.com/devcontainers/base:ubuntu", + "features": { + "ghcr.io/codspace/features/flower:1": {}, + "ghcr.io/codspace/features/color:1": {} + } +} diff --git a/src/test/container-features/configs/lockfile-outdated/.devcontainer.json b/src/test/container-features/configs/lockfile-outdated/.devcontainer.json new file mode 100644 index 000000000..caa30b47e --- /dev/null +++ b/src/test/container-features/configs/lockfile-outdated/.devcontainer.json @@ -0,0 +1,7 @@ +{ + "image": "mcr.microsoft.com/devcontainers/base:ubuntu", + "features": { + "ghcr.io/codspace/features/flower:1.0.0": {}, + "ghcr.io/codspace/features/color:1.0.5": {} + } +} diff --git a/src/test/container-features/configs/lockfile-outdated/.gitignore b/src/test/container-features/configs/lockfile-outdated/.gitignore new file mode 100644 index 000000000..743e650e4 --- /dev/null +++ b/src/test/container-features/configs/lockfile-outdated/.gitignore @@ -0,0 +1 @@ +.devcontainer-lock.json \ No newline at end of file diff --git a/src/test/container-features/configs/lockfile-outdated/expected.devcontainer-lock.json b/src/test/container-features/configs/lockfile-outdated/expected.devcontainer-lock.json new file mode 100644 index 000000000..d4491d4dc --- /dev/null +++ b/src/test/container-features/configs/lockfile-outdated/expected.devcontainer-lock.json @@ -0,0 +1,14 @@ +{ + "features": { + "ghcr.io/codspace/features/color:1.0.5": { + "version": "1.0.5", + "resolved": "ghcr.io/codspace/features/color@sha256:a20e55d6a70ad4bfb9eca962693de0982c607f41d732c3ff57ed1d75e9b59f9c", + "integrity": "sha256:a20e55d6a70ad4bfb9eca962693de0982c607f41d732c3ff57ed1d75e9b59f9c" + }, + "ghcr.io/codspace/features/flower:1.0.0": { + "version": "1.0.0", + "resolved": "ghcr.io/codspace/features/flower@sha256:c9cc1ac636b9ef595512b5ca7ecb3a35b7d3499cb6f86372edec76ae0cd71d43", + "integrity": "sha256:c9cc1ac636b9ef595512b5ca7ecb3a35b7d3499cb6f86372edec76ae0cd71d43" + } + } +} \ No newline at end of file diff --git a/src/test/container-features/configs/lockfile-outdated/original.devcontainer-lock.json b/src/test/container-features/configs/lockfile-outdated/original.devcontainer-lock.json new file mode 100644 index 000000000..d23747a9e --- /dev/null +++ b/src/test/container-features/configs/lockfile-outdated/original.devcontainer-lock.json @@ -0,0 +1,14 @@ +{ + "features": { + "ghcr.io/codspace/features/color:1.0.4": { + "version": "1.0.4", + "resolved": "ghcr.io/codspace/features/color@sha256:6e9d07c7f488fabc981e7508d8c25eea4b102ebe4b87f9fc6f233efa1d325908", + "integrity": "sha256:6e9d07c7f488fabc981e7508d8c25eea4b102ebe4b87f9fc6f233efa1d325908" + }, + "ghcr.io/codspace/features/flower:1.0.0": { + "version": "1.0.0", + "resolved": "ghcr.io/codspace/features/flower@sha256:c9cc1ac636b9ef595512b5ca7ecb3a35b7d3499cb6f86372edec76ae0cd71d43", + "integrity": "sha256:c9cc1ac636b9ef595512b5ca7ecb3a35b7d3499cb6f86372edec76ae0cd71d43" + } + } +} \ No newline at end of file diff --git a/src/test/container-features/configs/lockfile/.devcontainer.json b/src/test/container-features/configs/lockfile/.devcontainer.json new file mode 100644 index 000000000..e54a9012f --- /dev/null +++ b/src/test/container-features/configs/lockfile/.devcontainer.json @@ -0,0 +1,7 @@ +{ + "image": "mcr.microsoft.com/devcontainers/base:ubuntu", + "features": { + "ghcr.io/codspace/features/flower:1.0.0": {}, + "ghcr.io/codspace/features/color:1.0.4": {} + } +} diff --git a/src/test/container-features/configs/lockfile/.gitignore b/src/test/container-features/configs/lockfile/.gitignore new file mode 100644 index 000000000..743e650e4 --- /dev/null +++ b/src/test/container-features/configs/lockfile/.gitignore @@ -0,0 +1 @@ +.devcontainer-lock.json \ No newline at end of file diff --git a/src/test/container-features/configs/lockfile/expected.devcontainer-lock.json b/src/test/container-features/configs/lockfile/expected.devcontainer-lock.json new file mode 100644 index 000000000..d23747a9e --- /dev/null +++ b/src/test/container-features/configs/lockfile/expected.devcontainer-lock.json @@ -0,0 +1,14 @@ +{ + "features": { + "ghcr.io/codspace/features/color:1.0.4": { + "version": "1.0.4", + "resolved": "ghcr.io/codspace/features/color@sha256:6e9d07c7f488fabc981e7508d8c25eea4b102ebe4b87f9fc6f233efa1d325908", + "integrity": "sha256:6e9d07c7f488fabc981e7508d8c25eea4b102ebe4b87f9fc6f233efa1d325908" + }, + "ghcr.io/codspace/features/flower:1.0.0": { + "version": "1.0.0", + "resolved": "ghcr.io/codspace/features/flower@sha256:c9cc1ac636b9ef595512b5ca7ecb3a35b7d3499cb6f86372edec76ae0cd71d43", + "integrity": "sha256:c9cc1ac636b9ef595512b5ca7ecb3a35b7d3499cb6f86372edec76ae0cd71d43" + } + } +} \ No newline at end of file diff --git a/src/test/container-features/containerFeaturesOCI.test.ts b/src/test/container-features/containerFeaturesOCI.test.ts index c6493e00a..d10149e9f 100644 --- a/src/test/container-features/containerFeaturesOCI.test.ts +++ b/src/test/container-features/containerFeaturesOCI.test.ts @@ -251,7 +251,7 @@ describe('Test OCI Pull', async function () { if (!featureRef) { assert.fail('featureRef should not be undefined'); } - const blobResult = await getBlob({ output, env: process.env }, 'https://ghcr.io/v2/codspace/features/ruby/blobs/sha256:8f59630bd1ba6d9e78b485233a0280530b3d0a44338f472206090412ffbd3efb', '/tmp', '/tmp/featureTest', featureRef); + const blobResult = await getBlob({ output, env: process.env }, 'https://ghcr.io/v2/codspace/features/ruby/blobs/sha256:8f59630bd1ba6d9e78b485233a0280530b3d0a44338f472206090412ffbd3efb', '/tmp', '/tmp/featureTest', featureRef, 'sha256:8f59630bd1ba6d9e78b485233a0280530b3d0a44338f472206090412ffbd3efb'); assert.isDefined(blobResult); assert.isArray(blobResult?.files); }); diff --git a/src/test/container-features/containerFeaturesOrder.test.ts b/src/test/container-features/containerFeaturesOrder.test.ts index 782cd904d..fa2d3d949 100644 --- a/src/test/container-features/containerFeaturesOrder.test.ts +++ b/src/test/container-features/containerFeaturesOrder.test.ts @@ -165,6 +165,7 @@ describe('Container features install order', function () { }, layers: [] }, + manifestDigest: 'test', userFeatureId: id, userFeatureIdWithoutVersion: splitOnColon[0] }, diff --git a/src/test/container-features/e2e.test.ts b/src/test/container-features/e2e.test.ts index fe8061347..0949f1387 100644 --- a/src/test/container-features/e2e.test.ts +++ b/src/test/container-features/e2e.test.ts @@ -11,7 +11,7 @@ import { devContainerDown, devContainerUp, shellExec } from '../testUtils'; const pkg = require('../../../package.json'); describe('Dev Container Features E2E (remote)', function () { - this.timeout('240s'); + this.timeout('300s'); const tmp = path.relative(process.cwd(), path.join(__dirname, 'tmp')); const cli = `npx --prefix ${tmp} devcontainer`; @@ -106,7 +106,7 @@ describe('Dev Container Features E2E (remote)', function () { }); describe('Dev Container Features E2E - local cache/short-hand notation', function () { - this.timeout('240s'); + this.timeout('300s'); const tmp = path.resolve(process.cwd(), path.join(__dirname, 'tmp3')); const cli = `npx --prefix ${tmp} devcontainer`; diff --git a/src/test/container-features/featureHelpers.test.ts b/src/test/container-features/featureHelpers.test.ts index 167859805..194a52fe7 100644 --- a/src/test/container-features/featureHelpers.test.ts +++ b/src/test/container-features/featureHelpers.test.ts @@ -634,7 +634,8 @@ chmod +x ./install.sh size: 0, }, layers: [], - } + }, + manifestDigest: '', } }; const options = [ @@ -717,7 +718,8 @@ chmod +x ./install.sh size: 0, }, layers: [], - } + }, + manifestDigest: '', } }; const options = [ @@ -803,7 +805,8 @@ chmod +x ./install.sh size: 0, }, layers: [], - } + }, + manifestDigest: '', } }; const options = [ diff --git a/src/test/container-features/lockfile.test.ts b/src/test/container-features/lockfile.test.ts new file mode 100644 index 000000000..cf07af703 --- /dev/null +++ b/src/test/container-features/lockfile.test.ts @@ -0,0 +1,74 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import * as path from 'path'; +import { shellExec } from '../testUtils'; +import { cpLocal, readLocalFile, rmLocal } from '../../spec-utils/pfs'; + +const pkg = require('../../../package.json'); + +describe('Lockfile', function () { + this.timeout('240s'); + + const tmp = path.relative(process.cwd(), path.join(__dirname, 'tmp')); + const cli = `npx --prefix ${tmp} devcontainer`; + + before('Install', async () => { + await shellExec(`rm -rf ${tmp}/node_modules`); + await shellExec(`mkdir -p ${tmp}`); + await shellExec(`npm --prefix ${tmp} install devcontainers-cli-${pkg.version}.tgz`); + }); + + it('write lockfile', async () => { + const workspaceFolder = path.join(__dirname, 'configs/lockfile'); + + const lockfilePath = path.join(workspaceFolder, '.devcontainer-lock.json'); + await rmLocal(lockfilePath, { force: true }); + + const res = await shellExec(`${cli} build --workspace-folder ${workspaceFolder} --experimental-lockfile`); + const response = JSON.parse(res.stdout); + assert.equal(response.outcome, 'success'); + const actual = await readLocalFile(lockfilePath); + const expected = await readLocalFile(path.join(workspaceFolder, 'expected.devcontainer-lock.json')); + assert.equal(actual.toString(), expected.toString()); + }); + + it('frozen lockfile', async () => { + const workspaceFolder = path.join(__dirname, 'configs/lockfile-frozen'); + const lockfilePath = path.join(workspaceFolder, '.devcontainer-lock.json'); + const expected = await readLocalFile(lockfilePath); + const res = await shellExec(`${cli} build --workspace-folder ${workspaceFolder} --experimental-lockfile --experimental-frozen-lockfile`); + const response = JSON.parse(res.stdout); + assert.equal(response.outcome, 'success'); + const actual = await readLocalFile(lockfilePath); + assert.equal(actual.toString(), expected.toString()); + }); + + it('outdated lockfile', async () => { + const workspaceFolder = path.join(__dirname, 'configs/lockfile-outdated'); + + const lockfilePath = path.join(workspaceFolder, '.devcontainer-lock.json'); + await cpLocal(path.join(workspaceFolder, 'original.devcontainer-lock.json'), lockfilePath); + + { + try { + throw await shellExec(`${cli} build --workspace-folder ${workspaceFolder} --experimental-lockfile --experimental-frozen-lockfile`); + } catch (res) { + const response = JSON.parse(res.stdout); + assert.equal(response.outcome, 'error'); + } + } + + { + const res = await shellExec(`${cli} build --workspace-folder ${workspaceFolder} --experimental-lockfile`); + const response = JSON.parse(res.stdout); + assert.equal(response.outcome, 'success'); + const actual = await readLocalFile(lockfilePath); + const expected = await readLocalFile(path.join(workspaceFolder, 'expected.devcontainer-lock.json')); + assert.equal(actual.toString(), expected.toString()); + } + }); +}); \ No newline at end of file diff --git a/src/test/imageMetadata.test.ts b/src/test/imageMetadata.test.ts index 4dd56be4b..ae483eebb 100644 --- a/src/test/imageMetadata.test.ts +++ b/src/test/imageMetadata.test.ts @@ -551,7 +551,8 @@ function getFeaturesConfig(features: Feature[]): FeaturesConfig { size: 0, }, layers: [], - } + }, + manifestDigest: '', } })) };