Skip to content

Experimental lockfile #495

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chrmarti merged 1 commit into from
Apr 21, 2023
Merged

Experimental lockfile #495

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 17 additions & 5 deletions src/spec-configuration/containerCollectionsOCI.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,8 @@ export async function fetchOCIManifestIfExists(params: CommonParams, ref: OCIRef
}
const manifestUrl = `https://${ref.registry}/v2/${ref.path}/manifests/${reference}`;
output.write(`manifest url: ${manifestUrl}`, LogLevel.Trace);
const manifestContainer = await getManifest(params, manifestUrl, ref);
const expectedDigest = manifestDigest || ('digest' in ref ? ref.digest : undefined);
const manifestContainer = await getManifest(params, manifestUrl, ref, undefined, expectedDigest);

if (!manifestContainer || !manifestContainer.manifestObj) {
return;
Expand All @@ -294,7 +295,7 @@ export async function fetchOCIManifestIfExists(params: CommonParams, ref: OCIRef
return manifestContainer;
}

export async function getManifest(params: CommonParams, url: string, ref: OCIRef | OCICollectionRef, mimeType?: string): Promise<ManifestContainer | undefined> {
export async function getManifest(params: CommonParams, url: string, ref: OCIRef | OCICollectionRef, mimeType?: string, expectedDigest?: string): Promise<ManifestContainer | undefined> {
const { output } = params;
const res = await getBufferWithMimeType(params, url, ref, mimeType || 'application/vnd.oci.image.manifest.v1+json');
if (!res) {
Expand All @@ -309,11 +310,17 @@ export async function getManifest(params: CommonParams, url: string, ref: OCIRef
// That is useful to have, so if the server doesn't provide it, recalculate it outselves.
// Headers are always automatically downcased by node.
let contentDigest = headers['docker-content-digest'];
if (!contentDigest) {
output.write('Registry did not send a \'docker-content-digest\' header. Recalculating...', LogLevel.Trace);
if (!contentDigest || expectedDigest) {
if (!contentDigest) {
output.write('Registry did not send a \'docker-content-digest\' header. Recalculating...', LogLevel.Trace);
}
contentDigest = `sha256:${crypto.createHash('sha256').update(body).digest('hex')}`;
}

if (expectedDigest && contentDigest !== expectedDigest) {
throw new Error(`Digest did not match for ${url}.`);
}

return {
contentDigest,
manifestObj: JSON.parse(body.toString()),
Expand Down Expand Up @@ -479,7 +486,7 @@ export async function getPublishedVersions(params: CommonParams, ref: OCIRef, so
}
}

export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, ignoredFilesDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> {
export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, expectedDigest: string, ignoredFilesDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> {
// TODO: Parallelize if multiple layers (not likely).
// TODO: Seeking might be needed if the size is too large.

Expand Down Expand Up @@ -510,6 +517,11 @@ export async function getBlob(params: CommonParams, url: string, ociCacheDir: st
return;
}

const actualDigest = `sha256:${crypto.createHash('sha256').update(resBody).digest('hex')}`;
if (actualDigest !== expectedDigest) {
throw new Error(`Digest did not match for ${url}.`);
}

await mkdirpLocal(destCachePath);
await writeLocalFile(tempTarballPath, resBody);

Expand Down
59 changes: 36 additions & 23 deletions src/spec-configuration/containerFeaturesConfiguration.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import * as jsonc from 'jsonc-parser';
import * as path from 'path';
import * as URL from 'url';
import * as tar from 'tar';
import * as crypto from 'crypto';

import { DevContainerConfig, DevContainerFeature, VSCodeCustomizations } from './configuration';
import { mkdirpLocal, readLocalFile, rmLocal, writeLocalFile, cpDirectoryLocal, isLocalFile } from '../spec-utils/pfs';
import { Log, LogLevel } from '../spec-utils/log';
Expand All @@ -15,6 +17,7 @@ import { computeFeatureInstallationOrder } from './containerFeaturesOrder';
import { fetchOCIFeature, tryGetOCIFeatureSet, fetchOCIFeatureManifestIfExistsFromUserIdentifier } from './containerFeaturesOCI';
import { uriToFsPath } from './configurationCommonUtils';
import { CommonParams, OCIManifest, OCIRef } from './containerCollectionsOCI';
import { Lockfile, readLockfile, writeLockfile } from './lockfile';

// v1
const V1_ASSET_NAME = 'devcontainer-features.tgz';
Expand Down Expand Up @@ -124,6 +127,7 @@ export interface OCISourceInformation extends BaseSourceInformation {
type: 'oci';
featureRef: OCIRef;
manifest: OCIManifest;
manifestDigest: string;
userFeatureIdWithoutVersion: string;
}

Expand Down Expand Up @@ -164,6 +168,7 @@ export interface FeatureSet {
features: Feature[];
internalVersion?: string;
sourceInformation: SourceInformation;
computedDigest?: string;
}

export interface FeaturesConfig {
Expand Down Expand Up @@ -200,6 +205,8 @@ export interface ContainerFeatureInternalParams {
env: NodeJS.ProcessEnv;
skipFeatureAutoMapping: boolean;
platform: NodeJS.Platform;
experimentalLockfile?: boolean;
experimentalFrozenLockfile?: boolean;
}

export const multiStageBuildExploration = false;
Expand Down Expand Up @@ -561,7 +568,8 @@ export async function generateFeaturesConfig(params: ContainerFeatureInternalPar

// Read features and get the type.
output.write('--- Processing User Features ----', LogLevel.Trace);
featuresConfig = await processUserFeatures(params, config, workspaceRoot, userFeatures, featuresConfig);
const lockfile = await readLockfile(params, config);
featuresConfig = await processUserFeatures(params, config, workspaceRoot, userFeatures, featuresConfig, lockfile);
output.write(JSON.stringify(featuresConfig, null, 4), LogLevel.Trace);

const ociCacheDir = await prepareOCICache(dstFolder);
Expand All @@ -570,6 +578,8 @@ export async function generateFeaturesConfig(params: ContainerFeatureInternalPar
output.write('--- Fetching User Features ----', LogLevel.Trace);
await fetchFeatures(params, featuresConfig, locallyCachedFeatureSet, dstFolder, localFeaturesFolder, ociCacheDir);

await writeLockfile(params, config, featuresConfig);

const orderedFeatures = computeFeatureInstallationOrder(config, featuresConfig.featureSets);

output.write('--- Computed order ----', LogLevel.Trace);
Expand Down Expand Up @@ -628,15 +638,15 @@ function featuresToArray(config: DevContainerConfig, additionalFeatures: Record<

// Process features contained in devcontainer.json
// Creates one feature set per feature to aid in support of the previous structure.
async function processUserFeatures(params: ContainerFeatureInternalParams, config: DevContainerConfig, workspaceRoot: string, userFeatures: DevContainerFeature[], featuresConfig: FeaturesConfig): Promise<FeaturesConfig> {
async function processUserFeatures(params: ContainerFeatureInternalParams, config: DevContainerConfig, workspaceRoot: string, userFeatures: DevContainerFeature[], featuresConfig: FeaturesConfig, lockfile: Lockfile | undefined): Promise<FeaturesConfig> {
const { platform, output } = params;

let configPath = config.configFilePath && uriToFsPath(config.configFilePath, platform);
output.write(`configPath: ${configPath}`, LogLevel.Trace);

const updatedUserFeatures = updateDeprecatedFeaturesIntoOptions(userFeatures, output);
for (const userFeature of updatedUserFeatures) {
const newFeatureSet = await processFeatureIdentifier(params, configPath, workspaceRoot, userFeature);
const newFeatureSet = await processFeatureIdentifier(params, configPath, workspaceRoot, userFeature, lockfile);

if (!newFeatureSet) {
throw new Error(`Failed to process feature ${userFeature.id}`);
Expand Down Expand Up @@ -696,7 +706,7 @@ export function updateDeprecatedFeaturesIntoOptions(userFeatures: DevContainerFe
return updatedUserFeatures;
}

export async function getFeatureIdType(params: CommonParams, userFeatureId: string) {
export async function getFeatureIdType(params: CommonParams, userFeatureId: string, lockfile: Lockfile | undefined) {
const { output } = params;
// See the specification for valid feature identifiers:
// > https://github.com/devcontainers/spec/blob/main/proposals/devcontainer-features.md#referencing-a-feature
Expand Down Expand Up @@ -724,7 +734,7 @@ export async function getFeatureIdType(params: CommonParams, userFeatureId: stri
return { type: 'file-path', manifest: undefined };
}

const manifest = await fetchOCIFeatureManifestIfExistsFromUserIdentifier(params, userFeatureId);
const manifest = await fetchOCIFeatureManifestIfExistsFromUserIdentifier(params, userFeatureId, lockfile?.features[userFeatureId]?.integrity);
if (manifest) {
return { type: 'oci', manifest: manifest };
} else {
Expand Down Expand Up @@ -776,7 +786,7 @@ export function getBackwardCompatibleFeatureId(output: Log, id: string) {

// Strictly processes the user provided feature identifier to determine sourceInformation type.
// Returns a featureSet per feature.
export async function processFeatureIdentifier(params: CommonParams, configPath: string | undefined, _workspaceRoot: string, userFeature: DevContainerFeature, skipFeatureAutoMapping?: boolean): Promise<FeatureSet | undefined> {
export async function processFeatureIdentifier(params: CommonParams, configPath: string | undefined, _workspaceRoot: string, userFeature: DevContainerFeature, lockfile?: Lockfile, skipFeatureAutoMapping?: boolean): Promise<FeatureSet | undefined> {
const { output } = params;

output.write(`* Processing feature: ${userFeature.id}`);
Expand All @@ -788,7 +798,7 @@ export async function processFeatureIdentifier(params: CommonParams, configPath:
userFeature.id = getBackwardCompatibleFeatureId(output, userFeature.id);
}

const { type, manifest } = await getFeatureIdType(params, userFeature.id);
const { type, manifest } = await getFeatureIdType(params, userFeature.id, lockfile);

// cached feature
// Resolves deprecated features (fish, maven, gradle, homebrew, jupyterlab)
Expand Down Expand Up @@ -910,7 +920,7 @@ export async function processFeatureIdentifier(params: CommonParams, configPath:

// (6) Oci Identifier
if (type === 'oci' && manifest) {
return tryGetOCIFeatureSet(output, userFeature.id, userFeature.options, manifest.manifestObj, originalUserFeatureId);
return tryGetOCIFeatureSet(output, userFeature.id, userFeature.options, manifest, originalUserFeatureId);
}

output.write(`Github feature.`);
Expand Down Expand Up @@ -1019,13 +1029,13 @@ async function fetchFeatures(params: { extensionPath: string; cwd: string; outpu
if (sourceInfoType === 'oci') {
output.write(`Fetching from OCI`, LogLevel.Trace);
await mkdirpLocal(featCachePath);
const success = await fetchOCIFeature(params, featureSet, ociCacheDir, featCachePath);
if (!success) {
const res = await fetchOCIFeature(params, featureSet, ociCacheDir, featCachePath);
if (!res) {
const err = `Could not download OCI feature: ${featureSet.sourceInformation.featureRef.id}`;
throw new Error(err);
}

if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath))) {
if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath, featureSet.sourceInformation.manifestDigest))) {
const err = `Failed to parse feature '${featureDebugId}'. Please check your devcontainer.json 'features' attribute.`;
throw new Error(err);
}
Expand All @@ -1038,7 +1048,7 @@ async function fetchFeatures(params: { extensionPath: string; cwd: string; outpu
await mkdirpLocal(featCachePath);
await cpDirectoryLocal(localFeaturesFolder, featCachePath);

if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath))) {
if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath, undefined))) {
const err = `Failed to parse feature '${featureDebugId}'. Please check your devcontainer.json 'features' attribute.`;
throw new Error(err);
}
Expand All @@ -1051,7 +1061,7 @@ async function fetchFeatures(params: { extensionPath: string; cwd: string; outpu
const executionPath = featureSet.sourceInformation.resolvedFilePath;
await cpDirectoryLocal(executionPath, featCachePath);

if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath))) {
if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath, undefined))) {
const err = `Failed to parse feature '${featureDebugId}'. Please check your devcontainer.json 'features' attribute.`;
throw new Error(err);
}
Expand Down Expand Up @@ -1089,21 +1099,21 @@ async function fetchFeatures(params: { extensionPath: string; cwd: string; outpu
}

// Attempt to fetch from 'tarballUris' in order, until one succeeds.
let didSucceed: boolean = false;
let res: { computedDigest: string } | undefined;
for (const tarballUri of tarballUris) {
didSucceed = await fetchContentsAtTarballUri(tarballUri, featCachePath, headers, dstFolder, output);
res = await fetchContentsAtTarballUri(tarballUri, featCachePath, headers, dstFolder, output);

if (didSucceed) {
if (res) {
output.write(`Succeeded fetching ${tarballUri}`, LogLevel.Trace);
if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath))) {
if (!(await applyFeatureConfigToFeature(output, featureSet, feature, featCachePath, res.computedDigest))) {
const err = `Failed to parse feature '${featureDebugId}'. Please check your devcontainer.json 'features' attribute.`;
throw new Error(err);
}
break;
}
}

if (!didSucceed) {
if (!res) {
const msg = `(!) Failed to fetch tarball for ${featureDebugId} after attempting ${tarballUris.length} possibilities.`;
throw new Error(msg);
}
Expand All @@ -1115,7 +1125,7 @@ async function fetchFeatures(params: { extensionPath: string; cwd: string; outpu
}
}

async function fetchContentsAtTarballUri(tarballUri: string, featCachePath: string, headers: { 'user-agent': string; 'Authorization'?: string; 'Accept'?: string }, dstFolder: string, output: Log): Promise<boolean> {
async function fetchContentsAtTarballUri(tarballUri: string, featCachePath: string, headers: { 'user-agent': string; 'Authorization'?: string; 'Accept'?: string }, dstFolder: string, output: Log): Promise<{ computedDigest: string } | undefined> {
const tempTarballPath = path.join(dstFolder, 'temp.tgz');
try {
const options = {
Expand All @@ -1129,9 +1139,11 @@ async function fetchContentsAtTarballUri(tarballUri: string, featCachePath: stri

if (!tarball || tarball.length === 0) {
output.write(`Did not receive a response from tarball download URI: ${tarballUri}`, LogLevel.Trace);
return false;
return undefined;
}

const computedDigest = `sha256:${crypto.createHash('sha256').update(tarball).digest('hex')}`;

// Filter what gets emitted from the tar.extract().
const filter = (file: string, _: tar.FileStat) => {
// Don't include .dotfiles or the archive itself.
Expand All @@ -1155,11 +1167,11 @@ async function fetchContentsAtTarballUri(tarballUri: string, featCachePath: stri

await cleanupIterationFetchAndMerge(tempTarballPath, output);

return true;
return { computedDigest };
} catch (e) {
output.write(`Caught failure when fetching from URI '${tarballUri}': ${e}`, LogLevel.Trace);
await cleanupIterationFetchAndMerge(tempTarballPath, output);
return false;
return undefined;
}
}

Expand All @@ -1168,7 +1180,7 @@ async function fetchContentsAtTarballUri(tarballUri: string, featCachePath: stri
// Implements the latest ('internalVersion' = '2') parsing logic,
// Falls back to earlier implementation(s) if requirements not present.
// Returns a boolean indicating whether the feature was successfully parsed.
async function applyFeatureConfigToFeature(output: Log, featureSet: FeatureSet, feature: Feature, featCachePath: string): Promise<boolean> {
async function applyFeatureConfigToFeature(output: Log, featureSet: FeatureSet, feature: Feature, featCachePath: string, computedDigest: string | undefined): Promise<boolean> {
const innerJsonPath = path.join(featCachePath, DEVCONTAINER_FEATURE_FILE_NAME);

if (!(await isLocalFile(innerJsonPath))) {
Expand All @@ -1178,6 +1190,7 @@ async function applyFeatureConfigToFeature(output: Log, featureSet: FeatureSet,
}

featureSet.internalVersion = '2';
featureSet.computedDigest = computedDigest;
feature.cachePath = featCachePath;
const jsonString: Buffer = await readLocalFile(innerJsonPath);
const featureJson = jsonc.parse(jsonString.toString());
Expand Down
16 changes: 9 additions & 7 deletions src/spec-configuration/containerFeaturesOCI.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { Log, LogLevel } from '../spec-utils/log';
import { Feature, FeatureSet } from './containerFeaturesConfiguration';
import { CommonParams, fetchOCIManifestIfExists, getBlob, getRef, ManifestContainer, OCIManifest } from './containerCollectionsOCI';
import { CommonParams, fetchOCIManifestIfExists, getBlob, getRef, ManifestContainer } from './containerCollectionsOCI';

export function tryGetOCIFeatureSet(output: Log, identifier: string, options: boolean | string | Record<string, boolean | string | undefined>, manifest: OCIManifest, originalUserFeatureId: string): FeatureSet | undefined {
export function tryGetOCIFeatureSet(output: Log, identifier: string, options: boolean | string | Record<string, boolean | string | undefined>, manifest: ManifestContainer, originalUserFeatureId: string): FeatureSet | undefined {
const featureRef = getRef(output, identifier);
if (!featureRef) {
output.write(`Unable to parse '${identifier}'`, LogLevel.Error);
Expand All @@ -19,7 +19,8 @@ export function tryGetOCIFeatureSet(output: Log, identifier: string, options: bo
let featureSet: FeatureSet = {
sourceInformation: {
type: 'oci',
manifest: manifest,
manifest: manifest.manifestObj,
manifestDigest: manifest.contentDigest,
featureRef: featureRef,
userFeatureId: originalUserFeatureId,
userFeatureIdWithoutVersion
Expand All @@ -43,7 +44,7 @@ export async function fetchOCIFeatureManifestIfExistsFromUserIdentifier(params:

// Download a feature from which a manifest was previously downloaded.
// Specification: https://github.com/opencontainers/distribution-spec/blob/v1.0.1/spec.md#pulling-blobs
export async function fetchOCIFeature(params: CommonParams, featureSet: FeatureSet, ociCacheDir: string, featCachePath: string): Promise<boolean> {
export async function fetchOCIFeature(params: CommonParams, featureSet: FeatureSet, ociCacheDir: string, featCachePath: string) {
const { output } = params;

if (featureSet.sourceInformation.type !== 'oci') {
Expand All @@ -53,14 +54,15 @@ export async function fetchOCIFeature(params: CommonParams, featureSet: FeatureS

const { featureRef } = featureSet.sourceInformation;

const blobUrl = `https://${featureSet.sourceInformation.featureRef.registry}/v2/${featureSet.sourceInformation.featureRef.path}/blobs/${featureSet.sourceInformation.manifest?.layers[0].digest}`;
const expectedDigest = featureSet.sourceInformation.manifest?.layers[0].digest;
const blobUrl = `https://${featureSet.sourceInformation.featureRef.registry}/v2/${featureSet.sourceInformation.featureRef.path}/blobs/${expectedDigest}`;
output.write(`blob url: ${blobUrl}`, LogLevel.Trace);

const blobResult = await getBlob(params, blobUrl, ociCacheDir, featCachePath, featureRef);
const blobResult = await getBlob(params, blobUrl, ociCacheDir, featCachePath, featureRef, expectedDigest);

if (!blobResult) {
throw new Error(`Failed to download package for ${featureSet.sourceInformation.featureRef.resource}`);
}

return true;
return blobResult;
}
2 changes: 1 addition & 1 deletion src/spec-configuration/containerTemplatesOCI.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ export async function fetchTemplate(params: CommonParams, selectedTemplate: Sele
output.write(`blob url: ${blobUrl}`, LogLevel.Trace);

const tmpDir = userProvidedTmpDir || path.join(os.tmpdir(), 'vsch-template-temp', `${Date.now()}`);
const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, ['devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json');
const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, blobDigest, ['devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json');

if (!blobResult) {
throw new Error(`Failed to download package for ${templateRef.resource}`);
Expand Down
Loading