From 5d06af9260160ebb0dab8c25e1a71293fa88a7e8 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Thu, 13 Jul 2023 13:34:32 +0200 Subject: [PATCH 1/4] feat: support url replace via env --- src/cli/services/env.service.ts | 37 +++++++++++++++++++++----- src/cli/services/http.service.spec.ts | 38 +++++++++++++++++++++------ src/cli/services/http.service.ts | 19 ++++++++++++-- 3 files changed, 77 insertions(+), 17 deletions(-) diff --git a/src/cli/services/env.service.ts b/src/cli/services/env.service.ts index 4486951173..dc44568a3f 100644 --- a/src/cli/services/env.service.ts +++ b/src/cli/services/env.service.ts @@ -1,12 +1,15 @@ import { arch } from 'node:os'; -import { geteuid } from 'node:process'; +import { env, geteuid } from 'node:process'; import { injectable } from 'inversify'; -import type { Arch } from '../utils'; +import { type Arch, logger } from '../utils'; + +export type Replacements = Record; @injectable() export class EnvService { readonly arch: Arch; private uid: number; + private replacements: Replacements | undefined; constructor() { this.uid = geteuid?.() ?? 0; // fallback should never happen on linux @@ -24,7 +27,7 @@ export class EnvService { } get cacheDir(): string | null { - return process.env.CONTAINERBASE_CACHE_DIR ?? null; + return env.CONTAINERBASE_CACHE_DIR ?? null; } get isRoot(): boolean { @@ -32,15 +35,15 @@ export class EnvService { } get userHome(): string { - return process.env.USER_HOME ?? `/home/${this.userName}`; + return env.USER_HOME ?? `/home/${this.userName}`; } get userName(): string { - return process.env.USER_NAME ?? 'ubuntu'; + return env.USER_NAME ?? 'ubuntu'; } get userId(): number { - return parseInt(process.env.USER_ID ?? '1000', 10); + return parseInt(env.USER_ID ?? '1000', 10); } get umask(): number { @@ -48,6 +51,26 @@ export class EnvService { } get skipTests(): boolean { - return !!process.env.SKIP_VERSION; + return !!env.SKIP_VERSION; + } + + get urlReplacements(): Record { + if (this.replacements) { + return this.replacements; + } + const replacements: Record = {}; + const fromRe = /^URL_REPLACE_\d+_FROM$/; + for (const from of Object.keys(env).filter((key) => fromRe.test(key))) { + const to = from.replace(/_FROM$/, '_TO'); + if (env[from] && env[to]) { + replacements[env[from]!] = env[to]!; + } else { + logger.warn( + `Invalid URL replacement: ${from}=${env[from]!} ${to}=${env[to]!}` + ); + } + } + + return (this.replacements = replacements); } } diff --git a/src/cli/services/http.service.spec.ts b/src/cli/services/http.service.spec.ts index 3c687633ff..826520164c 100644 --- a/src/cli/services/http.service.spec.ts +++ b/src/cli/services/http.service.spec.ts @@ -9,19 +9,24 @@ describe('http.service', () => { beforeEach(() => { child = rootContainer.createChild(); + + for (const key of Object.keys(env)) { + if (key.startsWith('URL_REPLACE_')) { + delete env[key]; + } + } }); test('throws', async () => { - scope('https://example.com').get('/fail.txt').times(3).reply(404); + scope('https://example.com').get('/fail.txt').times(6).reply(404); const http = child.get(HttpService); await expect( http.download({ url: 'https://example.com/fail.txt' }) ).rejects.toThrow(); - // bug, currently resolves - // await expect( - // http.download({ url: 'https://example.com/fail.txt' }) - // ).rejects.toThrow(); + await expect( + http.download({ url: 'https://example.com/fail.txt' }) + ).rejects.toThrow(); }); test('download', async () => { @@ -33,8 +38,25 @@ describe('http.service', () => { `${env.CONTAINERBASE_CACHE_DIR}/d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt` ); // uses cache - // expect(await http.download({ url: 'https://example.com/test.txt' })).toBe( - // `${env.CONTAINERBASE_CACHE_DIR}/d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt` - // ); + expect(await http.download({ url: 'https://example.com/test.txt' })).toBe( + `${env.CONTAINERBASE_CACHE_DIR}/d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt` + ); + }); + + test('replaces url', async () => { + scope('https://example.org').get('/test.txt').reply(200, 'ok'); + + env.URL_REPLACE_0_FROM = 'https://example.com'; + env.URL_REPLACE_0_TO = 'https://example.org'; + + const http = child.get(HttpService); + + expect(await http.download({ url: 'https://example.org/test.txt' })).toBe( + `${env.CONTAINERBASE_CACHE_DIR}/d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt` + ); + // uses cache + expect(await http.download({ url: 'https://example.org/test.txt' })).toBe( + `${env.CONTAINERBASE_CACHE_DIR}/d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt` + ); }); }); diff --git a/src/cli/services/http.service.ts b/src/cli/services/http.service.ts index 1d6518536e..dbe2908ad1 100644 --- a/src/cli/services/http.service.ts +++ b/src/cli/services/http.service.ts @@ -1,5 +1,5 @@ import { createWriteStream } from 'node:fs'; -import { mkdir } from 'node:fs/promises'; +import { mkdir, rm } from 'node:fs/promises'; import { join } from 'node:path'; import { pipeline } from 'node:stream/promises'; import { got } from 'got'; @@ -65,9 +65,11 @@ export class HttpService { await mkdir(cachePath, { recursive: true }); + const nUrl = this.replaceUrl(url); + for (const run of [1, 2, 3]) { try { - await pipeline(got.stream(url), createWriteStream(filePath)); + await pipeline(got.stream(nUrl), createWriteStream(filePath)); return filePath; } catch (err) { if (run === 3) { @@ -77,6 +79,19 @@ export class HttpService { } } } + await rm(cachePath, { recursive: true }); throw new Error('download failed'); } + private replaceUrl(src: string): string { + let tgt = src; + const replacements = this.envSvc.urlReplacements; + + for (const from of Object.keys(replacements)) { + tgt = tgt.replace(from, replacements[from]); + } + if (tgt !== src) { + logger.debug({ src, tgt }, 'url replaced'); + } + return tgt; + } } From 5c70b7611863301ceb85e8ff4d245fdd624d0243 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Thu, 13 Jul 2023 13:44:59 +0200 Subject: [PATCH 2/4] docs: added sample to readme --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 789a6d51c5..5a76cd21c2 100644 --- a/README.md +++ b/README.md @@ -90,3 +90,19 @@ RUN install-tool node 14.17.3 # renovate: datasource=github-releases packageName=moby/moby RUN install-tool docker 20.10.7 ``` + +### Url replacement + +You can replace the default urls used to download the tools. +This is currently only supported by the `docker` tool installer. +Checkout #1067 for additional support. + +```Dockerfile +FROM containerbase/base + +ENV URL_REPLACE_0_FROM=https://download.docker.com/linux/static/stable +ENV URL_REPLACE_0_TO=https://artifactory.proxy.test/some/virtual/patch/docker + +# renovate: datasource=github-releases packageName=moby/moby +RUN install-tool docker v24.0.2 +``` From 3a66cb0899ef5d4b2cfdc9fa134d73efb8b43986 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Thu, 13 Jul 2023 13:55:43 +0200 Subject: [PATCH 3/4] test: fix tests --- src/cli/services/http.service.spec.ts | 25 +++++++++++++------------ test/latest/Dockerfile | 5 +++++ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/cli/services/http.service.spec.ts b/src/cli/services/http.service.spec.ts index 826520164c..83cbea8cfb 100644 --- a/src/cli/services/http.service.spec.ts +++ b/src/cli/services/http.service.spec.ts @@ -4,6 +4,7 @@ import { beforeEach, describe, expect, test } from 'vitest'; import { HttpService, rootContainer } from '.'; import { scope } from '~test/http-mock'; +const baseUrl = 'https://example.com'; describe('http.service', () => { let child!: Container; @@ -18,45 +19,45 @@ describe('http.service', () => { }); test('throws', async () => { - scope('https://example.com').get('/fail.txt').times(6).reply(404); + scope(baseUrl).get('/fail.txt').times(6).reply(404); const http = child.get(HttpService); await expect( - http.download({ url: 'https://example.com/fail.txt' }) + http.download({ url: `${baseUrl}/fail.txt` }) ).rejects.toThrow(); await expect( - http.download({ url: 'https://example.com/fail.txt' }) + http.download({ url: `${baseUrl}/fail.txt` }) ).rejects.toThrow(); }); test('download', async () => { - scope('https://example.com').get('/test.txt').reply(200, 'ok'); + scope(baseUrl).get('/test.txt').reply(200, 'ok'); const http = child.get(HttpService); - expect(await http.download({ url: 'https://example.com/test.txt' })).toBe( + expect(await http.download({ url: `${baseUrl}/test.txt` })).toBe( `${env.CONTAINERBASE_CACHE_DIR}/d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt` ); // uses cache - expect(await http.download({ url: 'https://example.com/test.txt' })).toBe( + expect(await http.download({ url: `${baseUrl}/test.txt` })).toBe( `${env.CONTAINERBASE_CACHE_DIR}/d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt` ); }); test('replaces url', async () => { - scope('https://example.org').get('/test.txt').reply(200, 'ok'); + scope('https://example.org').get('/replace.txt').reply(200, 'ok'); - env.URL_REPLACE_0_FROM = 'https://example.com'; + env.URL_REPLACE_0_FROM = baseUrl; env.URL_REPLACE_0_TO = 'https://example.org'; const http = child.get(HttpService); - expect(await http.download({ url: 'https://example.org/test.txt' })).toBe( - `${env.CONTAINERBASE_CACHE_DIR}/d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt` + expect(await http.download({ url: `${baseUrl}/replace.txt` })).toBe( + `${env.CONTAINERBASE_CACHE_DIR}/f4eba41457a330d0fa5289e49836326c6a0208bbc639862e70bb378c88c62642/replace.txt` ); // uses cache - expect(await http.download({ url: 'https://example.org/test.txt' })).toBe( - `${env.CONTAINERBASE_CACHE_DIR}/d1dc63218c42abba594fff6450457dc8c4bfdd7c22acf835a50ca0e5d2693020/test.txt` + expect(await http.download({ url: `${baseUrl}/replace.txt` })).toBe( + `${env.CONTAINERBASE_CACHE_DIR}/f4eba41457a330d0fa5289e49836326c6a0208bbc639862e70bb378c88c62642/replace.txt` ); }); }); diff --git a/test/latest/Dockerfile b/test/latest/Dockerfile index 3a7caf506b..d0e6595297 100644 --- a/test/latest/Dockerfile +++ b/test/latest/Dockerfile @@ -15,6 +15,11 @@ COPY src/ / RUN install-containerbase +ENV LOG_LEVEL=debug +RUN set -ex; \ + install-tool docker v24.0.2; \ + false + #-------------------------------------- # build #-------------------------------------- From 5b8104b2931dce8f08f1153355edadcd2c023fbc Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Thu, 13 Jul 2023 14:01:37 +0200 Subject: [PATCH 4/4] test: fix test --- test/latest/Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/latest/Dockerfile b/test/latest/Dockerfile index d0e6595297..3a7caf506b 100644 --- a/test/latest/Dockerfile +++ b/test/latest/Dockerfile @@ -15,11 +15,6 @@ COPY src/ / RUN install-containerbase -ENV LOG_LEVEL=debug -RUN set -ex; \ - install-tool docker v24.0.2; \ - false - #-------------------------------------- # build #--------------------------------------