From 7e6f62c12360cea798ee2e94594b5219d2ffd3b0 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 24 May 2018 14:48:22 +0200 Subject: [PATCH 1/7] This never got into the PR, but was meant to tidy up the solution --- nodejs/mnpm.js | 2 +- nodejs/mnpm.spec.js | 41 +++++++++++++++++++++++++++-------------- package.json | 2 ++ 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/nodejs/mnpm.js b/nodejs/mnpm.js index 8674c82..cf7ee52 100644 --- a/nodejs/mnpm.js +++ b/nodejs/mnpm.js @@ -1,6 +1,6 @@ const fs = require('fs'); const tar = require('tar-stream'); -const zlib = require('zlib'); +const zlib = require('./zlib-choice'); const tarContentMtime = new Date(946684800000); diff --git a/nodejs/mnpm.spec.js b/nodejs/mnpm.spec.js index b2c6461..6124df4 100644 --- a/nodejs/mnpm.spec.js +++ b/nodejs/mnpm.spec.js @@ -2,7 +2,7 @@ const os = require('os'); const path = require('path'); const fs = require('fs'); const crypto = require('crypto'); -const zlib = require('zlib'); +const zlib = require('./zlib-choice'); const stream = require('stream'); const tar = require('tar-stream'); @@ -19,13 +19,14 @@ describe("stringifyPackageJson", () => { describe("Our choice of gzip function", () => { - it("Is platform independent wrt result checksum", () => { + it("Is platform independent wrt result checksum", done => { const blob = new stream.PassThrough(); const sha256 = crypto.createHash('sha256'); const result = new stream.PassThrough(); result.on('data', d => sha256.update(d)); result.on('end', () => expect(sha256.digest('hex')).toBe( - 'c5f9a2352dadba9488900ba6ede0133270e12350ffa6d6ebbdefef9ee6aa2238')); + 'b13627bbeee31ae666d6696cf11e411ee6b0e40d4b235cb2a02da32693ba2d3c')); + result.on('end', done); // Note that this differs from `echo 'x' | gzip - | shasum -a 256 -` blob.pipe(zlib.createGzip()).pipe(result); blob.end('x\n'); @@ -36,7 +37,7 @@ describe("Our choice of gzip function", () => { expect(process.versions.zlib).toBe('1.2.11'); }); - it("Results may depend on zlib options", () => { + it("Results may depend on zlib options", done => { const options = { windowBits: 14, memLevel: 7, level: zlib.constants.Z_BEST_SPEED, @@ -47,13 +48,13 @@ describe("Our choice of gzip function", () => { const result = new stream.PassThrough(); result.on('data', d => sha256.update(d)); result.on('end', () => expect(sha256.digest('hex')).toBe( - '270b40b49af0dcc1de9631f231d022c93c07f29d2940e14d22ffdd797165e24f')); - // Note that this differs from `echo 'x' | gzip - | shasum -a 256 -` + 'dd8dbe0ba323ab288d9e9272efc1f2bf52f495a812122c6ee9f9c5e7d765fda5')); + result.on('end', done); blob.pipe(zlib.createGzip(options)).pipe(result); blob.end('x\n'); }); - it("Results may depend on zlib compression level", () => { + it("Results may depend on zlib compression level", done => { const options = { level: zlib.constants.Z_BEST_COMPRESSION }; @@ -62,13 +63,13 @@ describe("Our choice of gzip function", () => { const result = new stream.PassThrough(); result.on('data', d => sha256.update(d)); result.on('end', () => expect(sha256.digest('hex')).toBe( - '1437e4b499b5063c9530244d1655dba6986bcbd1258f81f122bdc4c983058ef4')); - // Note that this differs from `echo 'x' | gzip - | shasum -a 256 -` + '6cda46810118792ed89f1e1662549186b5c851e4ce240be861780bc646e850c6')); + result.on('end', done); blob.pipe(zlib.createGzip(options)).pipe(result); blob.end('x\n'); }); - it("Results may be more platform independent with no compression", () => { + it("Results may be more platform independent with no compression", done => { const options = { level: zlib.constants.Z_NO_COMPRESSION }; @@ -77,12 +78,24 @@ describe("Our choice of gzip function", () => { const result = new stream.PassThrough(); result.on('data', d => sha256.update(d)); result.on('end', () => expect(sha256.digest('hex')).toBe( - '35932a249baf5fe47b9fecaa3482309e447ed8d9b01e207a6ce2846724006784')); - // Note that this differs from `echo 'x' | gzip - | shasum -a 256 -` + 'f2b18200cd38c0d2c3dff4d3e2be9fd83069acd6b73cbc835c708fc3693c45d9')); + result.on('end', done); blob.pipe(zlib.createGzip(options)).pipe(result); blob.end('x\n'); }); + xit("Results may be better with inflate instead of gzip", done => { + const blob = new stream.PassThrough(); + const sha256 = crypto.createHash('sha256'); + const result = new stream.PassThrough(); + result.on('data', d => sha256.update(d)); + result.on('end', () => expect(sha256.digest('hex')).toBe( + 'c5f9a2352dadba9488900ba6ede0133270e12350ffa6d6ebbdefef9ee6aa2238')); + result.on('end', done); + blob.pipe(zlib.createInflate()).pipe(result); + blob.end('x\n'); + }); + }); describe("writeProdPackageTgzWithDeterministicHash", () => { @@ -104,10 +117,10 @@ describe("writeProdPackageTgzWithDeterministicHash", () => { const tgz = await fs.promises.readFile(filePath); const sha256 = crypto.createHash('sha256'); sha256.update(tgz); - expect(sha256.digest('hex')).toBe('3be69fccaf4716df00adee93c219cfe44f1425aa968d33b6a3a4e725192586be'); + expect(sha256.digest('hex')).toBe('ebfa2ce786383196d29b927d3f0a51539655ebe56d4bac52b96f7e13749ba79c'); const sha512 = crypto.createHash('sha512'); sha512.update(tgz); - expect(sha512.digest('base64')).toBe('M24fZ1mSsZYX8dSCGSd54842GgAKd80xInWqNUhSZH1/hx7syKOOx05qMhD8avcFdXNnDMG/N2i/YZJFJNW6rQ=='); + expect(sha512.digest('base64')).toBe('4aKJrQoeaGZuY8IDk/LmKX9drIVPoeQG00phQ7kZoR+SXHtrgeA19uUnBrclpm4Sm6xIv8/50V5u/dPxWg62Iw=='); await fs.promises.unlink(filePath); }); diff --git a/package.json b/package.json index 7eb81a8..f26d277 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,8 @@ "node": ">=10.1.0" }, "dependencies": { + "browserify-zlib": "0.2.0", + "pako": "1.0.6", "tar-stream": "1.6.1", "yamljs": "0.2.8" }, From d2a9cb1719fe1ec70c4a87883a99bc7b244c9409 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 25 Oct 2016 14:19:46 +0200 Subject: [PATCH 2/7] WIP https://github.com/Yolean/build-contract/issues/12 --- build-contract | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-contract b/build-contract index bceb3b3..1406444 100755 --- a/build-contract +++ b/build-contract @@ -74,8 +74,8 @@ for compose_file in $(ls $CONTRACTS_DIR | grep .yml); do # and volumes (including named volumes declared in the docker-compose.yml file) $docker_compose down -v - $docker_compose build $BUILD_FLAGS - $docker_compose up --force-recreate -d + #$docker_compose build $BUILD_FLAGS + $docker_compose up --build --force-recreate -d $docker_compose logs -f & bar=$(wait_for_contract $compose_name) echo " --- build-contract: Build Contract finished with $bar" From be6cb5b18544532a8b4b685fc38afdf17e677f9f Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 24 May 2018 14:51:26 +0200 Subject: [PATCH 3/7] Comments on the situation --- build-contract | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-contract b/build-contract index 1406444..0cf8455 100755 --- a/build-contract +++ b/build-contract @@ -74,6 +74,8 @@ for compose_file in $(ls $CONTRACTS_DIR | grep .yml); do # and volumes (including named volumes declared in the docker-compose.yml file) $docker_compose down -v + # build here fails when a build needs a depends_on to be built first, not pulled + # but up --build seems to work, see https://github.com/Yolean/build-contract/pull/13 #$docker_compose build $BUILD_FLAGS $docker_compose up --build --force-recreate -d $docker_compose logs -f & From 5582998566be026fe52b0acbd15e562526886475 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 24 May 2018 20:30:18 +0200 Subject: [PATCH 4/7] 1.6.0 doesn't run docker-compose build --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index fdb1113..2149e5e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "build-contract", - "version": "1.5.0", + "version": "1.6.0", "description": "Defines a successful build and test run for a microservice, from source to docker push", "repository": { "type": "git", From 136574067b2104d84c40e36191f7a2f0138b722b Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 24 May 2018 20:31:58 +0200 Subject: [PATCH 5/7] You get ugly error messages about failing to connect to localhost with localhost:5000/... image names if you forget the PUSH_TAG so let's have it in the example even if some builds don't use it --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d174931..4f55ecb 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,12 @@ Defines a successful build and test run for a microservice, from source to docke Invoke build-contract in current folder, use host's docker: ``` docker build --tag yolean/build-contract . -docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd)/:/source --rm --name mybuild yolean/build-contract test +docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd)/:/source --rm --name mybuild -e PUSH_TAG=testing yolean/build-contract test ``` Or for monorepo: ``` -docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd)/../:/source -w /source/$(basename $(pwd)) --rm --name mybuild yolean/build-contract test +docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd)/../:/source -w /source/$(basename $(pwd)) --rm --name mybuild -e PUSH_TAG=testing yolean/build-contract test ``` Add `-t` for colors and Ctrl+C support. From 6671feb26ee8384976afd1ed0d484b88fd7b02b0 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 25 May 2018 12:59:34 +0200 Subject: [PATCH 6/7] Removes BUILD_FLAGS, i.e. pull-always, requiring us to use images tagged with sha, which is a requirement anyway for builds to be stable over time. If we really want pull on every build we must find a docker-compose feature akin to imagePullPolicy in k8s --- build-contract | 9 --------- 1 file changed, 9 deletions(-) diff --git a/build-contract b/build-contract index 0cf8455..cd7ed1f 100755 --- a/build-contract +++ b/build-contract @@ -52,12 +52,6 @@ function wait_for_contract { fi } -# Perform tests -BUILD_FLAGS=''; -if [[ $DO_PUSH == true ]]; then - BUILD_FLAGS="$BUILD_FLAGS --pull"; -fi - MONOREPO_PRE=$(cat package.json | grep '"build-contract-predockerbuild"' | awk -F '"' '{ print $4 }') if [[ "$MONOREPO_PRE" == "#" ]]; then $DIR/nodejs/build-contract-predockerbuild elif [[ ! -z "$MONOREPO_PRE" ]]; then npm run build-contract-predockerbuild; fi @@ -74,9 +68,6 @@ for compose_file in $(ls $CONTRACTS_DIR | grep .yml); do # and volumes (including named volumes declared in the docker-compose.yml file) $docker_compose down -v - # build here fails when a build needs a depends_on to be built first, not pulled - # but up --build seems to work, see https://github.com/Yolean/build-contract/pull/13 - #$docker_compose build $BUILD_FLAGS $docker_compose up --build --force-recreate -d $docker_compose logs -f & bar=$(wait_for_contract $compose_name) From 576abacb15cd65384430fdca8244bdba6bfd0394 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 26 May 2018 22:17:34 +0200 Subject: [PATCH 7/7] Bumps to latest yolean/node --- Dockerfile | 2 +- package.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3de80c1..6c5afc7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM yolean/node@sha256:e591eac6f5d1f07876bd63bba2bbd1c1218521c5ed5312692851597b47089775 +FROM yolean/node@sha256:9f4a2b32e4ba065ea5471d4d83084d4ba1f860a8bde614518b386999bf0cf9b2 ENV docker_version=17.09.1~ce-0~debian ENV compose_version=1.21.0 compose_sha256=af639f5e9ca229442c8738135b5015450d56e2c1ae07c0aaa93b7da9fe09c2b0 diff --git a/package.json b/package.json index 2149e5e..6eebeb3 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "build-contract-predockerbuild": "./nodejs/build-contract-predockerbuild" }, "engines": { - "node": ">=10.1.0" + "node": ">=10.2.1" }, "dependencies": { "browserify-zlib": "0.2.0", @@ -32,7 +32,7 @@ "yamljs": "0.2.8" }, "peerDependencies": { - "npm": "6.0.1" + "npm": "6.1.0" }, "devDependencies": { "jest": "22.4.4"