From 1b95fa3de7a1a11978a7e78f3ca78fac8302c424 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 15 Jun 2022 20:51:36 +0000 Subject: [PATCH 01/23] test --- .github/actions/smoke-test/action.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/actions/smoke-test/action.yaml b/.github/actions/smoke-test/action.yaml index 9ad3ec42c3..1eb75cf83d 100644 --- a/.github/actions/smoke-test/action.yaml +++ b/.github/actions/smoke-test/action.yaml @@ -11,7 +11,16 @@ inputs: runs: using: composite + runs-on: ubuntu-latest steps: + - name: Free more space + id: free_space + run: | + set -e + # Ensure enough space is available for build + sudo apt-get autoremove -y + sudo apt-get clean -y + sudo rm -rf /usr/share/dotnet - name: Checkout main id: checkout_release uses: actions/checkout@v3 From d66d8782f5803d675b33edb2a0fcada65c199c09 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 15 Jun 2022 20:56:04 +0000 Subject: [PATCH 02/23] add as a job --- .github/actions/smoke-test/action.yaml | 54 +++++++++++++------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/actions/smoke-test/action.yaml b/.github/actions/smoke-test/action.yaml index 1eb75cf83d..46ac9349b6 100644 --- a/.github/actions/smoke-test/action.yaml +++ b/.github/actions/smoke-test/action.yaml @@ -9,30 +9,30 @@ inputs: required: true default: 'vscode' -runs: - using: composite - runs-on: ubuntu-latest - steps: - - name: Free more space - id: free_space - run: | - set -e - # Ensure enough space is available for build - sudo apt-get autoremove -y - sudo apt-get clean -y - sudo rm -rf /usr/share/dotnet - - name: Checkout main - id: checkout_release - uses: actions/checkout@v3 - with: - repository: 'devcontainers/images' - path: '__build' - ref: 'main' - - name: Build image - id: build_image - shell: bash - run: ${{ github.action_path }}/build.sh ${{ inputs.definition }} - - name: Test image - id: test_image - shell: bash - run: chmod +x ${{ github.action_path }}/test.sh && ${{ github.action_path }}/test.sh ${{ inputs.definition }} ${{ inputs.user }} +jobs: + build-and-test: + runs-on: ubuntu-latest + steps: + - name: Free more space + id: free_space + run: | + set -e + # Ensure enough space is available for build + sudo apt-get autoremove -y + sudo apt-get clean -y + sudo rm -rf /usr/share/dotnet + - name: Checkout main + id: checkout_release + uses: actions/checkout@v3 + with: + repository: 'devcontainers/images' + path: '__build' + ref: 'main' + - name: Build image + id: build_image + shell: bash + run: ${{ github.action_path }}/build.sh ${{ inputs.definition }} + - name: Test image + id: test_image + shell: bash + run: chmod +x ${{ github.action_path }}/test.sh && ${{ github.action_path }}/test.sh ${{ inputs.definition }} ${{ inputs.user }} From 8cc22d7bc71a252ba8330a3be554bf2a9d07b107 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 15 Jun 2022 21:05:30 +0000 Subject: [PATCH 03/23] fix action --- .github/actions/smoke-test/action.yaml | 45 +++++++++++++------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/.github/actions/smoke-test/action.yaml b/.github/actions/smoke-test/action.yaml index 46ac9349b6..95477f6e3a 100644 --- a/.github/actions/smoke-test/action.yaml +++ b/.github/actions/smoke-test/action.yaml @@ -11,28 +11,27 @@ inputs: jobs: build-and-test: + name: Build and test images runs-on: ubuntu-latest steps: - - name: Free more space - id: free_space - run: | - set -e - # Ensure enough space is available for build - sudo apt-get autoremove -y - sudo apt-get clean -y - sudo rm -rf /usr/share/dotnet - - name: Checkout main - id: checkout_release - uses: actions/checkout@v3 - with: - repository: 'devcontainers/images' - path: '__build' - ref: 'main' - - name: Build image - id: build_image - shell: bash - run: ${{ github.action_path }}/build.sh ${{ inputs.definition }} - - name: Test image - id: test_image - shell: bash - run: chmod +x ${{ github.action_path }}/test.sh && ${{ github.action_path }}/test.sh ${{ inputs.definition }} ${{ inputs.user }} + - name: Free more space + id: free_space + run: | + set -e + # Ensure enough space is available for build + sudo apt-get autoremove -y + sudo apt-get clean -y + sudo rm -rf /usr/share/dotnet + + - name: Checkout main + id: checkout_release + uses: actions/checkout@v2 + + - name: Build image + id: build_image + run: .github/actions/smoke-test/build.sh ${{ inputs.definition }} + + - name: Test image + id: test_image + shell: bash + run: chmod +x .github/actions/smoke-test/test.sh && .github/actions/smoke-test/test.sh ${{ inputs.definition }} ${{ inputs.user }} From d1bf97ea6e7d6a9936c9779812134c26b24d61f3 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 15 Jun 2022 21:14:32 +0000 Subject: [PATCH 04/23] revert changes --- .github/actions/smoke-test/action.yaml | 20 ++++++++++---------- .github/actions/smoke-test/build.sh | 4 ++++ .github/actions/smoke-test/test.sh | 3 +++ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/actions/smoke-test/action.yaml b/.github/actions/smoke-test/action.yaml index 95477f6e3a..51f1a3898b 100644 --- a/.github/actions/smoke-test/action.yaml +++ b/.github/actions/smoke-test/action.yaml @@ -9,11 +9,9 @@ inputs: required: true default: 'vscode' -jobs: - build-and-test: - name: Build and test images - runs-on: ubuntu-latest - steps: +runs: + using: composite + steps: - name: Free more space id: free_space run: | @@ -22,16 +20,18 @@ jobs: sudo apt-get autoremove -y sudo apt-get clean -y sudo rm -rf /usr/share/dotnet - - name: Checkout main id: checkout_release uses: actions/checkout@v2 - + with: + repository: 'devcontainers/images' + path: '__build' + ref: 'main' - name: Build image id: build_image - run: .github/actions/smoke-test/build.sh ${{ inputs.definition }} - + shell: bash + run: ${{ github.action_path }}/build.sh ${{ inputs.definition }} - name: Test image id: test_image shell: bash - run: chmod +x .github/actions/smoke-test/test.sh && .github/actions/smoke-test/test.sh ${{ inputs.definition }} ${{ inputs.user }} + run: chmod +x ${{ github.action_path }}/test.sh && ${{ github.action_path }}/test.sh ${{ inputs.definition }} ${{ inputs.user }} diff --git a/.github/actions/smoke-test/build.sh b/.github/actions/smoke-test/build.sh index a63409ed69..68ae6ee5ac 100755 --- a/.github/actions/smoke-test/build.sh +++ b/.github/actions/smoke-test/build.sh @@ -5,6 +5,10 @@ set -e export DOCKER_BUILDKIT=1 echo "(*) Pulling latest '@devcontainer/cli" + +docker ps -a +docker images + # npm install -g @devcontainers/cli #Temporarily installing cli from source until https://github.com/devcontainers/cli/pull/6 is merged diff --git a/.github/actions/smoke-test/test.sh b/.github/actions/smoke-test/test.sh index 0ea023a30e..7364bf6c23 100755 --- a/.github/actions/smoke-test/test.sh +++ b/.github/actions/smoke-test/test.sh @@ -10,6 +10,9 @@ echo "(*) Running test..." cd build chmod +x devcontainers-cli-0.3.0-1.tgz +docker ps -a +docker images + npx --yes devcontainers-cli-0.3.0-1.tgz exec --workspace-folder $(pwd)/../src/$DEFINITION /bin/sh -c 'set -e && if [ -f "test-project/test.sh" ]; then cd test-project && if [ "$(id -u)" = "0" ]; then chmod +x test.sh; else sudo chmod +x test.sh; fi && ./test.sh; else ls -a; fi' # Clean up From 75cd42e2cbfb364d6a16530793e4c8dd5e660b4b Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 15 Jun 2022 21:16:27 +0000 Subject: [PATCH 05/23] add shell --- .github/actions/smoke-test/action.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/smoke-test/action.yaml b/.github/actions/smoke-test/action.yaml index 51f1a3898b..bda1c6ae3d 100644 --- a/.github/actions/smoke-test/action.yaml +++ b/.github/actions/smoke-test/action.yaml @@ -13,7 +13,8 @@ runs: using: composite steps: - name: Free more space - id: free_space + id: free_space + shell: bash run: | set -e # Ensure enough space is available for build From d8980a323880b84cd847ceb894a032796e3e551c Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 15 Jun 2022 21:21:38 +0000 Subject: [PATCH 06/23] docker prune --- .github/actions/smoke-test/build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/smoke-test/build.sh b/.github/actions/smoke-test/build.sh index 68ae6ee5ac..2301a8b5b7 100755 --- a/.github/actions/smoke-test/build.sh +++ b/.github/actions/smoke-test/build.sh @@ -8,7 +8,9 @@ echo "(*) Pulling latest '@devcontainer/cli" docker ps -a docker images - +docker system prune --force +docker ps -a +docker images # npm install -g @devcontainers/cli #Temporarily installing cli from source until https://github.com/devcontainers/cli/pull/6 is merged From d9ee49233f2aa5b50d81a147058ff9dc334852cc Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 15 Jun 2022 21:39:34 +0000 Subject: [PATCH 07/23] test using vscdc --- .github/actions/smoke-test/action.yaml | 17 ++++++------- .github/actions/smoke-test/build.sh | 32 +++++++++++++----------- .github/actions/smoke-test/test.sh | 33 ++++++++++++++++++------- .github/workflows/smoke-alpine.yaml | 1 + .github/workflows/smoke-anaconda.yaml | 1 + .github/workflows/smoke-codespaces.yaml | 1 + .github/workflows/smoke-cpp.yaml | 1 + .github/workflows/smoke-debian.yaml | 1 + .github/workflows/smoke-dotnet.yaml | 1 + .github/workflows/smoke-go.yaml | 1 + .github/workflows/smoke-java.yaml | 1 + .github/workflows/smoke-jekyll.yaml | 1 + .github/workflows/smoke-miniconda.yaml | 1 + .github/workflows/smoke-php.yaml | 1 + .github/workflows/smoke-python.yaml | 2 +- .github/workflows/smoke-ruby.yaml | 1 + .github/workflows/smoke-rust.yaml | 1 + .github/workflows/smoke-ubuntu.yaml | 1 + 18 files changed, 64 insertions(+), 34 deletions(-) diff --git a/.github/actions/smoke-test/action.yaml b/.github/actions/smoke-test/action.yaml index bda1c6ae3d..c09f7388ed 100644 --- a/.github/actions/smoke-test/action.yaml +++ b/.github/actions/smoke-test/action.yaml @@ -4,6 +4,10 @@ inputs: description: 'Definition to test' required: true default: 'debian' + image: + description: 'Image to run smoke test in' + required: true + default: 'none' user: description: 'User in container image to use for smoke test' required: true @@ -12,15 +16,6 @@ inputs: runs: using: composite steps: - - name: Free more space - id: free_space - shell: bash - run: | - set -e - # Ensure enough space is available for build - sudo apt-get autoremove -y - sudo apt-get clean -y - sudo rm -rf /usr/share/dotnet - name: Checkout main id: checkout_release uses: actions/checkout@v2 @@ -28,11 +23,13 @@ runs: repository: 'devcontainers/images' path: '__build' ref: 'main' + - name: Build image id: build_image shell: bash run: ${{ github.action_path }}/build.sh ${{ inputs.definition }} + - name: Test image id: test_image shell: bash - run: chmod +x ${{ github.action_path }}/test.sh && ${{ github.action_path }}/test.sh ${{ inputs.definition }} ${{ inputs.user }} + run: ${{ github.action_path }}/test.sh ${{ inputs.definition }} ${{ inputs.image }} ${{ inputs.user }} diff --git a/.github/actions/smoke-test/build.sh b/.github/actions/smoke-test/build.sh index 2301a8b5b7..6850171a3b 100755 --- a/.github/actions/smoke-test/build.sh +++ b/.github/actions/smoke-test/build.sh @@ -4,19 +4,23 @@ DEFINITION="$1" set -e export DOCKER_BUILDKIT=1 -echo "(*) Pulling latest '@devcontainer/cli" -docker ps -a -docker images -docker system prune --force -docker ps -a -docker images -# npm install -g @devcontainers/cli +# Symlink build scripts from main to improve security when testing PRs +if [ -d "$GITHUB_WORKSPACE/__build/build" ]; then + cp -r "$GITHUB_WORKSPACE/__build/build" "$GITHUB_WORKSPACE/" +else + echo "WARNING: Using build/vscdc from $GITHUB_REF instead of main." +fi +rm -rf node_modules +yarn install -#Temporarily installing cli from source until https://github.com/devcontainers/cli/pull/6 is merged -cd build -chmod +x devcontainers-cli-0.3.0-1.tgz - -echo "(*) Building image - ${DEFINITION}" -npx --yes devcontainers-cli-0.3.0-1.tgz up --workspace-folder ../src/${DEFINITION} -# devcontainer build --workspace-folder "src/${DEFINITION}/" --image-name vsc-${DEFINITION} +# Run test build +chmod +x build/vscdc +build/vscdc push ${DEFINITION} \ + --no-push \ + --release dev \ + --github-repo "devcontainers" \ + --registry "mcr.microsoft.com" \ + --registry-path "devcontainers" \ + --stub-registry "mcr.microsoft.com" \ + --stub-registry-path "devcontainers" diff --git a/.github/actions/smoke-test/test.sh b/.github/actions/smoke-test/test.sh index 7364bf6c23..a5d2604436 100755 --- a/.github/actions/smoke-test/test.sh +++ b/.github/actions/smoke-test/test.sh @@ -1,19 +1,34 @@ #/bin/bash DEFINITION="$1" -USERNAME="$2" +IMAGE="$2" +USERNAME="$3" export DOCKER_BUILDKIT=1 set -e -# Run actual test -echo "(*) Running test..." -cd build -chmod +x devcontainers-cli-0.3.0-1.tgz - -docker ps -a docker images -npx --yes devcontainers-cli-0.3.0-1.tgz exec --workspace-folder $(pwd)/../src/$DEFINITION /bin/sh -c 'set -e && if [ -f "test-project/test.sh" ]; then cd test-project && if [ "$(id -u)" = "0" ]; then chmod +x test.sh; else sudo chmod +x test.sh; fi && ./test.sh; else ls -a; fi' +# Start container +echo "(*) Starting container..." +container_name="vscdc-test-container-$DEFINITION" +docker run -d --name ${container_name} --rm --init --privileged -v "$(pwd)/src/${DEFINITION}:/workspace" ${IMAGE}-uid /bin/sh -c 'while sleep 1000; do :; done' + +# Run actual test +echo "(*) Running test..." +docker exec -u "${USERNAME}" ${container_name} /bin/sh -c '\ + set -e \ + && cd /workspace \ + && if [ -f "test-project/test.sh" ]; then \ + cd test-project \ + && if [ "$(id -u)" = "0" ]; then \ + chmod +x test.sh; \ + else \ + sudo chmod +x test.sh; \ + fi \ + && ./test.sh; \ + else \ + ls -a; + fi' # Clean up -docker stop $(docker container ls -q) \ No newline at end of file +docker rm -f ${container_name} \ No newline at end of file diff --git a/.github/workflows/smoke-alpine.yaml b/.github/workflows/smoke-alpine.yaml index 1653537ee4..41a65a4410 100644 --- a/.github/workflows/smoke-alpine.yaml +++ b/.github/workflows/smoke-alpine.yaml @@ -24,4 +24,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: alpine + image: mcr.microsoft.com/vscode/devcontainers/base:dev-alpine user: vscode diff --git a/.github/workflows/smoke-anaconda.yaml b/.github/workflows/smoke-anaconda.yaml index 36e39adcd4..1e32a56576 100644 --- a/.github/workflows/smoke-anaconda.yaml +++ b/.github/workflows/smoke-anaconda.yaml @@ -26,4 +26,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: anaconda + image: mcr.microsoft.com/vscode/devcontainers/anaconda:dev-3 user: vscode diff --git a/.github/workflows/smoke-codespaces.yaml b/.github/workflows/smoke-codespaces.yaml index 0f77334a93..56faf5b1f4 100644 --- a/.github/workflows/smoke-codespaces.yaml +++ b/.github/workflows/smoke-codespaces.yaml @@ -29,4 +29,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: codespaces + image: mcr.microsoft.com/vscode/devcontainers/universal:dev-linux user: codespace diff --git a/.github/workflows/smoke-cpp.yaml b/.github/workflows/smoke-cpp.yaml index a803f495d8..43dec0f0e9 100644 --- a/.github/workflows/smoke-cpp.yaml +++ b/.github/workflows/smoke-cpp.yaml @@ -27,4 +27,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: cpp + image: mcr.microsoft.com/vscode/devcontainers/cpp:dev-debian user: vscode diff --git a/.github/workflows/smoke-debian.yaml b/.github/workflows/smoke-debian.yaml index 1379cd1848..10668f9d77 100644 --- a/.github/workflows/smoke-debian.yaml +++ b/.github/workflows/smoke-debian.yaml @@ -27,4 +27,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: debian + image: mcr.microsoft.com/vscode/devcontainers/base:dev-debian user: vscode diff --git a/.github/workflows/smoke-dotnet.yaml b/.github/workflows/smoke-dotnet.yaml index 245959952a..e4c73a9ce6 100644 --- a/.github/workflows/smoke-dotnet.yaml +++ b/.github/workflows/smoke-dotnet.yaml @@ -29,4 +29,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: dotnet + image: mcr.microsoft.com/vscode/devcontainers/dotnet:dev-6.0 user: vscode diff --git a/.github/workflows/smoke-go.yaml b/.github/workflows/smoke-go.yaml index ed4988a486..379371e3b5 100644 --- a/.github/workflows/smoke-go.yaml +++ b/.github/workflows/smoke-go.yaml @@ -29,4 +29,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: go + image: mcr.microsoft.com/vscode/devcontainers/go:dev-1 user: vscode diff --git a/.github/workflows/smoke-java.yaml b/.github/workflows/smoke-java.yaml index 9d45283a95..cb3513bcea 100644 --- a/.github/workflows/smoke-java.yaml +++ b/.github/workflows/smoke-java.yaml @@ -29,4 +29,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: java + image: mcr.microsoft.com/vscode/devcontainers/java:dev-17 user: vscode diff --git a/.github/workflows/smoke-jekyll.yaml b/.github/workflows/smoke-jekyll.yaml index c57e882532..cb07ae623c 100644 --- a/.github/workflows/smoke-jekyll.yaml +++ b/.github/workflows/smoke-jekyll.yaml @@ -27,4 +27,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: jekyll + image: mcr.microsoft.com/vscode/devcontainers/jekyll:dev-bullseye user: vscode diff --git a/.github/workflows/smoke-miniconda.yaml b/.github/workflows/smoke-miniconda.yaml index 154237c3f6..31eae507d1 100644 --- a/.github/workflows/smoke-miniconda.yaml +++ b/.github/workflows/smoke-miniconda.yaml @@ -29,4 +29,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: miniconda + image: mcr.microsoft.com/vscode/devcontainers/miniconda:dev-3 user: vscode diff --git a/.github/workflows/smoke-php.yaml b/.github/workflows/smoke-php.yaml index e7aaf239da..263e4908c1 100644 --- a/.github/workflows/smoke-php.yaml +++ b/.github/workflows/smoke-php.yaml @@ -29,4 +29,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: php + image: mcr.microsoft.com/vscode/devcontainers/php:dev-8 user: vscode diff --git a/.github/workflows/smoke-python.yaml b/.github/workflows/smoke-python.yaml index 4c1c866db0..e6007c4a28 100644 --- a/.github/workflows/smoke-python.yaml +++ b/.github/workflows/smoke-python.yaml @@ -29,5 +29,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: python - image: python + image: mcr.microsoft.com/vscode/devcontainers/python:dev-3image: python user: vscode diff --git a/.github/workflows/smoke-ruby.yaml b/.github/workflows/smoke-ruby.yaml index 13ae76f3a4..90fddb3258 100644 --- a/.github/workflows/smoke-ruby.yaml +++ b/.github/workflows/smoke-ruby.yaml @@ -29,4 +29,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: ruby + image: mcr.microsoft.com/vscode/devcontainers/ruby:dev-3 user: vscode diff --git a/.github/workflows/smoke-rust.yaml b/.github/workflows/smoke-rust.yaml index 3b7f8feb6c..d83b78cab3 100644 --- a/.github/workflows/smoke-rust.yaml +++ b/.github/workflows/smoke-rust.yaml @@ -29,4 +29,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: rust + image: mcr.microsoft.com/vscode/devcontainers/rust:dev-bullseye user: vscode diff --git a/.github/workflows/smoke-ubuntu.yaml b/.github/workflows/smoke-ubuntu.yaml index cd3eaf1eaa..b078121e8f 100644 --- a/.github/workflows/smoke-ubuntu.yaml +++ b/.github/workflows/smoke-ubuntu.yaml @@ -29,4 +29,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: ubuntu + image: mcr.microsoft.com/vscode/devcontainers/base:dev-ubuntu user: vscode From d6f9e3c1fd24a16f64759bb1d7d669164419ae85 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 15 Jun 2022 21:40:59 +0000 Subject: [PATCH 08/23] fix python nit --- .github/workflows/smoke-python.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/smoke-python.yaml b/.github/workflows/smoke-python.yaml index e6007c4a28..962ab17d04 100644 --- a/.github/workflows/smoke-python.yaml +++ b/.github/workflows/smoke-python.yaml @@ -29,5 +29,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: python - image: mcr.microsoft.com/vscode/devcontainers/python:dev-3image: python + image: mcr.microsoft.com/vscode/devcontainers/python:dev-3 user: vscode From 2646a1df1f78d540b886ed2738c16fb5ce1b5c3c Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 15 Jun 2022 21:42:57 +0000 Subject: [PATCH 09/23] fixing image name --- .github/actions/smoke-test/test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/smoke-test/test.sh b/.github/actions/smoke-test/test.sh index a5d2604436..1a3a0ebdcb 100755 --- a/.github/actions/smoke-test/test.sh +++ b/.github/actions/smoke-test/test.sh @@ -6,12 +6,14 @@ USERNAME="$3" export DOCKER_BUILDKIT=1 set -e +# List docker images +echo "(*) Listing docker images..." docker images # Start container echo "(*) Starting container..." container_name="vscdc-test-container-$DEFINITION" -docker run -d --name ${container_name} --rm --init --privileged -v "$(pwd)/src/${DEFINITION}:/workspace" ${IMAGE}-uid /bin/sh -c 'while sleep 1000; do :; done' +docker run -d --name ${container_name} --rm --init --privileged -v "$(pwd)/src/${DEFINITION}:/workspace" ${IMAGE} /bin/sh -c 'while sleep 1000; do :; done' # Run actual test echo "(*) Running test..." From 67533875897abcd6b914f412141edba30ac99e9c Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 15 Jun 2022 21:58:28 +0000 Subject: [PATCH 10/23] change image name and on: --- .github/workflows/smoke-alpine.yaml | 4 +++- .github/workflows/smoke-anaconda.yaml | 4 +++- .github/workflows/smoke-codespaces.yaml | 15 +++++++-------- .github/workflows/smoke-cpp.yaml | 15 +++++++-------- .github/workflows/smoke-debian.yaml | 15 +++++++-------- .github/workflows/smoke-dotnet.yaml | 15 +++++++-------- .github/workflows/smoke-go.yaml | 15 +++++++-------- .github/workflows/smoke-java.yaml | 15 +++++++-------- .github/workflows/smoke-jekyll.yaml | 15 +++++++-------- .github/workflows/smoke-miniconda.yaml | 15 +++++++-------- .github/workflows/smoke-php.yaml | 15 +++++++-------- .github/workflows/smoke-python.yaml | 15 +++++++-------- .github/workflows/smoke-ruby.yaml | 15 +++++++-------- .github/workflows/smoke-rust.yaml | 15 +++++++-------- .github/workflows/smoke-ubuntu.yaml | 15 +++++++-------- 15 files changed, 97 insertions(+), 106 deletions(-) diff --git a/.github/workflows/smoke-alpine.yaml b/.github/workflows/smoke-alpine.yaml index 41a65a4410..ea39ac6ee8 100644 --- a/.github/workflows/smoke-alpine.yaml +++ b/.github/workflows/smoke-alpine.yaml @@ -9,6 +9,8 @@ on: paths: - src/alpine/** - .github/workflows/smoke-alpine.yaml + - .github/actions/** + - build/** jobs: smoke-test: name: Smoke test @@ -24,5 +26,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: alpine - image: mcr.microsoft.com/vscode/devcontainers/base:dev-alpine + image: mcr.microsoft.com/devcontainers/base:dev-alpine user: vscode diff --git a/.github/workflows/smoke-anaconda.yaml b/.github/workflows/smoke-anaconda.yaml index 1e32a56576..491bb5fc1a 100644 --- a/.github/workflows/smoke-anaconda.yaml +++ b/.github/workflows/smoke-anaconda.yaml @@ -9,6 +9,8 @@ on: paths: - src/anaconda/** - .github/workflows/smoke-anaconda.yaml + - .github/actions/** + - build/** jobs: smoke-test: name: Smoke test @@ -26,5 +28,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: anaconda - image: mcr.microsoft.com/vscode/devcontainers/anaconda:dev-3 + image: mcr.microsoft.com/devcontainers/anaconda:dev-3 user: vscode diff --git a/.github/workflows/smoke-codespaces.yaml b/.github/workflows/smoke-codespaces.yaml index 56faf5b1f4..3c00186a94 100644 --- a/.github/workflows/smoke-codespaces.yaml +++ b/.github/workflows/smoke-codespaces.yaml @@ -2,16 +2,15 @@ name: Smoke test "codespaces" build on: workflow_dispatch: - push: - branches: [main] - paths: - - 'src/codespaces/**' - pull_request: branches: - - main + - main + pull_request: paths: - - 'src/codespaces/**' + - src/codespaces/** + - .github/codespaces/smoke-codespaces.yaml + - .github/actions/** + - build/** jobs: smoke-test: name: Smoke test @@ -29,5 +28,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: codespaces - image: mcr.microsoft.com/vscode/devcontainers/universal:dev-linux + image: mcr.microsoft.com/devcontainers/universal:dev-linux user: codespace diff --git a/.github/workflows/smoke-cpp.yaml b/.github/workflows/smoke-cpp.yaml index 43dec0f0e9..b434a6017b 100644 --- a/.github/workflows/smoke-cpp.yaml +++ b/.github/workflows/smoke-cpp.yaml @@ -2,16 +2,15 @@ name: Smoke test "cpp" build on: workflow_dispatch: - push: - branches: [main] - paths: - - 'src/cpp/**' - pull_request: branches: - - main + - main + pull_request: paths: - - 'src/cpp/**' + - src/cpp/** + - .github/workflows/smoke-cpp.yaml + - .github/actions/** + - build/** jobs: smoke-test: name: Smoke test @@ -27,5 +26,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: cpp - image: mcr.microsoft.com/vscode/devcontainers/cpp:dev-debian + image: mcr.microsoft.com/devcontainers/cpp:dev-debian user: vscode diff --git a/.github/workflows/smoke-debian.yaml b/.github/workflows/smoke-debian.yaml index 10668f9d77..b69e996393 100644 --- a/.github/workflows/smoke-debian.yaml +++ b/.github/workflows/smoke-debian.yaml @@ -2,16 +2,15 @@ name: Smoke test "debian" build on: workflow_dispatch: - push: - branches: [main] - paths: - - 'src/debian/**' - pull_request: branches: - - main + - main + pull_request: paths: - - 'src/debian/**' + - src/debian/** + - .github/workflows/smoke-debian.yaml + - .github/actions/** + - build/** jobs: smoke-test: name: Smoke test @@ -27,5 +26,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: debian - image: mcr.microsoft.com/vscode/devcontainers/base:dev-debian + image: mcr.microsoft.com/devcontainers/base:dev-debian user: vscode diff --git a/.github/workflows/smoke-dotnet.yaml b/.github/workflows/smoke-dotnet.yaml index e4c73a9ce6..9ac3a05c0d 100644 --- a/.github/workflows/smoke-dotnet.yaml +++ b/.github/workflows/smoke-dotnet.yaml @@ -2,16 +2,15 @@ name: Smoke test "dotnet" build on: workflow_dispatch: - push: - branches: [main] - paths: - - 'src/dotnet/**' - pull_request: branches: - - main + - main + pull_request: paths: - - 'src/dotnet/**' + - src/dotnet/** + - .github/workflows/smoke-dotnet.yaml + - .github/actions/** + - build/** jobs: smoke-test: name: Smoke test @@ -29,5 +28,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: dotnet - image: mcr.microsoft.com/vscode/devcontainers/dotnet:dev-6.0 + image: mcr.microsoft.com/devcontainers/dotnet:dev-6.0 user: vscode diff --git a/.github/workflows/smoke-go.yaml b/.github/workflows/smoke-go.yaml index 379371e3b5..b4e227c256 100644 --- a/.github/workflows/smoke-go.yaml +++ b/.github/workflows/smoke-go.yaml @@ -2,16 +2,15 @@ name: Smoke test "go" build on: workflow_dispatch: - push: - branches: [main] - paths: - - 'src/go/**' - pull_request: branches: - - main + - main + pull_request: paths: - - 'src/go/**' + - src/go/** + - .github/workflows/smoke-go.yaml + - .github/actions/** + - build/** jobs: smoke-test: name: Smoke test @@ -29,5 +28,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: go - image: mcr.microsoft.com/vscode/devcontainers/go:dev-1 + image: mcr.microsoft.com/devcontainers/go:dev-1 user: vscode diff --git a/.github/workflows/smoke-java.yaml b/.github/workflows/smoke-java.yaml index cb3513bcea..0d7af8d8a9 100644 --- a/.github/workflows/smoke-java.yaml +++ b/.github/workflows/smoke-java.yaml @@ -2,16 +2,15 @@ name: Smoke test "java" build on: workflow_dispatch: - push: - branches: [main] - paths: - - 'src/java/**' - pull_request: branches: - - main + - main + pull_request: paths: - - 'src/java/**' + - src/java/** + - .github/workflows/smoke-java.yaml + - .github/actions/** + - build/** jobs: smoke-test: name: Smoke test @@ -29,5 +28,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: java - image: mcr.microsoft.com/vscode/devcontainers/java:dev-17 + image: mcr.microsoft.com/devcontainers/java:dev-17 user: vscode diff --git a/.github/workflows/smoke-jekyll.yaml b/.github/workflows/smoke-jekyll.yaml index cb07ae623c..2221ccff28 100644 --- a/.github/workflows/smoke-jekyll.yaml +++ b/.github/workflows/smoke-jekyll.yaml @@ -2,16 +2,15 @@ name: Smoke test "jekyll" build on: workflow_dispatch: - push: - branches: [main] - paths: - - 'src/jekyll/**' - pull_request: branches: - - main + - main + pull_request: paths: - - 'src/jekyll/**' + - src/jekyll/** + - .github/workflows/smoke-jekyll.yaml + - .github/actions/** + - build/** jobs: smoke-test: name: Smoke test @@ -27,5 +26,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: jekyll - image: mcr.microsoft.com/vscode/devcontainers/jekyll:dev-bullseye + image: mcr.microsoft.com/devcontainers/jekyll:dev-bullseye user: vscode diff --git a/.github/workflows/smoke-miniconda.yaml b/.github/workflows/smoke-miniconda.yaml index 31eae507d1..223067e9b3 100644 --- a/.github/workflows/smoke-miniconda.yaml +++ b/.github/workflows/smoke-miniconda.yaml @@ -2,16 +2,15 @@ name: Smoke test "miniconda" build on: workflow_dispatch: - push: - branches: [main] - paths: - - 'src/miniconda/**' - pull_request: branches: - - main + - main + pull_request: paths: - - 'src/miniconda/**' + - src/miniconda/** + - .github/workflows/smoke-miniconda.yaml + - .github/actions/** + - build/** jobs: smoke-test: name: Smoke test @@ -29,5 +28,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: miniconda - image: mcr.microsoft.com/vscode/devcontainers/miniconda:dev-3 + image: mcr.microsoft.com/devcontainers/miniconda:dev-3 user: vscode diff --git a/.github/workflows/smoke-php.yaml b/.github/workflows/smoke-php.yaml index 263e4908c1..6c07d94991 100644 --- a/.github/workflows/smoke-php.yaml +++ b/.github/workflows/smoke-php.yaml @@ -2,16 +2,15 @@ name: Smoke test "php" build on: workflow_dispatch: - push: - branches: [main] - paths: - - 'src/php/**' - pull_request: branches: - - main + - main + pull_request: paths: - - 'src/php/**' + - src/php/** + - .github/workflows/smoke-php.yaml + - .github/actions/** + - build/** jobs: smoke-test: name: Smoke test @@ -29,5 +28,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: php - image: mcr.microsoft.com/vscode/devcontainers/php:dev-8 + image: mcr.microsoft.com/devcontainers/php:dev-8 user: vscode diff --git a/.github/workflows/smoke-python.yaml b/.github/workflows/smoke-python.yaml index 962ab17d04..f0b46b277b 100644 --- a/.github/workflows/smoke-python.yaml +++ b/.github/workflows/smoke-python.yaml @@ -2,16 +2,15 @@ name: Smoke test "python" build on: workflow_dispatch: - push: - branches: [main] - paths: - - 'src/python/**' - pull_request: branches: - - main + - main + pull_request: paths: - - 'src/python/**' + - src/python/** + - .github/workflows/smoke-python.yaml + - .github/actions/** + - build/** jobs: smoke-test: name: Smoke test @@ -29,5 +28,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: python - image: mcr.microsoft.com/vscode/devcontainers/python:dev-3 + image: mcr.microsoft.com/devcontainers/python:dev-3 user: vscode diff --git a/.github/workflows/smoke-ruby.yaml b/.github/workflows/smoke-ruby.yaml index 90fddb3258..ba739e916b 100644 --- a/.github/workflows/smoke-ruby.yaml +++ b/.github/workflows/smoke-ruby.yaml @@ -2,16 +2,15 @@ name: Smoke test "ruby" build on: workflow_dispatch: - push: - branches: [main] - paths: - - 'src/ruby/**' - pull_request: branches: - - main + - main + pull_request: paths: - - 'src/ruby/**' + - src/ruby/** + - .github/workflows/smoke-ruby.yaml + - .github/actions/** + - build/** jobs: smoke-test: name: Smoke test @@ -29,5 +28,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: ruby - image: mcr.microsoft.com/vscode/devcontainers/ruby:dev-3 + image: mcr.microsoft.com/devcontainers/ruby:dev-3 user: vscode diff --git a/.github/workflows/smoke-rust.yaml b/.github/workflows/smoke-rust.yaml index d83b78cab3..2a16e299d7 100644 --- a/.github/workflows/smoke-rust.yaml +++ b/.github/workflows/smoke-rust.yaml @@ -2,16 +2,15 @@ name: Smoke test "rust" build on: workflow_dispatch: - push: - branches: [main] - paths: - - 'src/rust/**' - pull_request: branches: - - main + - main + pull_request: paths: - - 'src/rust/**' + - src/rust/** + - .github/workflows/smoke-rust.yaml + - .github/actions/** + - build/** jobs: smoke-test: name: Smoke test @@ -29,5 +28,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: rust - image: mcr.microsoft.com/vscode/devcontainers/rust:dev-bullseye + image: mcr.microsoft.com/devcontainers/rust:dev-bullseye user: vscode diff --git a/.github/workflows/smoke-ubuntu.yaml b/.github/workflows/smoke-ubuntu.yaml index b078121e8f..731d439cbe 100644 --- a/.github/workflows/smoke-ubuntu.yaml +++ b/.github/workflows/smoke-ubuntu.yaml @@ -2,16 +2,15 @@ name: Smoke test "ubuntu" build on: workflow_dispatch: - push: - branches: [main] - paths: - - 'src/ubuntu/**' - pull_request: branches: - - main + - main + pull_request: paths: - - 'src/ubuntu/**' + - src/ubuntu/** + - .github/workflows/smoke-ubuntu.yaml + - .github/actions/** + - build/** jobs: smoke-test: name: Smoke test @@ -29,5 +28,5 @@ jobs: uses: ./.github/actions/smoke-test with: definition: ubuntu - image: mcr.microsoft.com/vscode/devcontainers/base:dev-ubuntu + image: mcr.microsoft.com/devcontainers/base:dev-ubuntu user: vscode From ec73bc01b1131959e8e7104e3ef289a878b53d11 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 15 Jun 2022 22:28:40 +0000 Subject: [PATCH 11/23] fix githubrepo name --- .github/actions/smoke-test/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/smoke-test/build.sh b/.github/actions/smoke-test/build.sh index 6850171a3b..fde6072b5d 100755 --- a/.github/actions/smoke-test/build.sh +++ b/.github/actions/smoke-test/build.sh @@ -19,7 +19,7 @@ chmod +x build/vscdc build/vscdc push ${DEFINITION} \ --no-push \ --release dev \ - --github-repo "devcontainers" \ + --github-repo "devcontainers/images" \ --registry "mcr.microsoft.com" \ --registry-path "devcontainers" \ --stub-registry "mcr.microsoft.com" \ From 36d8f503cb82c571b98887afd686a58fcc8857f0 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 15 Jun 2022 22:43:25 +0000 Subject: [PATCH 12/23] uid gid changes --- .github/actions/smoke-test/test.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/actions/smoke-test/test.sh b/.github/actions/smoke-test/test.sh index 1a3a0ebdcb..6ef04a47c2 100755 --- a/.github/actions/smoke-test/test.sh +++ b/.github/actions/smoke-test/test.sh @@ -10,10 +10,25 @@ set -e echo "(*) Listing docker images..." docker images + +# Update UID/GID for user in container - Actions uses different UID/GID than container +# which causes bind mounts to be read only and cause certain write tests to fail +# The dev container CLI handles this automatically but we're not using it. +local_uid=$(id -u) +local_gid=$(id -g) +echo "(*) Updating container user UID/GID..." +echo -e "FROM ${IMAGE}\n \ + RUN export sudo_cmd="" \ + && if [ "$(id -u)" != "0" ]; then export sudo_cmd=sudo; fi \ + && \${sudo_cmd} groupmod -g ${local_gid} ${USERNAME} \ + && \${sudo_cmd} usermod -u ${local_uid} -g ${local_gid} ${USERNAME}" > uid.Dockerfile +cat uid.Dockerfile +docker build -t ${IMAGE}-uid -f uid.Dockerfile . + # Start container echo "(*) Starting container..." container_name="vscdc-test-container-$DEFINITION" -docker run -d --name ${container_name} --rm --init --privileged -v "$(pwd)/src/${DEFINITION}:/workspace" ${IMAGE} /bin/sh -c 'while sleep 1000; do :; done' +docker run -d --name ${container_name} --rm --init --privileged -v "$(pwd)/src/${DEFINITION}:/workspace" ${IMAGE}-uid /bin/sh -c 'while sleep 1000; do :; done' # Run actual test echo "(*) Running test..." From 06dc62a5519474c371a3c1ebffabf2d3adc9b16a Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 15 Jun 2022 16:51:18 -0700 Subject: [PATCH 13/23] debug test --- src/codespaces/test-project/test.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/codespaces/test-project/test.sh b/src/codespaces/test-project/test.sh index fa01859ff6..673a0b26a3 100755 --- a/src/codespaces/test-project/test.sh +++ b/src/codespaces/test-project/test.sh @@ -111,5 +111,13 @@ check "zsh" zsh --version yarn check "run-puppeteer" node puppeteer.js +cd /usr/local +ls -la + +cd /usr/local/rvm/gems/ruby-3.1.2/gems/jekyll-4.2.2/ +ls -la + +cd /usr/local/rvm/gems/ruby-3.1.2/gems/ +ls -la # Report result reportResults From 4bc1ccacb0a36b312ce84c4adb21dda501392f68 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 15 Jun 2022 16:51:52 -0700 Subject: [PATCH 14/23] jekyll: use gem --- src/codespaces/local-features/jekyll/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/codespaces/local-features/jekyll/install.sh b/src/codespaces/local-features/jekyll/install.sh index 29f402472e..8e06d30e42 100644 --- a/src/codespaces/local-features/jekyll/install.sh +++ b/src/codespaces/local-features/jekyll/install.sh @@ -53,9 +53,9 @@ fi if ! jekyll --version > /dev/null ; then echo "Installing Jekyll..." if [ "${VERSION}" = "latest" ]; then - PATH="/usr/local/rvm/rubies/default/bin:${PATH}" /usr/local/rvm/rubies/default/bin/gem install jekyll + gem install jekyll else - PATH="/usr/local/rvm/rubies/default/bin:${PATH}" /usr/local/rvm/rubies/default/bin/gem install jekyll -v "${VERSION}" + gem install jekyll -v "${VERSION}" fi fi From d2cd2640beee72246ba23d7b0ffb32eb5aa050fd Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 15 Jun 2022 17:02:53 -0700 Subject: [PATCH 15/23] fix uid gid --- .github/actions/smoke-test/test.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/actions/smoke-test/test.sh b/.github/actions/smoke-test/test.sh index 6ef04a47c2..c54cade349 100755 --- a/.github/actions/smoke-test/test.sh +++ b/.github/actions/smoke-test/test.sh @@ -16,6 +16,11 @@ docker images # The dev container CLI handles this automatically but we're not using it. local_uid=$(id -u) local_gid=$(id -g) +if [ "${DEFINITION}" = "codespaces" ]; then + local_uid=1000 + local_gid=1000 +fi + echo "(*) Updating container user UID/GID..." echo -e "FROM ${IMAGE}\n \ RUN export sudo_cmd="" \ From 506fe142b5df30c3c5a79b7bd9cece82d37a2515 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Thu, 16 Jun 2022 16:56:15 +0000 Subject: [PATCH 16/23] fix path --- build/src/utils/image-content-extractor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/src/utils/image-content-extractor.js b/build/src/utils/image-content-extractor.js index 336966bf2c..1acf4d9e8f 100644 --- a/build/src/utils/image-content-extractor.js +++ b/build/src/utils/image-content-extractor.js @@ -501,7 +501,7 @@ async function getGoPackageInfo(imageTagOrContainerName, packages) { console.log(`(*) Gathering information about go modules and packages...`); const componentList = []; - const packageInstallOutput = await getCommandOutputFromContainer(imageTagOrContainerName, "cat /usr/local/etc/devcontainers/go.log"); + const packageInstallOutput = await getCommandOutputFromContainer(imageTagOrContainerName, "cat /usr/local/etc/vscode-dev-containers/go.log"); for(let package in packages) { if (typeof package === 'string') { const versionCommand = packages[package]; From 8adfc6d375df2352fbc449b087528029843e5624 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Thu, 16 Jun 2022 17:03:03 +0000 Subject: [PATCH 17/23] change to manifest --- .github/workflows/version-history.yml | 4 ++-- build/.vscode/settings.json | 2 +- build/README.md | 10 ++++----- build/src/image-info.js | 26 +++++++++++----------- build/src/utils/image-content-extractor.js | 2 +- build/vscdc | 10 ++++----- cgmanifest.json => manifest.json | 0 7 files changed, 27 insertions(+), 27 deletions(-) rename cgmanifest.json => manifest.json (100%) diff --git a/.github/workflows/version-history.yml b/.github/workflows/version-history.yml index 9318b70431..abef8410f3 100644 --- a/.github/workflows/version-history.yml +++ b/.github/workflows/version-history.yml @@ -8,7 +8,7 @@ on: required: true default: 'main' cg: - description: 'Generate cgmanifest.json' + description: 'Generate manifest.json' required: true default: 'false' push: @@ -75,7 +75,7 @@ jobs: cp -r "$GITHUB_WORKSPACE/ref/build" "$GITHUB_WORKSPACE/release/" yarn install - # Pull images and update cgmanifest.json, but output back to ref + # Pull images and update manifest.json, but output back to ref build/vscdc info --no-build \ --markdown \ --prune \ diff --git a/build/.vscode/settings.json b/build/.vscode/settings.json index c344465b39..0b83fd6f5a 100644 --- a/build/.vscode/settings.json +++ b/build/.vscode/settings.json @@ -3,7 +3,7 @@ "cSpell.words": [ "Distro", "Dockerfiles", - "cgmanifest", + "manifest", "copyfiles", "devcontainer", "devcontainers", diff --git a/build/README.md b/build/README.md index 7b1173b4a1..ef20cb495f 100644 --- a/build/README.md +++ b/build/README.md @@ -8,7 +8,7 @@ The Node.js based build CLI (`build/vsdc`) has commands to: 1. Build and push to a repository: `build/vsdc push` 2. Build, push, and npm package assets that are modified as described above: `build/vsdc package` -3. Generate cgmanifest.json and history markdown files: `build/vsdc cg`, `build/vsdc info` +3. Generate manifest.json and history markdown files: `build/vsdc cg`, `build/vsdc info` 4. Update all script source URLs in Dockerfiles to a tag or branch: `build/vsdc update-script-sources` 5. Overwrite scripts in the `.devcontainer/library-scripts` folder with the most recent copy from the `scripts-library` folder: `build/vscdc copy-library-scripts` @@ -16,11 +16,11 @@ Run with the `--help` option to see inputs. This CLI is used in the GitHub Actions workflows in this repository. -- `push-dev.yml`: Pushes a "dev" tag for each image to be generated in this repository and fires repository dispatch to trigger cgmanifest.json generation, and attaches an npm package with the definitions to the actions run. +- `push-dev.yml`: Pushes a "dev" tag for each image to be generated in this repository and fires repository dispatch to trigger manifest.json generation, and attaches an npm package with the definitions to the actions run. - `push-and-package.yml`: Triggers when a release tag is pushed (`vX.Y.Z`). Builds and pushes a release version of the images, creates a release, and attaches an npm package with the definitions to the release. Note that this update the tag with source files that contain a SHA hash for script sources. You may need to run `git fetch --tags --force` locally after it runs. - `push-again.yml`: A manually triggered workflow that can be used to push an updated version of an image for an existing release. This should only be used in cases where the image push to the registry only partially succeeded (e.g. `linux/amd64` was pushed, but a connection error happened when pushing `linux/arm64` for the same image.) - `smoke-*.yaml` (using the `smoke-test` action in this repository) - Runs a build without pushing and executes `test-project/test.sh` (if present) inside the container to verify that there are no breaking changes to the image when the repository contents are updated. -- `version-history.yml`: Listens for workflow dispatch events to trigger cgmanifest.json and history markdown generation. +- `version-history.yml`: Listens for workflow dispatch events to trigger manifest.json and history markdown generation. ## Setting up a container to be built @@ -58,7 +58,7 @@ Once you have your build configuration setup, you can use the `vscdc` CLI to tes docker run -it --init --privileged --rm mcr.microsoft.com/devcontainers/:dev- bash ``` -3. Finally, test cgmanifest/markdown generation by running: +3. Finally, test manifest/markdown generation by running: ```bash build/vscdc cg --registry mcr.microsoft.com --registry-path devcontainers --release main @@ -381,7 +381,7 @@ Following this is a list of libraries installed in the image by its Dockerfile. #### `dependencies.apt`, `dependencies.apk` -These two properties are arrays of either strings or objects that reference apt or apk package names. Given most installed packages are there simply for visibility because they come with the distro, these are not tracked in `cgmanifest.json` by default. When something comes from a 3rd party repository, the object syntax can be used to set `"cgIgnore": false`. An `annotation` property can also be used to for a description that should appear in history markdown files. +These two properties are arrays of either strings or objects that reference apt or apk package names. Given most installed packages are there simply for visibility because they come with the distro, these are not tracked in `manifest.json` by default. When something comes from a 3rd party repository, the object syntax can be used to set `"cgIgnore": false`. An `annotation` property can also be used to for a description that should appear in history markdown files. For example: diff --git a/build/src/image-info.js b/build/src/image-info.js index e7682edb41..22d5f1ded3 100644 --- a/build/src/image-info.js +++ b/build/src/image-info.js @@ -12,19 +12,19 @@ let releaseNotesHeaderTemplate, releaseNotesVariantPartTemplate; handlebars.registerHelper('anchor', (value) => value.toLowerCase().replace(/[^\w\- ]/g, '').replace(/ /g, '-')); async function generateImageInformationFiles(repo, release, registry, registryPath, - stubRegistry, stubRegistryPath, buildFirst, pruneBetweenDefinitions, generateCgManifest, generateMarkdown, overwrite, outputPath, definitionId) { + stubRegistry, stubRegistryPath, buildFirst, pruneBetweenDefinitions, generateManifest, generateMarkdown, overwrite, outputPath, definitionId) { // Load config files await configUtils.loadConfig(); const alreadyRegistered = {}; - const cgManifest = { + const manifest = { "Registrations": [], "Version": 1 } - // cgmanifest file path and whether it exists - const cgManifestPath = path.join(outputPath, 'cgmanifest.json'); - const cgManifestExists = await asyncUtils.exists(cgManifestPath); + // manifest file path and whether it exists + const manifestPath = path.join(outputPath, 'manifest.json'); + const manifestExists = await asyncUtils.exists(manifestPath); console.log('(*) Generating image information files...'); const definitions = definitionId ? [definitionId] : configUtils.getSortedDefinitionBuildList(); @@ -39,7 +39,7 @@ async function generateImageInformationFiles(repo, release, registry, registryPa // Skip if not overwriting and all files exist if(! overwrite && (! generateMarkdown || markdownExists) && - (! generateCgManifest || cgManifestExists)) { + (! generateManifest || manifestExists)) { console.log(`(*) Skipping ${currentDefinitionId}. Not in overwrite mode and content already exists.`); return; } @@ -55,8 +55,8 @@ async function generateImageInformationFiles(repo, release, registry, registryPa } // Add component registrations if we're using them - if (generateCgManifest) { - cgManifest.Registrations = cgManifest.Registrations.concat(definitionInfo.registrations); + if (generateManifest) { + manifest.Registrations = manifest.Registrations.concat(definitionInfo.registrations); } // Prune images if setting enabled if (pruneBetweenDefinitions) { @@ -64,12 +64,12 @@ async function generateImageInformationFiles(repo, release, registry, registryPa } }); - // Write final cgmanifest.json file if needed - if(generateCgManifest && (overwrite || ! cgManifestExists)) { - console.log('(*) Writing cgmanifest.json...'); + // Write final manifest.json file if needed + if(generateManifest && (overwrite || ! manifestExists)) { + console.log('(*) Writing manifest.json...'); await asyncUtils.writeFile( - path.join(outputPath, 'cgmanifest.json'), - JSON.stringify(cgManifest, undefined, 4)); + path.join(outputPath, 'manifest.json'), + JSON.stringify(manifest, undefined, 4)); } console.log('(*) Done!'); } diff --git a/build/src/utils/image-content-extractor.js b/build/src/utils/image-content-extractor.js index 1acf4d9e8f..1d1ea61226 100644 --- a/build/src/utils/image-content-extractor.js +++ b/build/src/utils/image-content-extractor.js @@ -179,7 +179,7 @@ async function getLinuxPackageInfo(imageTagOrContainerName, packageList, linuxDi return componentList; } -// Gets a package pool URL out of a download URL - Needed for registering in cgmanifest.json +// Gets a package pool URL out of a download URL - Needed for registering in manifest.json function getPoolUrlFromPackageVersionListOutput(packageUriCommandOutput, config, package, version) { // Handle regex reserved charters in regex strings and that ":" is treaded as "1%3a" on Debian/Ubuntu const sanitizedPackage = package.replace(/\+/g, '\\+').replace(/\./g, '\\.'); diff --git a/build/vscdc b/build/vscdc index 1013610c60..8f986723fa 100755 --- a/build/vscdc +++ b/build/vscdc @@ -142,7 +142,7 @@ require('yargs') } }) }, pushCommand) - .command('cg [devcontainer]', 'generate cgmanifest.json', (yargs) => { + .command('cg [devcontainer]', 'generate manifest.json', (yargs) => { yargs .positional('devcontainer', { describe: 'limits manifest generation to single definition', @@ -186,7 +186,7 @@ require('yargs') default: false }, 'cg': { - describe: 'whether to generate cgmanifest.json', + describe: 'whether to generate manifest.json', type: 'boolean', default: true }, @@ -200,7 +200,7 @@ require('yargs') default: path.resolve(__dirname, '..', configUtils.getConfig('informationFileOutputPath', '.')) }, 'overwrite': { - describe: 'whether to overwrite cgmanifest.json or markdown files', + describe: 'whether to overwrite manifest.json or markdown files', type: 'boolean', default: true } @@ -245,7 +245,7 @@ require('yargs') default: true }, 'cg': { - describe: 'whether to generate cgmanifest.json', + describe: 'whether to generate manifest.json', type: 'boolean', default: false }, @@ -264,7 +264,7 @@ require('yargs') default: path.resolve(__dirname, '..', configUtils.getConfig('informationFileOutputPath', '.')) }, 'overwrite': { - describe: 'whether to overwrite cgmanifest.json or markdown files', + describe: 'whether to overwrite manifest.json or markdown files', type: 'boolean', default: false } diff --git a/cgmanifest.json b/manifest.json similarity index 100% rename from cgmanifest.json rename to manifest.json From cddd99d08df203306a904cdb4e97553e2dcdba51 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Thu, 16 Jun 2022 17:36:16 +0000 Subject: [PATCH 18/23] codespaces: fix path manifest --- src/codespaces/manifest.json | 45 +++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/src/codespaces/manifest.json b/src/codespaces/manifest.json index a2d316f8e2..f93c2cba42 100644 --- a/src/codespaces/manifest.json +++ b/src/codespaces/manifest.json @@ -10,7 +10,7 @@ ] }, "dependencies": { - "annotation": "This document describes the base contents of the default GitHub Codespaces dev container image. Note that this image also includes detection logic to dynamically install additional language / runtime versions based on your repository's contents. Dynamically installed content can be found in sub-folders under `/opt`.", + "annotation": "This document describes the base contents of the default GitHub Codespaces dev container image. Note that this image also includes detection logic to dynamically install additional language / runtime versions based on your repository's contents. Dynamically installed content can be found in sub-folders under `/usr/local`.", "image": "mcr.microsoft.com/oryx/build:vso-focal-20210728.1", "imageLink": "https://github.com/microsoft/oryx", "apt": [{ @@ -83,8 +83,8 @@ ], "git": { "Oh My Zsh!": "/home/codespace/.oh-my-zsh", - "nvm": "/home/codespace/.nvm", - "nvs": "/home/codespace/.nvs", + "nvm": "/usr/local/share/nvm", + "nvs": "/usr/local/nvs", "rbenv": "/usr/local/share/rbenv", "ruby-build": "/usr/local/share/ruby-build" }, @@ -106,14 +106,14 @@ }, "other": { "git": { - "path": "/usr/local" + "path": "/usr/bin" }, "Xdebug": { - "path": "/opt/php/lts" + "path": "/usr/local/php/current" }, "Composer": { "cgIgnore": true, - "path": "/opt/php-composer" + "path": "/usr/local/php/current/bin" }, "kubectl": null, "Helm": null, @@ -123,13 +123,16 @@ "GitHub CLI": null, "yarn": { "cgIgnore": true, - "path": "/opt/yarn" + "path": "/usr/bin" }, "Maven": { "cgIgnore": true, - "path": "/opt/maven" + "path": "/usr/local/sdkman/candidates/maven/current/bin" + }, + "Gradle": { + "cgIgnore": true, + "path": "/usr/local/sdkman/candidates/gradle/current/bin" }, - "Gradle": null, "Docker (Moby) CLI & Engine": { "cgIgnore": true } @@ -137,33 +140,33 @@ "languages": { "Node.js": { "cgIgnore": true, - "versionCommand": "ls /opt/nodejs | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'", - "path": "/opt/nodejs/<version>" + "versionCommand": "ls /usr/local/share/nvm/versions/node | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'", + "path": "/usr/local/share/nvm/versions/node/<version>" }, "Python": { "cgIgnore": true, - "versionCommand": "ls /opt/python | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'", - "path": "/opt/python/<version>" + "versionCommand": "ls /usr/local/python | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'", + "path": "/usr/local/python/<version>" }, "Java": { "cgIgnore": true, - "versionCommand": "ls /opt/java | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'", - "path": "/opt/java/<version>" + "versionCommand": "ls /usr/local/sdkman/candidates/java | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'", + "path": "/usr/local/sdkman/candidates/java/<version>" }, ".NET": { "cgIgnore": true, - "versionCommand": "ls /opt/dotnet | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'", - "path": "/home/codespaces/.dotnet
/opt/dotnet" + "versionCommand": "ls /usr/local/dotnet | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'", + "path": "/usr/local/dotnet" }, "Ruby": { "cgIgnore": true, - "versionCommand": "ls /opt/ruby | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'", - "path": "/opt/ruby/<version>" + "versionCommand": "ls /usr/local/rvm/gems/default/bin/ | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'", + "path": "/usr/local/rvm/gems/default/bin//<version>" }, "PHP": { "cgIgnore": true, - "versionCommand": "ls /opt/php | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'", - "path": "/opt/php/<version>" + "versionCommand": "ls /usr/local/php | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'", + "path": "/usr/local/php/<version>" }, "GCC": { "cgIgnore": true From f34e1dfae75475f16a3732c7152cdcb9077c6984 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Thu, 16 Jun 2022 18:13:21 +0000 Subject: [PATCH 19/23] fix ruby path: manifest --- src/codespaces/manifest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/codespaces/manifest.json b/src/codespaces/manifest.json index f93c2cba42..d90a537d74 100644 --- a/src/codespaces/manifest.json +++ b/src/codespaces/manifest.json @@ -160,8 +160,8 @@ }, "Ruby": { "cgIgnore": true, - "versionCommand": "ls /usr/local/rvm/gems/default/bin/ | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'", - "path": "/usr/local/rvm/gems/default/bin//<version>" + "versionCommand": "ls /usr/local/rvm/rubies | grep -oE '[0-9]+\\.[0-9]+\\.[0-9]+'", + "path": "/usr/local/rvm/rubies/<version>" }, "PHP": { "cgIgnore": true, From 80f43f0a06ddd7c7de69b8e335ef0fe02bdb3e90 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Thu, 16 Jun 2022 20:48:52 +0000 Subject: [PATCH 20/23] fix paging --- .github/actions/smoke-test/test.sh | 4 ---- .github/workflows/push-and-package.yml | 4 ++-- .github/workflows/push-dev.yml | 4 ++-- src/codespaces/test-project/test.sh | 9 --------- src/miniconda/Dockerfile | 2 -- 5 files changed, 4 insertions(+), 19 deletions(-) diff --git a/.github/actions/smoke-test/test.sh b/.github/actions/smoke-test/test.sh index c54cade349..6df4b4afe4 100755 --- a/.github/actions/smoke-test/test.sh +++ b/.github/actions/smoke-test/test.sh @@ -16,10 +16,6 @@ docker images # The dev container CLI handles this automatically but we're not using it. local_uid=$(id -u) local_gid=$(id -g) -if [ "${DEFINITION}" = "codespaces" ]; then - local_uid=1000 - local_gid=1000 -fi echo "(*) Updating container user UID/GID..." echo -e "FROM ${IMAGE}\n \ diff --git a/.github/workflows/push-and-package.yml b/.github/workflows/push-and-package.yml index afa5095e95..20c89f68d5 100644 --- a/.github/workflows/push-and-package.yml +++ b/.github/workflows/push-and-package.yml @@ -10,8 +10,8 @@ jobs: name: Build and push images strategy: matrix: - page: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - page-total: [10] + page: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + page-total: [11] fail-fast: false runs-on: ubuntu-latest steps: diff --git a/.github/workflows/push-dev.yml b/.github/workflows/push-dev.yml index 844e2488e2..293f282fe2 100644 --- a/.github/workflows/push-dev.yml +++ b/.github/workflows/push-dev.yml @@ -11,8 +11,8 @@ jobs: if: "!contains(github.event.head_commit.message, 'Automated update') && !contains(github.event.head_commit.message, 'CI ignore')" strategy: matrix: - page: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - page-total: [10] + page: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + page-total: [11] fail-fast: true runs-on: ubuntu-latest steps: diff --git a/src/codespaces/test-project/test.sh b/src/codespaces/test-project/test.sh index 673a0b26a3..a623849730 100755 --- a/src/codespaces/test-project/test.sh +++ b/src/codespaces/test-project/test.sh @@ -9,15 +9,6 @@ checkCommon # Check Oryx check "oryx" oryx --version -# Check Jekyll dynamic install -mkdir jekyll-test -cd jekyll-test -touch _config.yml -check "oryx-build-jekyll" oryx build --apptype static-sites --manifest-dir /tmp -check "jekyll" gem list jekyll -cd .. -rm -rf jekyll-test - # Check .NET check "dotnet" dotnet --list-sdks check "oryx-install-dotnet-2.1" oryx prep --skip-detection --platforms-and-versions dotnet=2.1.12 diff --git a/src/miniconda/Dockerfile b/src/miniconda/Dockerfile index 118a21975d..4868edfce4 100644 --- a/src/miniconda/Dockerfile +++ b/src/miniconda/Dockerfile @@ -11,7 +11,6 @@ FROM mcr.microsoft.com/vscode/devcontainers/base:0-bullseye COPY --from=upstream /opt /opt/ # Copy library scripts to execute -# COPY .devcontainer/library-scripts/*.sh .devcontainer/add-notice.sh .devcontainer/library-scripts/*.env /tmp/library-scripts/ COPY add-notice.sh /tmp/library-scripts/ # Setup conda to mirror contents from https://github.com/ContinuumIO/docker-images/blob/master/miniconda3/debian/Dockerfile @@ -45,7 +44,6 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # Copy environment.yml (if found) to a temp locaition so we update the environment. Also # copy "noop.txt" so the COPY instruction does not fail if no environment.yml exists. -# COPY environment.yml* .devcontainer/noop.txt /tmp/conda-tmp/ COPY environment.yml* noop.txt /tmp/conda-tmp/ RUN if [ -f "/tmp/conda-tmp/environment.yml" ]; then umask 0002 && /opt/conda/bin/conda env update -n base -f /tmp/conda-tmp/environment.yml; fi \ && rm -rf /tmp/conda-tmp From 0a9263d1abc906c144b3ffb8817c0cfc991457f4 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Thu, 16 Jun 2022 21:45:06 +0000 Subject: [PATCH 21/23] fix null error --- build/src/utils/image-content-extractor.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/build/src/utils/image-content-extractor.js b/build/src/utils/image-content-extractor.js index 1d1ea61226..df7e23416d 100644 --- a/build/src/utils/image-content-extractor.js +++ b/build/src/utils/image-content-extractor.js @@ -428,11 +428,17 @@ async function getGemPackageInfo(imageTagOrContainerName, packageList) { console.log(`(*) Gathering information about gems...`); const gemListOutput = await getCommandOutputFromContainer(imageTagOrContainerName, "bash -l -c 'set -e && gem list -d --local' 2>/dev/null"); return packageList.map((gem) => { + console.log(gemListOutput); const gemVersionCaptureGroup = new RegExp(`^${gem}\\s\\(([^\\),]+)`,'m').exec(gemListOutput); - const gemVersion = gemVersionCaptureGroup[1]; - return { - name: gem, - version: gemVersion + console.log(gemVersionCaptureGroup) + if (gemVersionCaptureGroup !== null && gemVersionCaptureGroup[1] !== null) { + const gemVersion = gemVersionCaptureGroup[1]; + return { + name: gem, + version: gemVersion + } + } else { + return {}; } }); } From fea14038674f7c6efe6420e448b500000b4c22b1 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Thu, 16 Jun 2022 22:25:29 +0000 Subject: [PATCH 22/23] remove console statements --- build/src/utils/image-content-extractor.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/build/src/utils/image-content-extractor.js b/build/src/utils/image-content-extractor.js index df7e23416d..97ca775b42 100644 --- a/build/src/utils/image-content-extractor.js +++ b/build/src/utils/image-content-extractor.js @@ -428,9 +428,7 @@ async function getGemPackageInfo(imageTagOrContainerName, packageList) { console.log(`(*) Gathering information about gems...`); const gemListOutput = await getCommandOutputFromContainer(imageTagOrContainerName, "bash -l -c 'set -e && gem list -d --local' 2>/dev/null"); return packageList.map((gem) => { - console.log(gemListOutput); const gemVersionCaptureGroup = new RegExp(`^${gem}\\s\\(([^\\),]+)`,'m').exec(gemListOutput); - console.log(gemVersionCaptureGroup) if (gemVersionCaptureGroup !== null && gemVersionCaptureGroup[1] !== null) { const gemVersion = gemVersionCaptureGroup[1]; return { From c9332021a937d3792b7d557eef62db2dfab8921d Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Thu, 16 Jun 2022 22:27:28 +0000 Subject: [PATCH 23/23] remove debug statements --- src/codespaces/test-project/test.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/codespaces/test-project/test.sh b/src/codespaces/test-project/test.sh index a623849730..6f4f5984f4 100755 --- a/src/codespaces/test-project/test.sh +++ b/src/codespaces/test-project/test.sh @@ -102,13 +102,5 @@ check "zsh" zsh --version yarn check "run-puppeteer" node puppeteer.js -cd /usr/local -ls -la - -cd /usr/local/rvm/gems/ruby-3.1.2/gems/jekyll-4.2.2/ -ls -la - -cd /usr/local/rvm/gems/ruby-3.1.2/gems/ -ls -la # Report result reportResults