From b0bce854ed0ce0c12453e60d1f81a862566fb1f7 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 18 Jan 2024 12:15:34 -0600 Subject: [PATCH 01/59] Build RAPIDS. --- .github/workflows/test.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2a6653a30..90c1eb147 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -145,3 +145,12 @@ jobs: features: "${{ toJSON(matrix.features) }}" container_env: "${{ toJSON(matrix.env) }}" repo: "${{ vars.DOCKERHUB_REPOSITORY || github.repository }}" + + build-rapids: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.02 + with: + build_command: | + sccache -z; + build-all --verbose; + sccache -s; From 13c0c111054a916e847aab39806cfa83157d772f Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 18 Jan 2024 12:27:48 -0600 Subject: [PATCH 02/59] Update .github/workflows/test.yml Co-authored-by: Paul Taylor <178183+trxcllnt@users.noreply.github.com> --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 90c1eb147..3233ab951 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,6 +147,7 @@ jobs: repo: "${{ vars.DOCKERHUB_REPOSITORY || github.repository }}" build-rapids: + if: needs.check-event.outputs.ok == 'true' secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.02 with: From f506d4961134d638b1cd99d2c3d176288b51be19 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 18 Jan 2024 12:27:52 -0600 Subject: [PATCH 03/59] Update .github/workflows/test.yml Co-authored-by: Paul Taylor <178183+trxcllnt@users.noreply.github.com> --- .github/workflows/test.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3233ab951..2479d3b92 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -153,5 +153,12 @@ jobs: with: build_command: | sccache -z; - build-all --verbose; + clone-all --verbose; + build-all --verbose \ + -DBUILD_TESTS=ON \ + -DBUILD_BENCHMARKS=ON \ + -DBUILD_ANN_BENCH=ON \ + -DBUILD_PRIMS_BENCH=ON \ + -DNVBench_ENABLE_CUPTI=OFF \ + -j$(nproc --ignore=1); sccache -s; From 49a4ae792b98d1de401206e5a46fb9c054c87ebc Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 18 Jan 2024 12:44:49 -0600 Subject: [PATCH 04/59] Update .github/workflows/test.yml --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2479d3b92..660632a1a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -148,6 +148,7 @@ jobs: build-rapids: if: needs.check-event.outputs.ok == 'true' + needs: check-event secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.02 with: From 43ca157bd7111b083f58476fa3279ded71b596ab Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Thu, 18 Jan 2024 10:46:07 -0800 Subject: [PATCH 05/59] Update .github/workflows/test.yml --- .github/workflows/test.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 660632a1a..c076a69c9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -158,8 +158,7 @@ jobs: build-all --verbose \ -DBUILD_TESTS=ON \ -DBUILD_BENCHMARKS=ON \ - -DBUILD_ANN_BENCH=ON \ - -DBUILD_PRIMS_BENCH=ON \ - -DNVBench_ENABLE_CUPTI=OFF \ - -j$(nproc --ignore=1); + -DBUILD_ANN_BENCH=ON \ + -DBUILD_PRIMS_BENCH=ON \ + -j$(nproc --ignore=1); sccache -s; From fc92a2d4e5e489869c004427a5a690d4cb87dba0 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 10:57:54 -0800 Subject: [PATCH 06/59] make features symlinks before launching devcontainers --- .devcontainer/cuda11.8-conda/devcontainer.json | 2 +- .devcontainer/cuda11.8-pip/devcontainer.json | 2 +- .devcontainer/cuda12.0-conda/devcontainer.json | 2 +- .devcontainer/cuda12.0-pip/devcontainer.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json index 5aa82c5f4..7dfef8502 100644 --- a/.devcontainer/cuda11.8-conda/devcontainer.json +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -15,7 +15,7 @@ "overrideFeatureInstallOrder": [ "./features/src/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda11.8-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], + "initializeCommand": ["/bin/bash", "-c", "../../features ${localWorkspaceFolder}/.devcontainer/cuda11.8-conda/ && mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda11.8-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/devcontainers,type=bind,consistency=consistent", diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index 94212b758..a76593fe0 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -19,7 +19,7 @@ "./features/src/cuda", "./features/src/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda11.8-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], + "initializeCommand": ["/bin/bash", "-c", "../../features ${localWorkspaceFolder}/.devcontainer/cuda11.8-pip/ && mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda11.8-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/devcontainers,type=bind,consistency=consistent", diff --git a/.devcontainer/cuda12.0-conda/devcontainer.json b/.devcontainer/cuda12.0-conda/devcontainer.json index 40c6b8899..f2272ca2b 100644 --- a/.devcontainer/cuda12.0-conda/devcontainer.json +++ b/.devcontainer/cuda12.0-conda/devcontainer.json @@ -15,7 +15,7 @@ "overrideFeatureInstallOrder": [ "./features/src/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.0-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], + "initializeCommand": ["/bin/bash", "-c", "../../features ${localWorkspaceFolder}/.devcontainer/cuda12.0-conda/ && mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.0-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/devcontainers,type=bind,consistency=consistent", diff --git a/.devcontainer/cuda12.0-pip/devcontainer.json b/.devcontainer/cuda12.0-pip/devcontainer.json index 186498d18..1ca3fac24 100644 --- a/.devcontainer/cuda12.0-pip/devcontainer.json +++ b/.devcontainer/cuda12.0-pip/devcontainer.json @@ -19,7 +19,7 @@ "./features/src/cuda", "./features/src/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], + "initializeCommand": ["/bin/bash", "-c", "ln -sf ../../features ${localWorkspaceFolder}/.devcontainer/cuda12.0-pip/ && mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/devcontainers,type=bind,consistency=consistent", From 8227a1baeeaacefa2ca61f0ac28c63a4e11d755d Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 11:05:21 -0800 Subject: [PATCH 07/59] revert previous commit --- .devcontainer/cuda11.8-conda/devcontainer.json | 2 +- .devcontainer/cuda11.8-pip/devcontainer.json | 2 +- .devcontainer/cuda12.0-conda/devcontainer.json | 2 +- .devcontainer/cuda12.0-pip/devcontainer.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json index 7dfef8502..5aa82c5f4 100644 --- a/.devcontainer/cuda11.8-conda/devcontainer.json +++ b/.devcontainer/cuda11.8-conda/devcontainer.json @@ -15,7 +15,7 @@ "overrideFeatureInstallOrder": [ "./features/src/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "../../features ${localWorkspaceFolder}/.devcontainer/cuda11.8-conda/ && mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda11.8-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda11.8-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/devcontainers,type=bind,consistency=consistent", diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index a76593fe0..94212b758 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -19,7 +19,7 @@ "./features/src/cuda", "./features/src/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "../../features ${localWorkspaceFolder}/.devcontainer/cuda11.8-pip/ && mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda11.8-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda11.8-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/devcontainers,type=bind,consistency=consistent", diff --git a/.devcontainer/cuda12.0-conda/devcontainer.json b/.devcontainer/cuda12.0-conda/devcontainer.json index f2272ca2b..40c6b8899 100644 --- a/.devcontainer/cuda12.0-conda/devcontainer.json +++ b/.devcontainer/cuda12.0-conda/devcontainer.json @@ -15,7 +15,7 @@ "overrideFeatureInstallOrder": [ "./features/src/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "../../features ${localWorkspaceFolder}/.devcontainer/cuda12.0-conda/ && mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.0-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.0-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/devcontainers,type=bind,consistency=consistent", diff --git a/.devcontainer/cuda12.0-pip/devcontainer.json b/.devcontainer/cuda12.0-pip/devcontainer.json index 1ca3fac24..186498d18 100644 --- a/.devcontainer/cuda12.0-pip/devcontainer.json +++ b/.devcontainer/cuda12.0-pip/devcontainer.json @@ -19,7 +19,7 @@ "./features/src/cuda", "./features/src/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "ln -sf ../../features ${localWorkspaceFolder}/.devcontainer/cuda12.0-pip/ && mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/devcontainers,type=bind,consistency=consistent", From 05a4ec07caa91c8a99192f4ca29d9a8b19a2963c Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 11:09:59 -0800 Subject: [PATCH 08/59] use fix/devcontainer-json-location branch --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c076a69c9..f7e2b81f5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -150,7 +150,7 @@ jobs: if: needs.check-event.outputs.ok == 'true' needs: check-event secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.02 + uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location with: build_command: | sccache -z; From ccbc68dd45ce84b751575061cc1a039f132723fc Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 11:31:27 -0800 Subject: [PATCH 09/59] use a beefy machine --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f7e2b81f5..87a056e9c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -152,6 +152,7 @@ jobs: secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location with: + node_type: cpu32 build_command: | sccache -z; clone-all --verbose; From a2dcc018a6bc775c31e340ccc2c1b627f6cbd4a4 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 11:48:56 -0800 Subject: [PATCH 10/59] define GH_TOKEN so `gh auth status` works --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87a056e9c..d7a0705bd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -153,6 +153,7 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location with: node_type: cpu32 + gh_token: "${{ secrets.PERSONAL_ACCESS_TOKEN || env.GITHUB_TOKEN }}" build_command: | sccache -z; clone-all --verbose; From ce33147e5981b6bb80376fe138b17a979b109922 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 11:56:41 -0800 Subject: [PATCH 11/59] use GIST_REPO_READ_ORG_GITHUB_TOKEN secret --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d7a0705bd..fd46a5669 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -153,7 +153,7 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location with: node_type: cpu32 - gh_token: "${{ secrets.PERSONAL_ACCESS_TOKEN || env.GITHUB_TOKEN }}" + gh_token: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN }}" build_command: | sccache -z; clone-all --verbose; From 65c51c4a5fb35950a5f73cc5277ad6081665342f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 11:59:41 -0800 Subject: [PATCH 12/59] pass the secret name --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fd46a5669..46caa555a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -153,7 +153,7 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location with: node_type: cpu32 - gh_token: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN }}" + gh_token: GIST_REPO_READ_ORG_GITHUB_TOKEN build_command: | sccache -z; clone-all --verbose; From 7bb47031050bb6cc237484cc17d5c1ca9a474a44 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 18 Jan 2024 12:04:36 -0800 Subject: [PATCH 13/59] pass --no-update-env --- .github/workflows/test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 46caa555a..afa06ef97 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -156,7 +156,9 @@ jobs: gh_token: GIST_REPO_READ_ORG_GITHUB_TOKEN build_command: | sccache -z; - clone-all --verbose; + clone-all --verbose --no-update-env; + rapids-update-content-command; + rapids-post-attach-command; build-all --verbose \ -DBUILD_TESTS=ON \ -DBUILD_BENCHMARKS=ON \ From 8c2ba00a1fd553c1aecdc0b5e86b9ccb8c14e4be Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Mon, 22 Jan 2024 22:30:38 -0800 Subject: [PATCH 14/59] Apply suggestions from code review --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index afa06ef97..29bab99ec 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -157,12 +157,12 @@ jobs: build_command: | sccache -z; clone-all --verbose --no-update-env; - rapids-update-content-command; - rapids-post-attach-command; + rapids-make-${PYTHON_PACKAGE_MANAGER}-env; build-all --verbose \ -DBUILD_TESTS=ON \ -DBUILD_BENCHMARKS=ON \ -DBUILD_ANN_BENCH=ON \ -DBUILD_PRIMS_BENCH=ON \ + $([ "pip" = $PYTHON_PACKAGE_MANAGER ] && echo -DUSE_LIBARROW_FROM_PYARROW=ON || echo) \ -j$(nproc --ignore=1); sccache -s; From 7c7d5d1f33f4ce865817016381e59ef2b76beed4 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Mon, 22 Jan 2024 22:38:00 -0800 Subject: [PATCH 15/59] Update .github/workflows/test.yml --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 29bab99ec..9d1f7d947 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -155,6 +155,7 @@ jobs: node_type: cpu32 gh_token: GIST_REPO_READ_ORG_GITHUB_TOKEN build_command: | + set -Eeuo pipefail; sccache -z; clone-all --verbose --no-update-env; rapids-make-${PYTHON_PACKAGE_MANAGER}-env; From cd999c8fabc252477893f60c4e830fa8f0037d8e Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 22 Jan 2024 22:45:39 -0800 Subject: [PATCH 16/59] don't run in subshell --- features/src/rapids-build-utils/devcontainer-feature.json | 2 +- .../opt/rapids-build-utils/bin/make-conda-env.sh | 2 +- .../opt/rapids-build-utils/bin/make-pip-env.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index 588dde878..6adf8ce44 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "24.2.7", + "version": "24.2.8", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index a9c75e003..c7118afaa 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -99,6 +99,6 @@ make_conda_env() { . /opt/conda/etc/profile.d/conda.sh; . /opt/conda/etc/profile.d/mamba.sh; -(make_conda_env "${DEFAULT_CONDA_ENV:-rapids}" "$@"); +make_conda_env "${DEFAULT_CONDA_ENV:-rapids}" "$@"; . /etc/profile.d/*-mambaforge.sh; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh index 964ff0956..24612d54b 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh @@ -121,7 +121,7 @@ make_pip_env() { fi } -(make_pip_env "${DEFAULT_VIRTUAL_ENV:-rapids}" "$@"); +make_pip_env "${DEFAULT_VIRTUAL_ENV:-rapids}" "$@"; if test -f ~/.local/share/venvs/${DEFAULT_VIRTUAL_ENV:-rapids}/bin/activate; then . ~/.local/share/venvs/${DEFAULT_VIRTUAL_ENV:-rapids}/bin/activate; From f3856bd1befd7c791dd06248c0542a6cf717f267 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Mon, 22 Jan 2024 22:58:40 -0800 Subject: [PATCH 17/59] Update .github/workflows/test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9d1f7d947..263de2259 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -155,7 +155,7 @@ jobs: node_type: cpu32 gh_token: GIST_REPO_READ_ORG_GITHUB_TOKEN build_command: | - set -Eeuo pipefail; + set -Eeuxo pipefail; sccache -z; clone-all --verbose --no-update-env; rapids-make-${PYTHON_PACKAGE_MANAGER}-env; From 1c9b9c4d46aed7be74bcfead9e62d5e60cda92cc Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Mon, 22 Jan 2024 23:13:03 -0800 Subject: [PATCH 18/59] Update .github/workflows/test.yml --- .github/workflows/test.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 263de2259..720de6bdd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -159,11 +159,14 @@ jobs: sccache -z; clone-all --verbose --no-update-env; rapids-make-${PYTHON_PACKAGE_MANAGER}-env; - build-all --verbose \ + rapids_build_utils_debug=all \ + build-all \ + --verbose \ + -j$(nproc --ignore=1) \ -DBUILD_TESTS=ON \ -DBUILD_BENCHMARKS=ON \ -DBUILD_ANN_BENCH=ON \ -DBUILD_PRIMS_BENCH=ON \ - $([ "pip" = $PYTHON_PACKAGE_MANAGER ] && echo -DUSE_LIBARROW_FROM_PYARROW=ON || echo) \ - -j$(nproc --ignore=1); + $([ "pip" = ${PYTHON_PACKAGE_MANAGER} ] && echo -DUSE_LIBARROW_FROM_PYARROW=ON || echo) \ + ; sccache -s; From 7e128025287e87e891e3cde3c44e6e3dc8cfe24f Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Mon, 22 Jan 2024 23:28:26 -0800 Subject: [PATCH 19/59] Apply suggestions from code review --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 720de6bdd..36643c887 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -158,8 +158,8 @@ jobs: set -Eeuxo pipefail; sccache -z; clone-all --verbose --no-update-env; + rapids-generate-scripts; rapids-make-${PYTHON_PACKAGE_MANAGER}-env; - rapids_build_utils_debug=all \ build-all \ --verbose \ -j$(nproc --ignore=1) \ From c594e116b061a3c1819de3a03e3a32b726266fdd Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Tue, 23 Jan 2024 10:06:42 -0800 Subject: [PATCH 20/59] Update .github/workflows/test.yml --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 36643c887..798081357 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,6 +147,7 @@ jobs: repo: "${{ vars.DOCKERHUB_REPOSITORY || github.repository }}" build-rapids: + name: Build all RAPIDS repos if: needs.check-event.outputs.ok == 'true' needs: check-event secrets: inherit From fa839d22cea71c0fa11b73786df0adcaae6a84e2 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sun, 28 Jan 2024 23:02:03 -0800 Subject: [PATCH 21/59] don't set -E --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 798081357..e7600abeb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -156,7 +156,7 @@ jobs: node_type: cpu32 gh_token: GIST_REPO_READ_ORG_GITHUB_TOKEN build_command: | - set -Eeuxo pipefail; + set -euxo pipefail; sccache -z; clone-all --verbose --no-update-env; rapids-generate-scripts; From 49eaa6aaa0928663ffa6727abbdfb15985446162 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sun, 28 Jan 2024 23:12:24 -0800 Subject: [PATCH 22/59] trace output of rapids-make-conda-env --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e7600abeb..a0e1051b1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -156,10 +156,11 @@ jobs: node_type: cpu32 gh_token: GIST_REPO_READ_ORG_GITHUB_TOKEN build_command: | - set -euxo pipefail; + set -Eeuxo pipefail; sccache -z; clone-all --verbose --no-update-env; rapids-generate-scripts; + rapids_build_utils_debug='*' \ rapids-make-${PYTHON_PACKAGE_MANAGER}-env; build-all \ --verbose \ From 1265d8480dbf425c748b1ee40fbd412134665e0d Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Sun, 28 Jan 2024 23:29:31 -0800 Subject: [PATCH 23/59] trace output --- .../opt/rapids-build-utils/bin/make-conda-env.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index c7118afaa..286441ba7 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -96,6 +96,10 @@ make_conda_env() { fi } +if test -n "${rapids_build_utils_debug:-}"; then + PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; +fi + . /opt/conda/etc/profile.d/conda.sh; . /opt/conda/etc/profile.d/mamba.sh; From 7b4775c9777b5ba7735eec609808beba02914a92 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Sun, 28 Jan 2024 23:43:25 -0800 Subject: [PATCH 24/59] Don't set -Eeuxo pipefail --- .github/workflows/test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a0e1051b1..6836d6613 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -156,11 +156,9 @@ jobs: node_type: cpu32 gh_token: GIST_REPO_READ_ORG_GITHUB_TOKEN build_command: | - set -Eeuxo pipefail; sccache -z; clone-all --verbose --no-update-env; rapids-generate-scripts; - rapids_build_utils_debug='*' \ rapids-make-${PYTHON_PACKAGE_MANAGER}-env; build-all \ --verbose \ From fdbb17c03e56b05e5f459ef4717d4dfceff75809 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Sun, 28 Jan 2024 23:54:50 -0800 Subject: [PATCH 25/59] don't trace output --- .../opt/rapids-build-utils/bin/make-conda-env.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh index 286441ba7..c7118afaa 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh @@ -96,10 +96,6 @@ make_conda_env() { fi } -if test -n "${rapids_build_utils_debug:-}"; then - PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x; -fi - . /opt/conda/etc/profile.d/conda.sh; . /opt/conda/etc/profile.d/mamba.sh; From f852948aa0a69a00e80fb1d7ed49f75063812c6c Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 2 Feb 2024 21:43:26 -0600 Subject: [PATCH 26/59] Rerun CI. From 7b95772e440d7f5e471e5c91903f3cd5d984f5fd Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Sat, 3 Feb 2024 00:05:53 -0800 Subject: [PATCH 27/59] rename shared-action-workflows -> shared-workflows --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6836d6613..4ed0a9f83 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -151,7 +151,7 @@ jobs: if: needs.check-event.outputs.ok == 'true' needs: check-event secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location with: node_type: cpu32 gh_token: GIST_REPO_READ_ORG_GITHUB_TOKEN From f21e0ada4f9625c63fc5fbfeb4cb3a46ad6fe5d6 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 3 Feb 2024 00:34:38 -0800 Subject: [PATCH 28/59] don't do any gh or glab API calls when --clone-upstream is passed --- .../opt/devcontainer/bin/github/repo/clone.sh | 31 +++++++++++++------ .../opt/devcontainer/bin/gitlab/repo/clone.sh | 31 +++++++++++++------ 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh index fc2f1e870..aff6b36eb 100755 --- a/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh +++ b/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh @@ -59,8 +59,6 @@ clone_github_repo() { set -euo pipefail; - source devcontainer-utils-init-github-cli; - local branch=; local no_fork=; local clone_upstream=; @@ -92,7 +90,17 @@ clone_github_repo() { local fork=; local owner=; - if test -z "${clone_upstream:-}"; then + if test -z "${no_fork:-}" && \ + test -z "${clone_upstream:-}" && \ + devcontainer-utils-shell-is-interactive; then + # shellcheck disable=SC1091 + . devcontainer-utils-init-github-cli; + user="${GITHUB_USER:-}"; + fi + + if test -n "${clone_upstream:-}"; then + fork="${upstream}"; + else name="$(get_repo_name "${upstream}")"; owner="$(get_repo_owner "${upstream}")"; user="${GITHUB_USER:-"${owner}"}"; @@ -106,7 +114,7 @@ clone_github_repo() { devcontainer-utils-shell-is-interactive; then while true; do local CHOICE; - read -p "'${GITHUB_HOST:-github.com}/${user}/${name}.git' not found. + read -rp "'${GITHUB_HOST:-github.com}/${user}/${name}.git' not found. Fork '${upstream}' into '${user}/${name}' now (y/n)? " CHOICE <$(tty) case "${CHOICE:-}" in [Nn]* ) origin="${upstream}"; break;; @@ -118,12 +126,17 @@ clone_github_repo() { done fi - if [ "$(gh config get git_protocol)" = "ssh" ]; then - origin="$(get_repo_ssh_url "${origin}")"; - upstream="$(get_repo_ssh_url "${upstream}")"; + if gh auth status >/dev/null 2>&1; then + if [ "$(gh config get git_protocol)" = "ssh" ]; then + origin="$(get_repo_ssh_url "${origin}")"; + upstream="$(get_repo_ssh_url "${upstream}")"; + else + origin="$(get_repo_git_url "${origin}")"; + upstream="$(get_repo_git_url "${upstream}")"; + fi else - origin="$(get_repo_git_url "${origin}")"; - upstream="$(get_repo_git_url "${upstream}")"; + origin="https://${GITHUB_HOST:-github.com}/${origin}.git"; + upstream="https://${GITHUB_HOST:-github.com}/${upstream}.git"; fi devcontainer-utils-clone-git-repo \ diff --git a/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh index 55d1b161c..82c51ac52 100755 --- a/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh +++ b/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh @@ -85,8 +85,6 @@ clone_gitlab_repo() { set -euo pipefail; - source devcontainer-utils-init-gitlab-cli; - local branch=; local no_fork=; local clone_upstream=; @@ -118,7 +116,17 @@ clone_gitlab_repo() { local user=; local owner=; - if test -z "${clone_upstream:-}"; then + if test -z "${no_fork:-}" && \ + test -z "${clone_upstream:-}" && \ + devcontainer-utils-shell-is-interactive; then + # shellcheck disable=SC1091 + . devcontainer-utils-init-gitlab-cli; + user="${GITLAB_USER:-}"; + fi + + if test -n "${clone_upstream:-}"; then + fork="${upstream}"; + else name="$(get_repo_name "${upstream}")"; owner="$(get_repo_owner "${upstream}")"; user="${GITLAB_USER:-"${owner}"}"; @@ -132,7 +140,7 @@ clone_gitlab_repo() { devcontainer-utils-shell-is-interactive; then while true; do local CHOICE; - read -p "'${GITLAB_HOST:-gitlab.com}/${user}/${name}.git' not found. + read -rp "'${GITLAB_HOST:-gitlab.com}/${user}/${name}.git' not found. Fork '${upstream}' into '${user}/${name}' now (y/n)? " CHOICE <$(tty) case "${CHOICE:-}" in [Nn]* ) origin="${upstream}"; break;; @@ -144,12 +152,17 @@ clone_gitlab_repo() { done fi - if [ "$(glab config get git_protocol)" = "ssh" ]; then - origin="$(get_repo_ssh_url "${origin}")"; - upstream="$(get_repo_ssh_url "${upstream}")"; + if ! glab auth status 2>&1 | grep -q "No token provided"; then + if [ "$(glab config get git_protocol)" = "ssh" ]; then + origin="$(get_repo_ssh_url "${origin}")"; + upstream="$(get_repo_ssh_url "${upstream}")"; + else + origin="$(get_repo_git_url "${origin}")"; + upstream="$(get_repo_git_url "${upstream}")"; + fi else - origin="$(get_repo_git_url "${origin}")"; - upstream="$(get_repo_git_url "${upstream}")"; + origin="https://${GITLAB_HOST:-gitlab.com}/${origin}.git"; + upstream="https://${GITLAB_HOST:-gitlab.com}/${upstream}.git"; fi devcontainer-utils-clone-git-repo \ From 307e75e2251ba3d1e99f311a31d03b7e54c89425 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 3 Feb 2024 00:34:47 -0800 Subject: [PATCH 29/59] bump feature version --- features/src/utils/devcontainer-feature.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json index d319c04f9..3af663f5b 100644 --- a/features/src/utils/devcontainer-feature.json +++ b/features/src/utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "devcontainer-utils", "id": "utils", - "version": "24.4.4", + "version": "24.4.5", "description": "A feature to install RAPIDS devcontainer utility scripts", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" From 521c69f4cb0fe6cc1b965b58523afc3d955a31f5 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 3 Feb 2024 00:35:19 -0800 Subject: [PATCH 30/59] use --clone-upstream, remove GH_TOKEN secret --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4ed0a9f83..ff1e4eceb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -154,10 +154,9 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location with: node_type: cpu32 - gh_token: GIST_REPO_READ_ORG_GITHUB_TOKEN build_command: | sccache -z; - clone-all --verbose --no-update-env; + clone-all --verbose --clone-upstream --no-update-env; rapids-generate-scripts; rapids-make-${PYTHON_PACKAGE_MANAGER}-env; build-all \ From 0db37398b478473cebed253e3a0dc082a9b2d564 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 3 Feb 2024 01:18:21 -0800 Subject: [PATCH 31/59] debug --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ff1e4eceb..ba76f56ac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -159,6 +159,7 @@ jobs: clone-all --verbose --clone-upstream --no-update-env; rapids-generate-scripts; rapids-make-${PYTHON_PACKAGE_MANAGER}-env; + rapids_build_utils_debug='*' \ build-all \ --verbose \ -j$(nproc --ignore=1) \ From 5ee6aaff4cba21a7648eae582f3598e7224f9049 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 6 Feb 2024 10:41:11 -0800 Subject: [PATCH 32/59] add cumlprims_mg and cugraph-ops private deploy keys --- .github/workflows/test.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6836d6613..b8e3b3e1b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -156,6 +156,17 @@ jobs: node_type: cpu32 gh_token: GIST_REPO_READ_ORG_GITHUB_TOKEN build_command: | + # Load private repo deploy keys + if ! grep -qE "^$" <<< "${{ secrets.CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY }}" \ + || ! grep -qE "^$" <<< "${{ secrets.CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY }}"; then + if ! pgrep ssh-agent >/dev/null 2>&1; then eval "$(ssh-agent -s)"; fi; + ssh-add - << "__EOF" + ${{ secrets.CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY }} + ${{ secrets.CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY }} + __EOF + devcontainer-utils-init-ssh-deploy-keys || true; + fi + sccache -z; clone-all --verbose --no-update-env; rapids-generate-scripts; From bdf1b80522f3b69c5d573e86568dcae891bf9e5a Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Mon, 29 Jan 2024 10:46:30 -0800 Subject: [PATCH 33/59] temporarily allow unbound variables while activating the conda env (#216) --- features/src/mambaforge/.bashrc | 12 ++++++++++-- features/src/mambaforge/devcontainer-feature.json | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/features/src/mambaforge/.bashrc b/features/src/mambaforge/.bashrc index 1d0543804..7ef4b92be 100644 --- a/features/src/mambaforge/.bashrc +++ b/features/src/mambaforge/.bashrc @@ -5,8 +5,16 @@ for default_conda_env_name in ${DEFAULT_CONDA_ENV:-} ${CONDA_DEFAULT_ENV:-} base if grep -qF "(${default_conda_env_name})" <<< "${CONDA_PROMPT_MODIFIER:-}"; then break; fi - conda activate "$default_conda_env_name" 2>/dev/null && break || continue; -done; + # Temporarily allow unbound variables for conda activation. + oldstate="$(shopt -po; shopt -p)"; [[ -o errexit ]] && oldstate="${oldstate}; set -e"; set +u; + if conda activate "${default_conda_env_name}" 2>/dev/null; then + { set +vx; } 2>/dev/null; eval "${oldstate}"; unset oldstate; + break; + else + { set +vx; } 2>/dev/null; eval "${oldstate}"; unset oldstate; + continue; + fi +done if [ -n "${CONDA_EXE:-}" ]; then conda_bin_paths="$(dirname "$(dirname "${CONDA_EXE}")")/condabin"; diff --git a/features/src/mambaforge/devcontainer-feature.json b/features/src/mambaforge/devcontainer-feature.json index 6ef848bb5..5e921ed0e 100644 --- a/features/src/mambaforge/devcontainer-feature.json +++ b/features/src/mambaforge/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "Mambaforge", "id": "mambaforge", - "version": "24.4.0", + "version": "24.4.1", "description": "A feature to install mambaforge", "options": { "version": { From 52e1d5e7ef15680a089bdb7e37c7e5be3ce1617d Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 2 Feb 2024 21:43:26 -0600 Subject: [PATCH 34/59] Rerun CI. From b973cb08a4c6b6c8a0d7824c285beca6f50514cc Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Sat, 3 Feb 2024 00:05:53 -0800 Subject: [PATCH 35/59] rename shared-action-workflows -> shared-workflows --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b8e3b3e1b..cb2d9757b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -151,7 +151,7 @@ jobs: if: needs.check-event.outputs.ok == 'true' needs: check-event secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location with: node_type: cpu32 gh_token: GIST_REPO_READ_ORG_GITHUB_TOKEN From fdf733b11ed0baaad25633405ff123ddd74606e9 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 3 Feb 2024 00:34:38 -0800 Subject: [PATCH 36/59] don't do any gh or glab API calls when --clone-upstream is passed --- .../opt/devcontainer/bin/github/repo/clone.sh | 31 +++++++++++++------ .../opt/devcontainer/bin/gitlab/repo/clone.sh | 31 +++++++++++++------ 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh index fc2f1e870..aff6b36eb 100755 --- a/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh +++ b/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh @@ -59,8 +59,6 @@ clone_github_repo() { set -euo pipefail; - source devcontainer-utils-init-github-cli; - local branch=; local no_fork=; local clone_upstream=; @@ -92,7 +90,17 @@ clone_github_repo() { local fork=; local owner=; - if test -z "${clone_upstream:-}"; then + if test -z "${no_fork:-}" && \ + test -z "${clone_upstream:-}" && \ + devcontainer-utils-shell-is-interactive; then + # shellcheck disable=SC1091 + . devcontainer-utils-init-github-cli; + user="${GITHUB_USER:-}"; + fi + + if test -n "${clone_upstream:-}"; then + fork="${upstream}"; + else name="$(get_repo_name "${upstream}")"; owner="$(get_repo_owner "${upstream}")"; user="${GITHUB_USER:-"${owner}"}"; @@ -106,7 +114,7 @@ clone_github_repo() { devcontainer-utils-shell-is-interactive; then while true; do local CHOICE; - read -p "'${GITHUB_HOST:-github.com}/${user}/${name}.git' not found. + read -rp "'${GITHUB_HOST:-github.com}/${user}/${name}.git' not found. Fork '${upstream}' into '${user}/${name}' now (y/n)? " CHOICE <$(tty) case "${CHOICE:-}" in [Nn]* ) origin="${upstream}"; break;; @@ -118,12 +126,17 @@ clone_github_repo() { done fi - if [ "$(gh config get git_protocol)" = "ssh" ]; then - origin="$(get_repo_ssh_url "${origin}")"; - upstream="$(get_repo_ssh_url "${upstream}")"; + if gh auth status >/dev/null 2>&1; then + if [ "$(gh config get git_protocol)" = "ssh" ]; then + origin="$(get_repo_ssh_url "${origin}")"; + upstream="$(get_repo_ssh_url "${upstream}")"; + else + origin="$(get_repo_git_url "${origin}")"; + upstream="$(get_repo_git_url "${upstream}")"; + fi else - origin="$(get_repo_git_url "${origin}")"; - upstream="$(get_repo_git_url "${upstream}")"; + origin="https://${GITHUB_HOST:-github.com}/${origin}.git"; + upstream="https://${GITHUB_HOST:-github.com}/${upstream}.git"; fi devcontainer-utils-clone-git-repo \ diff --git a/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh index 55d1b161c..82c51ac52 100755 --- a/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh +++ b/features/src/utils/opt/devcontainer/bin/gitlab/repo/clone.sh @@ -85,8 +85,6 @@ clone_gitlab_repo() { set -euo pipefail; - source devcontainer-utils-init-gitlab-cli; - local branch=; local no_fork=; local clone_upstream=; @@ -118,7 +116,17 @@ clone_gitlab_repo() { local user=; local owner=; - if test -z "${clone_upstream:-}"; then + if test -z "${no_fork:-}" && \ + test -z "${clone_upstream:-}" && \ + devcontainer-utils-shell-is-interactive; then + # shellcheck disable=SC1091 + . devcontainer-utils-init-gitlab-cli; + user="${GITLAB_USER:-}"; + fi + + if test -n "${clone_upstream:-}"; then + fork="${upstream}"; + else name="$(get_repo_name "${upstream}")"; owner="$(get_repo_owner "${upstream}")"; user="${GITLAB_USER:-"${owner}"}"; @@ -132,7 +140,7 @@ clone_gitlab_repo() { devcontainer-utils-shell-is-interactive; then while true; do local CHOICE; - read -p "'${GITLAB_HOST:-gitlab.com}/${user}/${name}.git' not found. + read -rp "'${GITLAB_HOST:-gitlab.com}/${user}/${name}.git' not found. Fork '${upstream}' into '${user}/${name}' now (y/n)? " CHOICE <$(tty) case "${CHOICE:-}" in [Nn]* ) origin="${upstream}"; break;; @@ -144,12 +152,17 @@ clone_gitlab_repo() { done fi - if [ "$(glab config get git_protocol)" = "ssh" ]; then - origin="$(get_repo_ssh_url "${origin}")"; - upstream="$(get_repo_ssh_url "${upstream}")"; + if ! glab auth status 2>&1 | grep -q "No token provided"; then + if [ "$(glab config get git_protocol)" = "ssh" ]; then + origin="$(get_repo_ssh_url "${origin}")"; + upstream="$(get_repo_ssh_url "${upstream}")"; + else + origin="$(get_repo_git_url "${origin}")"; + upstream="$(get_repo_git_url "${upstream}")"; + fi else - origin="$(get_repo_git_url "${origin}")"; - upstream="$(get_repo_git_url "${upstream}")"; + origin="https://${GITLAB_HOST:-gitlab.com}/${origin}.git"; + upstream="https://${GITLAB_HOST:-gitlab.com}/${upstream}.git"; fi devcontainer-utils-clone-git-repo \ From a6973ac30d32045866fe1be432e46e835e4e5a29 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 3 Feb 2024 00:34:47 -0800 Subject: [PATCH 37/59] bump feature version --- features/src/utils/devcontainer-feature.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json index d319c04f9..3af663f5b 100644 --- a/features/src/utils/devcontainer-feature.json +++ b/features/src/utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "devcontainer-utils", "id": "utils", - "version": "24.4.4", + "version": "24.4.5", "description": "A feature to install RAPIDS devcontainer utility scripts", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" From 2bad0487a23553c77f14839dc388cd41b61a6f88 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 3 Feb 2024 00:35:19 -0800 Subject: [PATCH 38/59] use --clone-upstream, remove GH_TOKEN secret --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cb2d9757b..5f695bb1d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -154,7 +154,6 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location with: node_type: cpu32 - gh_token: GIST_REPO_READ_ORG_GITHUB_TOKEN build_command: | # Load private repo deploy keys if ! grep -qE "^$" <<< "${{ secrets.CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY }}" \ @@ -168,7 +167,7 @@ jobs: fi sccache -z; - clone-all --verbose --no-update-env; + clone-all --verbose --clone-upstream --no-update-env; rapids-generate-scripts; rapids-make-${PYTHON_PACKAGE_MANAGER}-env; build-all \ From 9dc4b4d55ec60271367713d73508db8c1720e651 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sat, 3 Feb 2024 01:18:21 -0800 Subject: [PATCH 39/59] debug --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5f695bb1d..b55339890 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -170,6 +170,7 @@ jobs: clone-all --verbose --clone-upstream --no-update-env; rapids-generate-scripts; rapids-make-${PYTHON_PACKAGE_MANAGER}-env; + rapids_build_utils_debug='*' \ build-all \ --verbose \ -j$(nproc --ignore=1) \ From e3f952005789136655ee4c3390157a0e82e24656 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Fri, 2 Feb 2024 23:24:43 -0800 Subject: [PATCH 40/59] Fix mambaforge shell history (#219) * disable history when restoring shell options * Update devcontainer-feature.json --- features/src/mambaforge/.bashrc | 6 +++--- features/src/mambaforge/devcontainer-feature.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/features/src/mambaforge/.bashrc b/features/src/mambaforge/.bashrc index 7ef4b92be..c6804046a 100644 --- a/features/src/mambaforge/.bashrc +++ b/features/src/mambaforge/.bashrc @@ -6,12 +6,12 @@ for default_conda_env_name in ${DEFAULT_CONDA_ENV:-} ${CONDA_DEFAULT_ENV:-} base break; fi # Temporarily allow unbound variables for conda activation. - oldstate="$(shopt -po; shopt -p)"; [[ -o errexit ]] && oldstate="${oldstate}; set -e"; set +u; + oldstate="$(shopt -po | grep -E '(nounset|verbose|xtrace)')"; set +u; if conda activate "${default_conda_env_name}" 2>/dev/null; then - { set +vx; } 2>/dev/null; eval "${oldstate}"; unset oldstate; + { set +vxo history; } 2>/dev/null; eval "${oldstate}"; unset oldstate; break; else - { set +vx; } 2>/dev/null; eval "${oldstate}"; unset oldstate; + { set +vxo history; } 2>/dev/null; eval "${oldstate}"; unset oldstate; continue; fi done diff --git a/features/src/mambaforge/devcontainer-feature.json b/features/src/mambaforge/devcontainer-feature.json index 5e921ed0e..77013cb00 100644 --- a/features/src/mambaforge/devcontainer-feature.json +++ b/features/src/mambaforge/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "Mambaforge", "id": "mambaforge", - "version": "24.4.1", + "version": "24.4.2", "description": "A feature to install mambaforge", "options": { "version": { From 134c91da231070141bade505caa54a83d12c5a36 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Fri, 2 Feb 2024 23:24:56 -0800 Subject: [PATCH 41/59] Add cugraph-equivariant (#218) --- .../rapids-build-utils/opt/rapids-build-utils/manifest.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml index 534918dd9..bf1667563 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml @@ -171,6 +171,9 @@ repos: - name: cugraph-dgl sub_dir: python/cugraph-dgl depends: [cugraph] + - name: cugraph-equivariant + sub_dir: python/cugraph-equivariant + depends: [cugraph] - name: cugraph_pyg sub_dir: python/cugraph-pyg depends: [cugraph] From 806a6fdb1ee628ade6f37a36aaf22a1125d3a1f5 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Sun, 4 Feb 2024 19:48:58 -0800 Subject: [PATCH 42/59] Fix buildx setup on github runners (#220) * Fix buildx setup on github runners * change conditional to `env.RUNNER_ENV == 'self-hosted' --- .github/actions/setup-runner-env/action.yml | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-runner-env/action.yml b/.github/actions/setup-runner-env/action.yml index b52487360..ae701557f 100644 --- a/.github/actions/setup-runner-env/action.yml +++ b/.github/actions/setup-runner-env/action.yml @@ -5,27 +5,30 @@ description: Setup self-hosted runner environment runs: using: composite steps: + - name: Dump env + shell: bash -eo pipefail {0} + run: echo "${{ toJSON(env) }}" - - if: env.RUNNER_ENVIRONMENT == 'github-hosted' + - if: env.RUNNER_ENVIRONMENT != 'self-hosted' name: Free up disk space uses: ./.github/actions/free-disk-space with: tool_cache: "${{ runner.tool_cache }}" - - if: env.RUNNER_ENVIRONMENT != 'github-hosted' + - if: env.RUNNER_ENVIRONMENT == 'self-hosted' name: Setup self-hosted runner environment shell: bash -eo pipefail {0} run: | echo "HOME=${{ runner.workspace }}" >> $GITHUB_ENV; echo "TMPDIR=${{ runner.temp }}" >> $GITHUB_ENV; - - if: env.RUNNER_ENVIRONMENT != 'github-hosted' + - if: env.RUNNER_ENVIRONMENT == 'self-hosted' name: Setup Node.js uses: actions/setup-node@v3 with: node-version: '16' - - if: env.RUNNER_ENVIRONMENT == 'github-hosted' + - if: env.RUNNER_ENVIRONMENT != 'self-hosted' name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -33,7 +36,15 @@ runs: shell: bash run: docker context create builder - - name: Setup docker buildx + - if: env.RUNNER_ENVIRONMENT != 'self-hosted' + name: Setup docker buildx on github-hosted runners + uses: docker/setup-buildx-action@v2 + with: + buildkitd-flags: --debug + endpoint: builder + + - if: env.RUNNER_ENVIRONMENT == 'self-hosted' + name: Setup docker buildx on self-hosted runners uses: docker/setup-buildx-action@v2 with: buildkitd-flags: --debug --config /etc/buildkit/buildkitd.toml From 1e2de4be95cb21c80dd2878a2fa402a8e316ae60 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 6 Feb 2024 10:50:17 -0800 Subject: [PATCH 43/59] pass repo deploy key names to action --- .github/workflows/test.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b55339890..73407bbdd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -154,18 +154,9 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location with: node_type: cpu32 + extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY + extra-repo-deploy-key-2: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY build_command: | - # Load private repo deploy keys - if ! grep -qE "^$" <<< "${{ secrets.CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY }}" \ - || ! grep -qE "^$" <<< "${{ secrets.CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY }}"; then - if ! pgrep ssh-agent >/dev/null 2>&1; then eval "$(ssh-agent -s)"; fi; - ssh-add - << "__EOF" - ${{ secrets.CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY }} - ${{ secrets.CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY }} - __EOF - devcontainer-utils-init-ssh-deploy-keys || true; - fi - sccache -z; clone-all --verbose --clone-upstream --no-update-env; rapids-generate-scripts; From 3034eed82907b269d5a962d23c8aacdca05efa70 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 6 Feb 2024 10:54:22 -0800 Subject: [PATCH 44/59] remove debug flag --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 73407bbdd..c51a5b22a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -161,7 +161,6 @@ jobs: clone-all --verbose --clone-upstream --no-update-env; rapids-generate-scripts; rapids-make-${PYTHON_PACKAGE_MANAGER}-env; - rapids_build_utils_debug='*' \ build-all \ --verbose \ -j$(nproc --ignore=1) \ From 2bcb95dc3306664b3ad1ac21ada4eae88adce6b4 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Sun, 4 Feb 2024 22:36:27 -0800 Subject: [PATCH 45/59] login to dockerhub so we don't hit rate limits --- .github/workflows/build-and-test-feature.yml | 6 ++++++ .../workflows/build-test-and-push-linux-image.yml | 6 ++++++ .../workflows/build-test-and-push-windows-image.yml | 13 ++++++------- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-and-test-feature.yml b/.github/workflows/build-and-test-feature.yml index fcca147d8..e94ae0896 100644 --- a/.github/workflows/build-and-test-feature.yml +++ b/.github/workflows/build-and-test-feature.yml @@ -25,6 +25,12 @@ jobs: with: persist-credentials: false + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} + password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} + - name: ${{ inputs.name }} uses: ./.github/actions/build-and-test-feature with: diff --git a/.github/workflows/build-test-and-push-linux-image.yml b/.github/workflows/build-test-and-push-linux-image.yml index ad4ae47d9..2b267a00e 100644 --- a/.github/workflows/build-test-and-push-linux-image.yml +++ b/.github/workflows/build-test-and-push-linux-image.yml @@ -51,6 +51,12 @@ jobs: features: "${{ inputs.features }}" container_env: "${{ inputs.container_env }}" + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} + password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} + - name: Build ${{ steps.json.outputs.tag }}-${{ matrix.arch }} uses: ./.github/actions/build-linux-image with: diff --git a/.github/workflows/build-test-and-push-windows-image.yml b/.github/workflows/build-test-and-push-windows-image.yml index 35eacdbe9..69a9d4f57 100644 --- a/.github/workflows/build-test-and-push-windows-image.yml +++ b/.github/workflows/build-test-and-push-windows-image.yml @@ -59,6 +59,12 @@ jobs: tag=${version}-cuda${cuda}-cl${cl}-${{ inputs.os }}${{ matrix.edition }} EOF + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} + password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} + - name: Build ${{ steps.info.outputs.tag }} uses: ./.github/actions/build-windows-image with: @@ -79,13 +85,6 @@ jobs: version: "${{ steps.info.outputs.version }}" edition: "${{ matrix.edition }}" - - if: inputs.push == 'true' - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} - password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} - - if: inputs.push == 'true' name: Push ${{ steps.info.outputs.tag }} shell: powershell From 3d91ddfb0a81ab2be5b8c13496e6536310785de9 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 7 Feb 2024 14:48:43 -0800 Subject: [PATCH 46/59] move rapids build job into reusable workflow, schedule it to run nightly --- .github/workflows/build-all-rapids-repos.yml | 48 ++++++++++++++++++++ .github/workflows/test.yml | 23 +--------- 2 files changed, 50 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/build-all-rapids-repos.yml diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml new file mode 100644 index 000000000..a181af723 --- /dev/null +++ b/.github/workflows/build-all-rapids-repos.yml @@ -0,0 +1,48 @@ +name: Build all RAPIDS repositories + +on: + schedule: + - cron: '0 0 * * *' + workflow_call: + +jobs: + check-event: + name: Check GH Event + runs-on: ubuntu-latest + outputs: + ok: ${{ steps.check_gh_event.outputs.ok }} + steps: + - id: check_gh_event + name: Check GH Event + shell: bash + run: | + [[ '${{ github.event_name }}' == 'workflow_call' ]] || \ + [[ '${{ github.event_name }}' == 'schedule' && '${{ github.repository }}' == 'rapidsai/devcontainers' ]] \ + && echo "ok=true" | tee -a $GITHUB_OUTPUT \ + || echo "ok=false" | tee -a $GITHUB_OUTPUT; + + build-all-rapids-repos: + name: Build all RAPIDS repos + if: needs.check-event.outputs.ok == 'true' + needs: check-event + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location + with: + node_type: cpu32 + extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY + extra-repo-deploy-key-2: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + build_command: | + sccache -z; + clone-all --verbose --clone-upstream --no-update-env; + rapids-generate-scripts; + rapids-make-${PYTHON_PACKAGE_MANAGER}-env; + build-all \ + --verbose \ + -j$(nproc --ignore=1) \ + -DBUILD_TESTS=ON \ + -DBUILD_BENCHMARKS=ON \ + -DBUILD_ANN_BENCH=ON \ + -DBUILD_PRIMS_BENCH=ON \ + $([ "pip" = ${PYTHON_PACKAGE_MANAGER} ] && echo -DUSE_LIBARROW_FROM_PYARROW=ON || echo) \ + ; + sccache -s; diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c51a5b22a..644b16856 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -146,28 +146,9 @@ jobs: container_env: "${{ toJSON(matrix.env) }}" repo: "${{ vars.DOCKERHUB_REPOSITORY || github.repository }}" - build-rapids: + build-all-rapids-repos: name: Build all RAPIDS repos if: needs.check-event.outputs.ok == 'true' needs: check-event secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location - with: - node_type: cpu32 - extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY - extra-repo-deploy-key-2: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - build_command: | - sccache -z; - clone-all --verbose --clone-upstream --no-update-env; - rapids-generate-scripts; - rapids-make-${PYTHON_PACKAGE_MANAGER}-env; - build-all \ - --verbose \ - -j$(nproc --ignore=1) \ - -DBUILD_TESTS=ON \ - -DBUILD_BENCHMARKS=ON \ - -DBUILD_ANN_BENCH=ON \ - -DBUILD_PRIMS_BENCH=ON \ - $([ "pip" = ${PYTHON_PACKAGE_MANAGER} ] && echo -DUSE_LIBARROW_FROM_PYARROW=ON || echo) \ - ; - sccache -s; + uses: ./.github/workflows/build-all-rapids-repos.yml From e9e36e19089a5c7fe27dbcd67f08167b335abdd7 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 7 Feb 2024 14:51:19 -0800 Subject: [PATCH 47/59] fix event name check --- .github/workflows/build-all-rapids-repos.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index a181af723..8db56b6bb 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -16,8 +16,9 @@ jobs: name: Check GH Event shell: bash run: | - [[ '${{ github.event_name }}' == 'workflow_call' ]] || \ - [[ '${{ github.event_name }}' == 'schedule' && '${{ github.repository }}' == 'rapidsai/devcontainers' ]] \ + [[ '${{ github.event_name }}' == 'push' && '${{ github.repository }}' == 'rapidsai/devcontainers' ]] || \ + [[ '${{ github.event_name }}' == 'schedule' && '${{ github.repository }}' == 'rapidsai/devcontainers' ]] || \ + [[ '${{ github.event_name }}' == 'pull_request' && '${{ github.repository }}' != 'rapidsai/devcontainers' ]] \ && echo "ok=true" | tee -a $GITHUB_OUTPUT \ || echo "ok=false" | tee -a $GITHUB_OUTPUT; From 40529a5ef93760c8334e415ffae9a1f5f884ffc1 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 12 Feb 2024 13:55:31 -0800 Subject: [PATCH 48/59] pass arch and cuda to build-in-devcontainer.yaml --- .github/workflows/build-all-rapids-repos.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 8db56b6bb..5c2aa35cb 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -29,6 +29,8 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location with: + arch: '["amd64", "arm64"]' + cuda: '["12.0"]' node_type: cpu32 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY extra-repo-deploy-key-2: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY From b8d3c77bfb9ef57ed831c7f8fe8efeff6b3dec04 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 12 Feb 2024 14:15:57 -0800 Subject: [PATCH 49/59] add more system deps for pip containers --- .devcontainer/rapids.Dockerfile | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile index 2400d6aa9..5e633b6ae 100644 --- a/.devcontainer/rapids.Dockerfile +++ b/.devcontainer/rapids.Dockerfile @@ -7,10 +7,23 @@ FROM ${BASE} as pip-base RUN apt update -y \ && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \ + # C++ build tools + doxygen \ + graphviz \ + # C++ test dependencies + libgmock-dev \ + libgtest-dev \ # needed by libcudf_kafka librdkafka-dev \ + # cuML/cuGraph dependencies + libblas-dev \ + liblapack-dev \ # needed by libcuspatial - sqlite3 libsqlite3-dev libtiff-dev libcurl4-openssl-dev \ + libgdal-dev \ + sqlite3 \ + libsqlite3-dev \ + libtiff-dev \ + libcurl4-openssl-dev \ && rm -rf /tmp/* /var/tmp/* /var/cache/apt/* /var/lib/apt/lists/*; ENV DEFAULT_VIRTUAL_ENV=rapids From 8607a24ac5f6325c6fc2a8389b02b438cfbe5277 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 12 Feb 2024 14:16:07 -0800 Subject: [PATCH 50/59] build CUDA 12.2 --- .github/workflows/build-all-rapids-repos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 5c2aa35cb..8bd8f1314 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -30,7 +30,7 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location with: arch: '["amd64", "arm64"]' - cuda: '["12.0"]' + cuda: '["12.0", "12.2"]' node_type: cpu32 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY extra-repo-deploy-key-2: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY From b2f3f2d22d61606a8dfa840eee0f8dc7c53bb1af Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 12 Feb 2024 14:16:20 -0800 Subject: [PATCH 51/59] temporarily disable building features/images when test.yml changes --- .github/actions/feature-matrix/action.yml | 3 --- .github/actions/image-matrix/action.yml | 3 --- 2 files changed, 6 deletions(-) diff --git a/.github/actions/feature-matrix/action.yml b/.github/actions/feature-matrix/action.yml index f59855064..3252165ae 100644 --- a/.github/actions/feature-matrix/action.yml +++ b/.github/actions/feature-matrix/action.yml @@ -30,9 +30,6 @@ runs: files: | features/src/** features/test/** - .github/workflows/test.yml - .github/actions/feature-matrix/action.sh - .github/actions/feature-matrix/action.yml .github/workflows/build-and-test-feature.yml .github/actions/build-and-test-feature/action.yml diff --git a/.github/actions/image-matrix/action.yml b/.github/actions/image-matrix/action.yml index 326691b9d..65c71ec29 100644 --- a/.github/actions/image-matrix/action.yml +++ b/.github/actions/image-matrix/action.yml @@ -38,10 +38,7 @@ runs: windows/** matrix.yml .github/**/*windows* - .github/workflows/test.yml .github/actions/build-linux-image/action.yml - .github/actions/image-matrix/action.sh - .github/actions/image-matrix/action.yml .github/workflows/build-and-test-image.yml - name: Report changes From 8c5d7f70c4af571baae07ef1dbf3757794604210 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 12 Feb 2024 22:30:39 +0000 Subject: [PATCH 52/59] escape periods --- .github/actions/feature-matrix/action.sh | 2 +- .github/actions/image-matrix/action.sh | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/actions/feature-matrix/action.sh b/.github/actions/feature-matrix/action.sh index 2765d8cf0..a819d49b9 100755 --- a/.github/actions/feature-matrix/action.sh +++ b/.github/actions/feature-matrix/action.sh @@ -25,7 +25,7 @@ files="${@}"; features=""; scenarios=""; -if [ "${full_matrix}" == "1" ] || echo "$files" | grep -q ".github/"; then +if [ "${full_matrix}" == "1" ] || grep -q "\.github/" <<< "$files"; then features="$(find_features_with_tests 'test.sh' 'features/test')"; scenarios="$(find_features_with_tests 'scenarios.json' 'features/test')"; elif echo "$files" | grep -q "features/"; then diff --git a/.github/actions/image-matrix/action.sh b/.github/actions/image-matrix/action.sh index 73a7a805f..2ab15f423 100755 --- a/.github/actions/image-matrix/action.sh +++ b/.github/actions/image-matrix/action.sh @@ -31,18 +31,18 @@ features="$(echo \ || echo '' \ )"; -if `# Include all images if full_matrix is true` \ - [ "${full_matrix}" == "1" ] \ - `# Include all images if matrix or workflows changed` \ -|| grep -q 'matrix.yml' <<< "${files}" \ -|| grep -q '.github/workflows/test.yml' <<< "${files}" \ -|| grep -q '.github/actions/build-linux-image/action.yml' <<< "${files}" \ -|| grep -q '.github/actions/image-matrix/action.sh' <<< "${files}" \ -|| grep -q '.github/actions/image-matrix/action.yml' <<< "${files}" \ -|| grep -q '.github/workflows/build-and-test-image.yml' <<< "${files}" \ - `# Include all images if cmake, ninja, sccache, ` \ - `# gitlab-cli, or utils features changed` \ -|| grep -qE "(${common_features})" <<< "${features}" \ +if `# Include all images if full_matrix is true` \ + [ "${full_matrix}" == "1" ] \ + `# Include all images if matrix or workflows changed` \ +|| grep -q 'matrix\.yml' <<< "${files}" \ +|| grep -q '\.github/workflows/test\.yml' <<< "${files}" \ +|| grep -q '\.github/actions/build-linux-image/action\.yml' <<< "${files}" \ +|| grep -q '\.github/actions/image-matrix/action\.sh' <<< "${files}" \ +|| grep -q '\.github/actions/image-matrix/action\.yml' <<< "${files}" \ +|| grep -q '\.github/workflows/build-and-test-image\.yml' <<< "${files}" \ + `# Include all images if cmake, ninja, sccache, ` \ + `# gitlab-cli, or utils features changed` \ +|| grep -qE "(${common_features})" <<< "${features}" \ ; then features="$( \ find features/src -mindepth 1 -maxdepth 1 -type d -print0 \ From deb45cb06b887d70d0e3dd7310e643f8f87b7090 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 14 Feb 2024 18:10:21 -0800 Subject: [PATCH 53/59] just build x86 CUDA 12.0 --- .github/workflows/build-all-rapids-repos.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 8bd8f1314..2c02374a2 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -29,8 +29,8 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location with: - arch: '["amd64", "arm64"]' - cuda: '["12.0", "12.2"]' + arch: '["amd64"]' + cuda: '["12.0"]' node_type: cpu32 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY extra-repo-deploy-key-2: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY From 47887c824f7e405f9a8cab020d4eb815ee6df691 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 14 Feb 2024 18:31:19 -0800 Subject: [PATCH 54/59] use CUDA 12.3 for new pip container, install additional CTK libs in pip containers for RAFT --- .devcontainer/cuda11.8-pip/devcontainer.json | 2 +- .devcontainer/cuda12.0-pip/devcontainer.json | 2 +- .../devcontainer.json | 14 +++++++------- .../{cuda12.2-pip => cuda12.3-pip}/features | 0 4 files changed, 9 insertions(+), 9 deletions(-) rename .devcontainer/{cuda12.2-pip => cuda12.3-pip}/devcontainer.json (85%) rename .devcontainer/{cuda12.2-pip => cuda12.3-pip}/features (100%) diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json index ae4b06a39..e5d8e9e98 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda11.8-pip/devcontainer.json @@ -11,7 +11,7 @@ "hostRequirements": {"gpu": "optional"}, "features": { "./features/src/ucx": {"version": "1.14.1"}, - "./features/src/cuda": {"installcuDNN": true}, + "./features/src/cuda": {"version": "11.8", "installcuBLAS": true, "installcuDNN": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true}, "./features/src/rapids-build-utils": {} }, "overrideFeatureInstallOrder": [ diff --git a/.devcontainer/cuda12.0-pip/devcontainer.json b/.devcontainer/cuda12.0-pip/devcontainer.json index 9dd89cb4d..87e8cb824 100644 --- a/.devcontainer/cuda12.0-pip/devcontainer.json +++ b/.devcontainer/cuda12.0-pip/devcontainer.json @@ -11,7 +11,7 @@ "hostRequirements": {"gpu": "optional"}, "features": { "./features/src/ucx": {"version": "1.14.1"}, - "./features/src/cuda": {"installcuDNN": true}, + "./features/src/cuda": {"version": "12.0", "installcuBLAS": true, "installcuDNN": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true}, "./features/src/rapids-build-utils": {} }, "overrideFeatureInstallOrder": [ diff --git a/.devcontainer/cuda12.2-pip/devcontainer.json b/.devcontainer/cuda12.3-pip/devcontainer.json similarity index 85% rename from .devcontainer/cuda12.2-pip/devcontainer.json rename to .devcontainer/cuda12.3-pip/devcontainer.json index a5824870e..74254b07a 100644 --- a/.devcontainer/cuda12.2-pip/devcontainer.json +++ b/.devcontainer/cuda12.3-pip/devcontainer.json @@ -3,15 +3,15 @@ "context": "${localWorkspaceFolder}/.devcontainer", "dockerfile": "${localWorkspaceFolder}/.devcontainer/rapids.Dockerfile", "args": { - "CUDA": "12.2", + "CUDA": "12.3", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.04-cpp-llvm16-cuda12.2-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.04-cpp-llvm16-cuda12.3-ubuntu22.04" } }, "hostRequirements": {"gpu": "optional"}, "features": { "./features/src/ucx": {"version": "1.14.1"}, - "./features/src/cuda": {"installcuDNN": true}, + "./features/src/cuda": {"version": "12.3", "installcuBLAS": true, "installcuDNN": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true}, "./features/src/rapids-build-utils": {} }, "overrideFeatureInstallOrder": [ @@ -19,7 +19,7 @@ "./features/src/cuda", "./features/src/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.2-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.3-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/devcontainers,type=bind,consistency=consistent", @@ -38,9 +38,9 @@ "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.2-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/.devcontainer/cuda12.2-pip/features/src/utils/opt/devcontainer/bin,target=/opt/devcontainer/bin,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/.devcontainer/cuda12.2-pip/features/src/rapids-build-utils/opt/rapids-build-utils,target=/opt/rapids-build-utils,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.3-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/.devcontainer/cuda12.3-pip/features/src/utils/opt/devcontainer/bin,target=/opt/devcontainer/bin,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/.devcontainer/cuda12.3-pip/features/src/rapids-build-utils/opt/rapids-build-utils,target=/opt/rapids-build-utils,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.devcontainer/cuda12.2-pip/features b/.devcontainer/cuda12.3-pip/features similarity index 100% rename from .devcontainer/cuda12.2-pip/features rename to .devcontainer/cuda12.3-pip/features From ff021e1523e96ee13576d64dccb3a2d6b723ddf3 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 14 Feb 2024 18:35:02 -0800 Subject: [PATCH 55/59] make conda container use CUDA 12.3 --- .../devcontainer.json | 10 +++++----- .../{cuda12.2-conda => cuda12.3-conda}/features | 0 2 files changed, 5 insertions(+), 5 deletions(-) rename .devcontainer/{cuda12.2-conda => cuda12.3-conda}/devcontainer.json (91%) rename .devcontainer/{cuda12.2-conda => cuda12.3-conda}/features (100%) diff --git a/.devcontainer/cuda12.2-conda/devcontainer.json b/.devcontainer/cuda12.3-conda/devcontainer.json similarity index 91% rename from .devcontainer/cuda12.2-conda/devcontainer.json rename to .devcontainer/cuda12.3-conda/devcontainer.json index 6ab96272f..83a80faeb 100644 --- a/.devcontainer/cuda12.2-conda/devcontainer.json +++ b/.devcontainer/cuda12.3-conda/devcontainer.json @@ -3,7 +3,7 @@ "context": "${localWorkspaceFolder}/.devcontainer", "dockerfile": "${localWorkspaceFolder}/.devcontainer/rapids.Dockerfile", "args": { - "CUDA": "12.2", + "CUDA": "12.3", "PYTHON_PACKAGE_MANAGER": "conda", "BASE": "rapidsai/devcontainers:24.04-cpp-mambaforge-ubuntu22.04" } @@ -15,7 +15,7 @@ "overrideFeatureInstallOrder": [ "./features/src/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.2-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.3-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/devcontainers,type=bind,consistency=consistent", @@ -35,9 +35,9 @@ "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.2-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/.devcontainer/cuda12.2-conda/features/src/utils/opt/devcontainer/bin,target=/opt/devcontainer/bin,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/.devcontainer/cuda12.2-conda/features/src/rapids-build-utils/opt/rapids-build-utils,target=/opt/rapids-build-utils,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.3-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/.devcontainer/cuda12.3-conda/features/src/utils/opt/devcontainer/bin,target=/opt/devcontainer/bin,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/.devcontainer/cuda12.3-conda/features/src/rapids-build-utils/opt/rapids-build-utils,target=/opt/rapids-build-utils,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.devcontainer/cuda12.2-conda/features b/.devcontainer/cuda12.3-conda/features similarity index 100% rename from .devcontainer/cuda12.2-conda/features rename to .devcontainer/cuda12.3-conda/features From 875eec4bc690cb17f968a40c44ebfd21f767f329 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 16 Feb 2024 02:12:05 -0800 Subject: [PATCH 56/59] build CUDA 12.0 and 12.2 now that we have pip containers --- .github/workflows/build-all-rapids-repos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 2c02374a2..86299ec22 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -30,7 +30,7 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@fix/devcontainer-json-location with: arch: '["amd64"]' - cuda: '["12.0"]' + cuda: '["12.0", "12.2"]' node_type: cpu32 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY extra-repo-deploy-key-2: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY From cd842ae38b9e284640c4e76715e1b7d55622c56f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 16 Feb 2024 02:13:59 -0800 Subject: [PATCH 57/59] 12.3 -> 12.2 --- .../devcontainer.json | 10 +++++----- .../{cuda12.3-conda => cuda12.2-conda}/features | 0 .../devcontainer.json | 14 +++++++------- .../{cuda12.3-pip => cuda12.2-pip}/features | 0 4 files changed, 12 insertions(+), 12 deletions(-) rename .devcontainer/{cuda12.3-conda => cuda12.2-conda}/devcontainer.json (91%) rename .devcontainer/{cuda12.3-conda => cuda12.2-conda}/features (100%) rename .devcontainer/{cuda12.3-pip => cuda12.2-pip}/devcontainer.json (88%) rename .devcontainer/{cuda12.3-pip => cuda12.2-pip}/features (100%) diff --git a/.devcontainer/cuda12.3-conda/devcontainer.json b/.devcontainer/cuda12.2-conda/devcontainer.json similarity index 91% rename from .devcontainer/cuda12.3-conda/devcontainer.json rename to .devcontainer/cuda12.2-conda/devcontainer.json index 83a80faeb..6ab96272f 100644 --- a/.devcontainer/cuda12.3-conda/devcontainer.json +++ b/.devcontainer/cuda12.2-conda/devcontainer.json @@ -3,7 +3,7 @@ "context": "${localWorkspaceFolder}/.devcontainer", "dockerfile": "${localWorkspaceFolder}/.devcontainer/rapids.Dockerfile", "args": { - "CUDA": "12.3", + "CUDA": "12.2", "PYTHON_PACKAGE_MANAGER": "conda", "BASE": "rapidsai/devcontainers:24.04-cpp-mambaforge-ubuntu22.04" } @@ -15,7 +15,7 @@ "overrideFeatureInstallOrder": [ "./features/src/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.3-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.2-envs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/devcontainers,type=bind,consistency=consistent", @@ -35,9 +35,9 @@ "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.3-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/.devcontainer/cuda12.3-conda/features/src/utils/opt/devcontainer/bin,target=/opt/devcontainer/bin,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/.devcontainer/cuda12.3-conda/features/src/rapids-build-utils/opt/rapids-build-utils,target=/opt/rapids-build-utils,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.2-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/.devcontainer/cuda12.2-conda/features/src/utils/opt/devcontainer/bin,target=/opt/devcontainer/bin,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/.devcontainer/cuda12.2-conda/features/src/rapids-build-utils/opt/rapids-build-utils,target=/opt/rapids-build-utils,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.devcontainer/cuda12.3-conda/features b/.devcontainer/cuda12.2-conda/features similarity index 100% rename from .devcontainer/cuda12.3-conda/features rename to .devcontainer/cuda12.2-conda/features diff --git a/.devcontainer/cuda12.3-pip/devcontainer.json b/.devcontainer/cuda12.2-pip/devcontainer.json similarity index 88% rename from .devcontainer/cuda12.3-pip/devcontainer.json rename to .devcontainer/cuda12.2-pip/devcontainer.json index 7447e3078..2dffc1cfd 100644 --- a/.devcontainer/cuda12.3-pip/devcontainer.json +++ b/.devcontainer/cuda12.2-pip/devcontainer.json @@ -3,15 +3,15 @@ "context": "${localWorkspaceFolder}/.devcontainer", "dockerfile": "${localWorkspaceFolder}/.devcontainer/rapids.Dockerfile", "args": { - "CUDA": "12.3", + "CUDA": "12.2", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.04-cpp-cuda12.3-ubuntu22.04" + "BASE": "rapidsai/devcontainers:24.04-cpp-cuda12.2-ubuntu22.04" } }, "hostRequirements": {"gpu": "optional"}, "features": { "./features/src/ucx": {"version": "1.14.1"}, - "./features/src/cuda": {"version": "12.3", "installcuBLAS": true, "installcuDNN": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true}, + "./features/src/cuda": {"version": "12.2", "installcuBLAS": true, "installcuDNN": true, "installcuSOLVER": true, "installcuRAND": true, "installcuSPARSE": true}, "./features/src/rapids-build-utils": {} }, "overrideFeatureInstallOrder": [ @@ -19,7 +19,7 @@ "./features/src/cuda", "./features/src/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.3-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.2-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,ucxx,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/devcontainers,type=bind,consistency=consistent", @@ -38,9 +38,9 @@ "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.3-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/.devcontainer/cuda12.3-pip/features/src/utils/opt/devcontainer/bin,target=/opt/devcontainer/bin,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/.devcontainer/cuda12.3-pip/features/src/rapids-build-utils/opt/rapids-build-utils,target=/opt/rapids-build-utils,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.2-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/.devcontainer/cuda12.2-pip/features/src/utils/opt/devcontainer/bin,target=/opt/devcontainer/bin,type=bind,consistency=consistent", + "source=${localWorkspaceFolder}/.devcontainer/cuda12.2-pip/features/src/rapids-build-utils/opt/rapids-build-utils,target=/opt/rapids-build-utils,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.devcontainer/cuda12.3-pip/features b/.devcontainer/cuda12.2-pip/features similarity index 100% rename from .devcontainer/cuda12.3-pip/features rename to .devcontainer/cuda12.2-pip/features From e0ea956d364918308f8ab74fdd27fbd90bddb1b7 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 16 Feb 2024 02:15:29 -0800 Subject: [PATCH 58/59] set -DBUILD_SHARED_LIBS=ON --- .github/workflows/build-all-rapids-repos.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 86299ec22..4ce0551fb 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -42,6 +42,7 @@ jobs: build-all \ --verbose \ -j$(nproc --ignore=1) \ + -DBUILD_SHARED_LIBS=ON \ -DBUILD_TESTS=ON \ -DBUILD_BENCHMARKS=ON \ -DBUILD_ANN_BENCH=ON \ From bd5001ccdf555d583d0575f71d159e5a67903f08 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 20 Feb 2024 13:16:02 -0800 Subject: [PATCH 59/59] revert changes to feature-matrix and image-matrix actions [skip-ci] --- .github/actions/feature-matrix/action.yml | 3 +++ .github/actions/image-matrix/action.yml | 3 +++ .github/workflows/build-all-rapids-repos.yml | 3 ++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/actions/feature-matrix/action.yml b/.github/actions/feature-matrix/action.yml index 3252165ae..f59855064 100644 --- a/.github/actions/feature-matrix/action.yml +++ b/.github/actions/feature-matrix/action.yml @@ -30,6 +30,9 @@ runs: files: | features/src/** features/test/** + .github/workflows/test.yml + .github/actions/feature-matrix/action.sh + .github/actions/feature-matrix/action.yml .github/workflows/build-and-test-feature.yml .github/actions/build-and-test-feature/action.yml diff --git a/.github/actions/image-matrix/action.yml b/.github/actions/image-matrix/action.yml index 65c71ec29..256f37b12 100644 --- a/.github/actions/image-matrix/action.yml +++ b/.github/actions/image-matrix/action.yml @@ -39,6 +39,9 @@ runs: matrix.yml .github/**/*windows* .github/actions/build-linux-image/action.yml + .github/actions/image-matrix/action.sh + .github/actions/image-matrix/action.yml + .github/workflows/test.yml .github/workflows/build-and-test-image.yml - name: Report changes diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 4ce0551fb..06cc28f38 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -36,6 +36,7 @@ jobs: extra-repo-deploy-key-2: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY build_command: | sccache -z; + sccache --show-adv-stats; clone-all --verbose --clone-upstream --no-update-env; rapids-generate-scripts; rapids-make-${PYTHON_PACKAGE_MANAGER}-env; @@ -49,4 +50,4 @@ jobs: -DBUILD_PRIMS_BENCH=ON \ $([ "pip" = ${PYTHON_PACKAGE_MANAGER} ] && echo -DUSE_LIBARROW_FROM_PYARROW=ON || echo) \ ; - sccache -s; + sccache --show-adv-stats;