diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a9f725780da0..21db307dabc1e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -446,19 +446,41 @@ check-polkadot-companion-status: script: - ./.maintain/gitlab/check_polkadot_companion_status.sh -check-polkadot-companion-build: - stage: build +.check-companion-build: &check-companion-build + stage: test <<: *docker-env <<: *test-refs-no-trigger - needs: - - job: test-linux-stable-int - artifacts: false script: - - ./.maintain/gitlab/check_polkadot_companion_build.sh + - ./.maintain/gitlab/check_companion_build.sh "$COMPANION_ORG" "$COMPANION_REPO" "$COMPANION_BUILD" "$COMPANION_DEPENDENCY" after_script: - - cd polkadot && git rev-parse --abbrev-ref HEAD + - cd "$COMPANION_REPO" && git rev-parse --abbrev-ref HEAD allow_failure: true +check-polkadot-companion-build: + <<: *check-companion-build + variables: + COMPANION_ORG: paritytech + COMPANION_REPO: polkadot + COMPANION_DEPENDENCY: paritytech/grandpa-bridge-gadget + allow_failure: true + +check-cumulus-companion-build: + <<: *check-companion-build + variables: + COMPANION_ORG: paritytech + COMPANION_REPO: cumulus + COMPANION_BUILD: "cargo check --all-targets --workspace" + COMPANION_DEPENDENCY: paritytech/polkadot + allow_failure: true + +check-beefy-companion-build: + <<: *check-companion-build + variables: + COMPANION_ORG: paritytech + COMPANION_REPO: grandpa-bridge-gadget + COMPANION_BUILD: "cargo check --all-targets --workspace" + allow_failure: true + test-browser-node: stage: build <<: *docker-env diff --git a/.maintain/common/lib.sh b/.maintain/common/lib.sh index ce6c566d799ab..e0104f5da9976 100755 --- a/.maintain/common/lib.sh +++ b/.maintain/common/lib.sh @@ -2,6 +2,18 @@ api_base="https://api.github.com/repos" + +set_github_token(){ + # These will exist if the function is called in Gitlab. + # If the function's called in Github, we should have GITHUB_ACCESS_TOKEN set + # already. + if [ -n "$GITHUB_RELEASE_TOKEN" ]; then + GITHUB_TOKEN="$GITHUB_RELEASE_TOKEN" + elif [ -n "$GITHUB_PR_TOKEN" ]; then + GITHUB_TOKEN="$GITHUB_PR_TOKEN" + fi +} + # Function to take 2 git tags/commits and get any lines from commit messages # that contain something that looks like a PR reference: e.g., (#1234) sanitised_git_logs(){ @@ -66,15 +78,7 @@ has_label(){ repo="$1" pr_id="$2" label="$3" - - # These will exist if the function is called in Gitlab. - # If the function's called in Github, we should have GITHUB_ACCESS_TOKEN set - # already. - if [ -n "$GITHUB_RELEASE_TOKEN" ]; then - GITHUB_TOKEN="$GITHUB_RELEASE_TOKEN" - elif [ -n "$GITHUB_PR_TOKEN" ]; then - GITHUB_TOKEN="$GITHUB_PR_TOKEN" - fi + set_github_token out=$(curl -H "Authorization: token $GITHUB_TOKEN" -s "$api_base/$repo/pulls/$pr_id") [ -n "$(echo "$out" | tr -d '\r\n' | jq ".labels | .[] | select(.name==\"$label\")")" ] @@ -115,3 +119,23 @@ has_runtime_changes() { return 1 fi } + +# Given an origin repo & PR,and a repository, will check if the given PR has a +# companion PR in the target repo +get_companion() { + origin_repo="$1" + pr_num="$2" + companion_repo="$3" + set_github_token + github_header="Authorization: token ${GITHUB_TOKEN}" + # get the last reference to a pr in the target repo + pr_body=$( + curl -sSL -H "${github_header}" "${api_base}/$origin_repo/pulls/$pr_num" | \ + jq -r '.body' + ) + + echo "${pr_body}" | sed -n -r \ + -e "s;^.*[Cc]ompanion.*$companion_repo#([0-9]+).*$;\1;p" \ + -e "s;^.*[Cc]ompanion.*https://github.com/$companion_repo/pull/([0-9]+).*$;\1;p" \ + | tail -n 1 +} diff --git a/.maintain/gitlab/check_companion_build.sh b/.maintain/gitlab/check_companion_build.sh new file mode 100755 index 0000000000000..2d6a2f896e198 --- /dev/null +++ b/.maintain/gitlab/check_companion_build.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# +# check if a pr is compatible with companion pr or master if not available +# +# to override one that was just mentioned mark companion pr in the body of the +# pr like +# +# $REPO companion: $ORGANISATION/$REPO#567 +# +# $ORGANISATION and $REPO are set using $1 and $2. You can also specify a custom +# build command with $3 +# Every argument after $3 is for specifying *additional* dependencies this +# project has that depend on substrate, which might also have companion PRs. + +# Example: Cumulus relies on both substrate and polkadot. If this substrate PR +# requires a companion build on polkadot, when we are testing that cumulus builds +# with this commit of substrate, we *also* need to clone & patch polkadot, and tell +# cumulus to use this cloned+patched version, else the build is guaranteed to fail +# (since it doesn't have the changes to polkadot that were required in the polkadot +# companion PR) + +# So invoke this script like (arguments in [] indicate optional arguments) +# ./check_companion_build.sh paritytech cumulus ['cargo test --release'] [paritytech/polkadot] + +set -e + +# Include the common functions library +#shellcheck source=../common/lib.sh +. "$(dirname "${0}")/../common/lib.sh" + +ORGANISATION=$1 +REPO=$2 +BUILDSTRING=${3:-cargo test --workspace --release} +DEPS=("${@:4}") +SUBSTRATE_DIR="$(pwd)" + +boldprint () { printf "|\n| \033[1m%s\033[0m\n|\n" "${@}"; } +boldcat () { printf "|\n"; while read -r l; do printf "| \033[1m%s\033[0m\n" "${l}"; done; printf "|\n" ; } + +boldcat <<-EOT + + +check_${REPO}_companion_build +============================== + +this job checks if there is a string in the description of the pr like + +$REPO companion: $ORGANISATION/$REPO#567 + + +it will then run cargo check from this ${REPO}'s branch with substrate code +from this pull request. otherwise, it will uses master instead + + +EOT + +# Set the user name and email to make merging work +git config --global user.name 'CI system' +git config --global user.email '<>' + +# Merge master into our branch before building to make sure we don't miss +# any commits that are required. +git fetch --depth 100 origin +git merge origin/master + +# Clone the current master branch into a local directory +# NOTE: we need to pull enough commits to be able to find a common +# ancestor for successfully performing merges below. +git clone --depth 20 "https://github.com/${ORGANISATION}/${REPO}.git" + +cd "$REPO" + +# either it's a pull request then check for a companion otherwise use +# master +# shellcheck disable=SC2003 +if expr match "${CI_COMMIT_REF_NAME}" '^[0-9]\+$' >/dev/null +then + boldprint "this is pull request no ${CI_COMMIT_REF_NAME}" + pr_companion="$(get_companion "paritytech/substrate" "$CI_COMMIT_REF_NAME" "$ORGANISATION/$REPO")" + if [ "$pr_companion" ] + then + boldprint "companion pr specified/detected: #${pr_companion}" + git fetch origin "refs/pull/${pr_companion}/head:pr/${pr_companion}" + git checkout "pr/${pr_companion}" + git merge origin/master + else + boldprint "no companion branch found - building ${REPO}:master" + fi + + # If this repo has any additional dependencies, we should check whether they + # are mentioned as companions as well. If they are, we patch this repo to + # use that companion build as well. See example at top of this script + # Note: Will only work with repos supported by diener + declare -A diener_commands + diener_commands=() + diener_commands["paritytech/polkadot"]='--polkadot' + diener_commands["paritytech/substrate"]='--substrate' + diener_commands["paritytech/grandpa-bridge-gadget"]='--beefy' + + for dep in "${DEPS[@]}"; do + dep_companion="$(get_companion "paritytech/substrate" "$CI_COMMIT_REF_NAME" "$dep")" + if [ "$dep_companion" ]; then + echo "Companion PR found for $dep, need to patch $REPO to use that" + git clone --depth 20 "https://github.com/$dep.git" "$dep" + git -C "$dep" fetch origin "refs/pull/${dep_companion}/head:pr/${dep_companion}" + git -C "$dep" checkout "pr/${dep_companion}" + git -C "$dep" merge origin/master + # Tell this dependency to use the version of substrate in this PR + diener patch --crates-to-patch "$SUBSTRATE_DIR" --substrate --path "$dep/Cargo.toml" + # then we tell this repository to point at our locally cloned dependency + diener patch --crates-to-patch "$dep" "${diener_commands[$dep]}" --path "Cargo.toml" + fi + + done + +else + boldprint "this is not a pull request - building ${REPO}:master" +fi + +# Test pr or master branch with this Substrate commit. +diener patch --crates-to-patch ".." --substrate --path "Cargo.toml" + +eval "$BUILDSTRING" diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh deleted file mode 100755 index 72bfaf7151522..0000000000000 --- a/.maintain/gitlab/check_polkadot_companion_build.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env sh -# -# check if a pr is compatible with polkadot companion pr or master if not -# available -# -# to override one that was just mentioned mark companion pr in the body of the -# polkadot pr like -# -# polkadot companion: paritytech/polkadot#567 -# - -set -e - -github_api_substrate_pull_url="https://api.github.com/repos/paritytech/substrate/pulls" -# use github api v3 in order to access the data without authentication -github_header="Authorization: token ${GITHUB_PR_TOKEN}" - -boldprint () { printf "|\n| \033[1m${@}\033[0m\n|\n" ; } -boldcat () { printf "|\n"; while read l; do printf "| \033[1m${l}\033[0m\n"; done; printf "|\n" ; } - - - -boldcat <<-EOT - - -check_polkadot_companion_build -============================== - -this job checks if there is a string in the description of the pr like - -polkadot companion: paritytech/polkadot#567 - - -it will then run cargo check from this polkadot's branch with substrate code -from this pull request. otherwise, it will uses master instead - - -EOT - -# Set the user name and email to make merging work -git config --global user.name 'CI system' -git config --global user.email '<>' - -# Merge master into our branch before building Polkadot to make sure we don't miss -# any commits that are required by Polkadot. -git fetch --depth 100 origin -git merge origin/master - -# Clone the current Polkadot master branch into ./polkadot. -# NOTE: we need to pull enough commits to be able to find a common -# ancestor for successfully performing merges below. -git clone --depth 20 https://github.com/paritytech/polkadot.git - -cd polkadot - -# either it's a pull request then check for a companion otherwise use -# polkadot:master -if expr match "${CI_COMMIT_REF_NAME}" '^[0-9]\+$' >/dev/null -then - boldprint "this is pull request no ${CI_COMMIT_REF_NAME}" - - pr_data_file="$(mktemp)" - # get the last reference to a pr in polkadot - curl -sSL -H "${github_header}" -o "${pr_data_file}" \ - "${github_api_substrate_pull_url}/${CI_COMMIT_REF_NAME}" - - pr_body="$(sed -n -r 's/^[[:space:]]+"body": (".*")[^"]+$/\1/p' "${pr_data_file}")" - - pr_companion="$(echo "${pr_body}" | sed -n -r \ - -e 's;^.*[Cc]ompanion.*paritytech/polkadot#([0-9]+).*$;\1;p' \ - -e 's;^.*[Cc]ompanion.*https://github.com/paritytech/polkadot/pull/([0-9]+).*$;\1;p' \ - | tail -n 1)" - - if [ "${pr_companion}" ] - then - boldprint "companion pr specified/detected: #${pr_companion}" - git fetch origin refs/pull/${pr_companion}/head:pr/${pr_companion} - git checkout pr/${pr_companion} - git merge origin/master - else - boldprint "no companion branch found - building polkadot:master" - fi - rm -f "${pr_data_file}" -else - boldprint "this is not a pull request - building polkadot:master" -fi - -# Patch all Substrate crates in Polkadot -diener patch --crates-to-patch ../ --substrate --path Cargo.toml - -# We need to update specifically our patched Substrate crates so that other -# crates that depend on them (e.g. Polkadot, BEEFY) use this unified version -# NOTE: There's no way to only update patched crates, so we use a heuristic -# of updating a crucial Substrate crate (`sp-core`) to minimize the impact of -# updating unrelated dependencies -cargo update -p sp-core - -# Test Polkadot pr or master branch with this Substrate commit. -time cargo test --workspace --release --verbose --features=runtime-benchmarks