From 4c3d6929f7a9e3721a032c790791c51f501a2d80 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Tue, 28 Jan 2025 18:58:13 +0000 Subject: [PATCH 1/3] .github/zephyr: simplify and fix the git describe performance hack Fixes the git describe/tag performance hack added in commit 23284780d7cb (".github/zephyr.yml: fix tags missing from `git -C zephyr/ describe`") which worked for an amazingly long time (1.5 year) but apparently ran its course. Git version 2.48 apparently does not like it anymore. Replace it with something slower but simpler and safer. Should fix build reproducibility issue #9797, much more details there. Also fixes commit 4bc6488b24e4 (".github/zephyr: de-hardcode the name of the zephyr remote") Signed-off-by: Marc Herbert --- .github/workflows/zephyr.yml | 49 ++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/.github/workflows/zephyr.yml b/.github/workflows/zephyr.yml index 3fe3e8982d58..d093ddc96150 100644 --- a/.github/workflows/zephyr.yml +++ b/.github/workflows/zephyr.yml @@ -154,8 +154,8 @@ jobs: time west update --narrow --fetch-opt=--filter=tree:0 - name: select zephyr revision + working-directory: ${{ github.workspace }}/workspace run: | - cd workspace if [ 'mnfst' = '${{ matrix.zephyr_revision }}' ]; then rem_rev=$(git -C zephyr rev-parse HEAD) else @@ -170,6 +170,11 @@ jobs: time west update --narrow --fetch-opt=--filter=tree:0 fi + # Get some tags to fix `git describe` hence BUILD_VERSION, etc. + # Keep in sync with build-windows below + - name: Fetch tags for git describe + working-directory: ${{ github.workspace }}/workspace/zephyr + run: | # Because we used git tricks to speed things up, we now have two git # problems: # @@ -179,19 +184,24 @@ jobs: # does not use --tags. # # 2. west fetches using the remote URL, not the remote name. So remote - # branches (if any) are missing from --decorate below - # - # => an "empty" and quick fetch _with_ a refspec and the remote name fixes - # both issues in no time. - - cd zephyr - time git fetch --filter=tree:0 "$(git remote |head -n1)" "$rem_rev":_branch_placeholder - git branch -D _branch_placeholder + # branches are missing from --decorate below. Cosmetic but annoying; + # especially in the "zmain" case. set -x + # Fix problem 2. Do NOT assume anything about remote names: nothing is guaranteed. + _zurl=$(west list -f '{url}' zephyr) + # Use an ugly remote name to avoid a collision + git remote add sof_zep_rem "$_zurl" + time git fetch --filter=tree:0 sof_zep_rem "$rem_rev" + # Fix problem 1. Indirectly fetches useless branches but is very quick thanks to + # the --filter + time git fetch --filter=tree:0 --tags sof_zep_rem + west list west status git log --oneline -n 5 --decorate --graph --no-abbrev-commit + git describe --long --always --dirty + git describe --long --always --dirty --tags # Not strictly necessary but saves a lot of scrolling in the next step # Caching a 12G image is unfortunately not possible: @@ -327,12 +337,25 @@ jobs: west init -l sof west update --narrow --fetch-opt=--filter=tree:0 - # Get some tags to fix `git describe`, see build-linux comments above. - cd zephyr + # Get some tags to fix `git describe` etc., see detailed build-linux comments above. + - name: Fetch tags for git describe + working-directory: ${{ github.workspace }}/workspace/zephyr + # Keep in sync with build-linux above + run: | + # Set-PSDebug -Trace 2 $_rev = "$(git rev-parse HEAD)" - git fetch --filter=tree:0 "$(west list -f '{url}' zephyr)" "${_rev}:_branch_placeholder" - git branch -D _branch_placeholder + $_zurl = "$(west list -f '{url}' zephyr)" + git remote add sof_zep_rem "${_zurl}" + # Unlike Linux above, hardcode "main" for now. Will make no difference most + # of the time but keeps this bit consistent with Linux, tested and ready to use. + git fetch --filter=tree:0 sof_zep_rem main + git fetch --filter=tree:0 --tags sof_zep_rem + west list + west status + git log --oneline -n 5 --decorate --graph --no-abbrev-commit + git describe --long --always --dirty + git describe --long --always --dirty --tags # Call Setup Python again to save the PIP packages in cache - name: Setup Python From 7f4327942abeda2aca07e1583765dcc33eed2d54 Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Tue, 28 Jan 2025 20:22:46 +0000 Subject: [PATCH 2/3] .github/zephyr: build LNL and PTL on Windows too This matters because the build reproducibility test is based on the list of Windows builds. Also: - Bump -d(ebug) build to something newer: LNL - Group MTL and LNL to save one line Signed-off-by: Marc Herbert --- .github/workflows/zephyr.yml | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/zephyr.yml b/.github/workflows/zephyr.yml index d093ddc96150..d02d67861560 100644 --- a/.github/workflows/zephyr.yml +++ b/.github/workflows/zephyr.yml @@ -114,8 +114,10 @@ jobs: IPC_platforms: [ # - IPC3 default imx8 imx8x imx8m imx8ulp, - # - IPC4 default - mtl, lnl, ptl, + # - IPC4 default, released + mtl lnl, + # active development + ptl, # Temporary testbed for Zephyr development. tgl tgl-h, ] @@ -126,7 +128,7 @@ jobs: # specify one extra -d combination without affecting the main matrix - build_opts: -d zephyr_revision: mnfst - IPC_platforms: mtl + IPC_platforms: lnl # This is "duplication of effort" but it makes sure no one # breaks --all, see for instance #9262 and previous commit. @@ -243,15 +245,18 @@ jobs: platforms: [ # - IPC3 default imx8 imx8x imx8m imx8ulp, - # - IPC4 default - mtl, + # - IPC4 default, released + mtl lnl, + # active development + ptl, + # legacy tgl tgl-h, ] build_opts: [""] # Sparse matrices are complicated, see comments on Linux matrix above. include: - build_opts: -d - platforms: mtl + platforms: lnl steps: @@ -451,10 +456,12 @@ jobs: run: | diffs=0; ls -l - # Check not too much is missing (it happened!) + # Look for a few samples to + # check not too much is missing (it happened!) # http://mywiki.wooledge.org/ParsingLs - for regdir in 'linux-build *mtl' \ - 'windows-build *mtl' \ + for regdir in 'linux-build *-d *.*' \ + 'linux-build *.*lnl.*' \ + 'windows-build *.*mtl.*' \ 'windows-build *tgl tgl-h'; do find . -maxdepth 1 | grep -q "\./${regdir}\$" || { >&2 printf 'Missing %s\n' "${regdir}"; exit 1; } From 5e51eb396852f5022d23c9f79c43fad1d41886dc Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Tue, 28 Jan 2025 20:28:54 +0000 Subject: [PATCH 3/3] .github: extract new sof-docs.yml from pull-request.yml More progress towards trimming down pull-request.yml Important benefit and "secret" agenda: remove the constantly failing sof-docs from daily builds, which will make them green again. Green daily builds are important to quickly spot regressions like for instance #9797 Signed-off-by: Marc Herbert --- .github/workflows/pull-request.yml | 38 -------------------- .github/workflows/sof-docs.yml | 57 ++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/sof-docs.yml diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 3f26a1093541..32a5f61ba301 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -63,44 +63,6 @@ jobs: ninja -C docbuild -v doc - # This is unfortunately a mix of sof-docs/.github/ + the previous job - # above, duplicating a bit of both: please keep in sync with both. We - # build sof-docs here too because waiting for sof-docs CI to find - # regressions introduced in sof.git is very time-consuming, see - # example in - # https://github.com/thesofproject/sof/pull/5731#issuecomment-1175630147 - sof-docs: - runs-on: ubuntu-20.04 # sof-docs is still stuck to this for now - - steps: - - uses: actions/checkout@v4 - with: - filter: 'tree:0' - - - name: apt-get update - run: sudo apt-get update - - - name: apt-get install - run: sudo apt-get -y install - doxygen make default-jre graphviz cmake ninja-build - - - name: doxygen - run: cmake -GNinja -S doc/ -B doxybuild/ && ninja -C doxybuild/ -v doc - - # Nested git clone, this is fine - - name: clone sof-docs - run: git clone https://github.com/thesofproject/sof-docs sof-docs/ - - - name: PATH += .local/bin - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: 'pip install -r sof-docs/scripts/requirements.txt' - run: pip install -r sof-docs/scripts/requirements.txt - - - name: build sof-docs - run: make -C sof-docs/ html SOF_DOC_BUILD="$(pwd)"/doxybuild/ - - # This is a bit redundant with the other jobs below and with the (much # faster!) installer[.yml] but it may differ in which platforms are # built. This makes sure platforms without any open-source toolchain diff --git a/.github/workflows/sof-docs.yml b/.github/workflows/sof-docs.yml new file mode 100644 index 000000000000..da8fd71b4ab7 --- /dev/null +++ b/.github/workflows/sof-docs.yml @@ -0,0 +1,57 @@ +--- +# yamllint disable-line rule:truthy +on: + push: + branches: + - 'main' + - 'stable-**' + - '**-stable' + pull_request: + branches: + - 'main' + - 'stable-**' + - '**-stable' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + # Allows to call this forkflow from other workflows + workflow_call: + +jobs: + + # This is unfortunately a mix of sof-docs/.github/ + pull-request.yml#doxygen + # duplicating a bit of both: please keep in sync with both. We + # build sof-docs here too because waiting for sof-docs CI to find + # regressions introduced in sof.git is very time-consuming, see + # example in + # https://github.com/thesofproject/sof/pull/5731#issuecomment-1175630147 + sof-docs: + runs-on: ubuntu-20.04 # sof-docs is still stuck to this for now + + steps: + - uses: actions/checkout@v4 + with: + filter: 'tree:0' + + - name: apt-get update + run: sudo apt-get update + + - name: apt-get install + run: sudo apt-get -y install + doxygen make default-jre graphviz cmake ninja-build + + - name: doxygen + run: cmake -GNinja -S doc/ -B doxybuild/ && ninja -C doxybuild/ -v doc + + # Nested git clone, this is fine + - name: clone sof-docs + run: git clone https://github.com/thesofproject/sof-docs sof-docs/ + + - name: PATH += .local/bin + run: echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: 'pip install -r sof-docs/scripts/requirements.txt' + run: pip install -r sof-docs/scripts/requirements.txt + + - name: build sof-docs + run: make -C sof-docs/ html SOF_DOC_BUILD="$(pwd)"/doxybuild/