From b61bb353f8f8ef7a5a2ea64c8557179756f8fa4b Mon Sep 17 00:00:00 2001 From: Reinhold Gschweicher Date: Mon, 2 Jun 2025 11:50:04 +0200 Subject: [PATCH 01/10] check-urls: add executable bit with shebang header --- maintenance/check-urls.py | 3 +++ 1 file changed, 3 insertions(+) mode change 100644 => 100755 maintenance/check-urls.py diff --git a/maintenance/check-urls.py b/maintenance/check-urls.py old mode 100644 new mode 100755 index 1b201ee781..bb55dc2796 --- a/maintenance/check-urls.py +++ b/maintenance/check-urls.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python3 +# # -*- coding: utf-8 -*- + from os.path import dirname, abspath, join from glob import glob import re From fe3b59a6acb18da6a4edc68b6b892ca49a23fe95 Mon Sep 17 00:00:00 2001 From: Reinhold Gschweicher Date: Mon, 2 Jun 2025 13:42:09 +0200 Subject: [PATCH 02/10] check-urls: rework script to run on changed files --- maintenance/check-urls.py | 136 ++++++++++++++++++++++++++++++-------- 1 file changed, 109 insertions(+), 27 deletions(-) diff --git a/maintenance/check-urls.py b/maintenance/check-urls.py index bb55dc2796..c34991f8b3 100755 --- a/maintenance/check-urls.py +++ b/maintenance/check-urls.py @@ -1,12 +1,37 @@ #!/usr/bin/env python3 # # -*- coding: utf-8 -*- -from os.path import dirname, abspath, join +import argparse from glob import glob +import json +import pathlib +import os +from os.path import dirname, abspath, join import re +import signal import sys import pycurl +# failed projects, global to print on early abort with Ctrl+C +failed_projects: dict[str, list[str]] = dict() + + +def print_failed_projects(): + if failed_projects: + print("failed projects:") + print(json.dumps(failed_projects, indent=2)) + else: + print("all clear! no failed projects found") + + +def signal_handler(signal, frame): + # force exit as pycurl.error after KeyboardInterrupt is caught + # and then the next url is checked + print("You pressed Ctrl+C!") + print_failed_projects() + sys.exit(1) + + def getResponseStausCode(url): try: c = pycurl.Curl() @@ -21,36 +46,93 @@ def getResponseStausCode(url): except pycurl.error: return 999 -hunterDir = dirname(dirname(abspath(__file__))) -projectsDir = join(hunterDir, 'cmake', 'projects') -project = '' -if len(sys.argv) > 1: - project = sys.argv[1] +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "projects", + help="project names to process, used for local debugging", + nargs="*", + ) + parser.add_argument( + "-o", + "--output", + help="specify file to write failed projects and URLs to, default write to stdout", + type=str, + default="", + ) + args = parser.parse_args() + + repo_root = pathlib.Path(__file__).parent.parent + projects_dir = repo_root / "cmake" / "projects" -projectsFiles = join(projectsDir, project, '**', '*.cmake') + projects = set() + if args.projects: + if "all" in args.projects: + print("project 'all' specified, checking all projects") + project_hunter_files = projects_dir / "*" / "hunter.cmake" + for hunter_file in glob(project_hunter_files.as_posix(), recursive=False): + project = pathlib.Path(hunter_file).parent.name + projects.add(project) + else: + for project in args.projects: + if (projects_dir / project).is_dir(): + projects.add(project) + else: + raise RuntimeError( + f"provided project doesn't exist: {project}: expected dir: {projects_dir / project}" + ) + else: + try: + with open(os.environ.get("HOME") + "/files.json") as json_files: + files = json.load(json_files) + except IOError: + raise RuntimeError("Can't read changed files from files.json") -checkedFile = join(hunterDir, 'maintenance', 'checked.txt') -try: - checkedStream = open(checkedFile, "r+") - checked = checkedStream.readlines() -except FileNotFoundError: - checkedStream = open(checkedFile, "w") - checked = [] + p = re.compile("cmake/projects/([^/]+)") + for file in files: + if p.match(file): + project = p.match(file).group(1) + if (projects_dir / project).is_dir(): + projects.add(project) -projects = dict() + # override signal handler to make it possible to hard exit at Ctrl+C + # and print a status if we've found a failing URL or not yet + signal.signal(signal.SIGINT, signal_handler) -for projectFile in glob(projectsFiles, recursive=True): - with open(projectFile, "r") as file: - content = file.read() + for project in sorted(projects): + print() + print(f"checking project: {project}") + hunter_file = projects_dir / project / "hunter.cmake" + if not hunter_file.is_file(): + raise RuntimeError(f"hunter.cmake file not found: {hunter_file}") + with open(hunter_file, "r", encoding="utf-8") as file: + content = file.read() - entries = re.findall(r'hunter_add_version\s*\(\s*PACKAGE_NAME\s+"*(.*?)"*\s+VERSION\s+"*(.*?)"*\s+URL\s+"*(.*?)"*\s+SHA1\s+"*(.*?)"*\s+.*?\)', content, re.MULTILINE | re.DOTALL) - if len(entries): + entries = re.findall( + r'hunter_add_version\s*\(\s*PACKAGE_NAME\s+"*(.*?)"*\s+VERSION\s+"*(.*?)"*\s+URL\s+"*(.*?)"*\s+SHA1\s+"*(.*?)"*\s+.*?\)', + content, + re.MULTILINE | re.DOTALL, + ) + if len(entries) == 0: + raise RuntimeError( + f"no URLs found for project '{project}' in file: {hunter_file}" + ) for name, version, url, _ in entries: - if not any(url == x.rstrip('\n') for x in checked): - statusCode = getResponseStausCode(url) - print(str(statusCode) + ' ' + url) - if statusCode > 200: - checkedStream.write(str(statusCode) + ' ' + url + '\n') - -checkedStream.close() + status_code = getResponseStausCode(url) + print(f"{status_code} {url}") + if status_code > 200: + if project not in failed_projects: + failed_projects[project] = [] + failed_projects[project].append(f"{status_code} {url}") + if len(failed_projects) > 0: + print_failed_projects() + if args.output: + with open(args.output, "w", encoding="utf-8") as file: + json.dump(failed_projects, file, indent=2) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 0ab1c4fc8813e1d4ff4d7b91fae479ef6550988c Mon Sep 17 00:00:00 2001 From: Reinhold Gschweicher Date: Mon, 2 Jun 2025 15:29:34 +0200 Subject: [PATCH 03/10] CI: run check-urls.py --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 643ec3657b..94d470b402 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,10 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} + - name: Check changed projects for working URLs + run: | + python maintenance/check-urls.py + - name: Set matrix for subsequent build id: set-matrix run: | From e5ac171eb5b20208d27aeac6134e39ebe6c71df5 Mon Sep 17 00:00:00 2001 From: Reinhold Gschweicher Date: Mon, 2 Jun 2025 15:32:03 +0200 Subject: [PATCH 04/10] [WIP] eigen: add typo in URL --- cmake/projects/Eigen/hunter.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/projects/Eigen/hunter.cmake b/cmake/projects/Eigen/hunter.cmake index b2dba28986..2a03474fae 100644 --- a/cmake/projects/Eigen/hunter.cmake +++ b/cmake/projects/Eigen/hunter.cmake @@ -19,7 +19,7 @@ hunter_add_version( VERSION "3.4.0" URL - "https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz" + "https://gitlab.com/libeigens/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz" SHA1 d222db69a9e87d9006608e029d1039039f360b52 ) From f0b48f0393f95ed0d850e467ab4aeeb9c0464fa1 Mon Sep 17 00:00:00 2001 From: Reinhold Gschweicher Date: Mon, 2 Jun 2025 15:33:49 +0200 Subject: [PATCH 05/10] CI: install python3-pycurl --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94d470b402..5226d2bbd8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,7 @@ jobs: - name: Check changed projects for working URLs run: | + apt install -yq python3-pycurl python maintenance/check-urls.py - name: Set matrix for subsequent build From a80402e5bfe976fe6b4ece142aeed9e549f61367 Mon Sep 17 00:00:00 2001 From: Reinhold Gschweicher Date: Mon, 2 Jun 2025 15:34:42 +0200 Subject: [PATCH 06/10] CI: need to sudo, use apt-get --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5226d2bbd8..390d4a8c7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: - name: Check changed projects for working URLs run: | - apt install -yq python3-pycurl + sudo apt-get install -yq python3-pycurl python maintenance/check-urls.py - name: Set matrix for subsequent build From 6312906b1cf7108252317873bb500baf71543903 Mon Sep 17 00:00:00 2001 From: Reinhold Gschweicher Date: Mon, 2 Jun 2025 15:37:53 +0200 Subject: [PATCH 07/10] CI: check-urls as it own job --- .github/workflows/ci.yml | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 390d4a8c7b..0f0de1d1e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,29 @@ on: jobs: + check_urls: + runs-on: ubuntu-latest + steps: + - name: Git checkout + uses: actions/checkout@v4.1.1 + + - name: Manual dispatch, get project name from input + if: github.event_name == 'workflow_dispatch' + run: | + echo '["cmake/projects/${{ github.event.inputs.project }}/hunter.cmake"]' > ${HOME}/files.json + + - name: Get changed files and save them to ${HOME}/files.json + if: github.event_name != 'workflow_dispatch' + id: files + uses: lots0logs/gh-action-get-changed-files@2.2.2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Check changed projects for working URLs + run: | + sudo apt-get install -yq python3-pycurl + python maintenance/check-urls.py + set_matrix: runs-on: ubuntu-latest outputs: @@ -44,11 +67,6 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} - - name: Check changed projects for working URLs - run: | - sudo apt-get install -yq python3-pycurl - python maintenance/check-urls.py - - name: Set matrix for subsequent build id: set-matrix run: | From 97dc9593ef8895c89533ad5849bee3df0a670f6a Mon Sep 17 00:00:00 2001 From: Reinhold Gschweicher Date: Mon, 2 Jun 2025 15:39:27 +0200 Subject: [PATCH 08/10] CI: check_urls: own named step for install pycurl --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f0de1d1e5..81196fb20f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,9 +40,12 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} - - name: Check changed projects for working URLs + - name: Install dependencies run: | sudo apt-get install -yq python3-pycurl + + - name: Check changed projects for working URLs + run: | python maintenance/check-urls.py set_matrix: From fa619249d78ea239410a8468bc554c7548822310 Mon Sep 17 00:00:00 2001 From: Reinhold Gschweicher Date: Mon, 2 Jun 2025 15:41:49 +0200 Subject: [PATCH 09/10] Revert "[WIP] eigen: add typo in URL" This reverts commit e5ac171eb5b20208d27aeac6134e39ebe6c71df5. --- cmake/projects/Eigen/hunter.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/projects/Eigen/hunter.cmake b/cmake/projects/Eigen/hunter.cmake index 2a03474fae..b2dba28986 100644 --- a/cmake/projects/Eigen/hunter.cmake +++ b/cmake/projects/Eigen/hunter.cmake @@ -19,7 +19,7 @@ hunter_add_version( VERSION "3.4.0" URL - "https://gitlab.com/libeigens/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz" + "https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz" SHA1 d222db69a9e87d9006608e029d1039039f360b52 ) From 5291d732d033fd637704303839bf8b6c25c25df6 Mon Sep 17 00:00:00 2001 From: Reinhold Gschweicher Date: Mon, 2 Jun 2025 15:47:18 +0200 Subject: [PATCH 10/10] check_urls: always print success --- maintenance/check-urls.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/maintenance/check-urls.py b/maintenance/check-urls.py index c34991f8b3..af004d75e5 100755 --- a/maintenance/check-urls.py +++ b/maintenance/check-urls.py @@ -21,7 +21,7 @@ def print_failed_projects(): print("failed projects:") print(json.dumps(failed_projects, indent=2)) else: - print("all clear! no failed projects found") + print("all clear! No project with failing URL found") def signal_handler(signal, frame): @@ -125,13 +125,11 @@ def main(): if project not in failed_projects: failed_projects[project] = [] failed_projects[project].append(f"{status_code} {url}") - if len(failed_projects) > 0: - print_failed_projects() - if args.output: - with open(args.output, "w", encoding="utf-8") as file: - json.dump(failed_projects, file, indent=2) - return 1 - return 0 + print_failed_projects() + if len(failed_projects) > 0 and args.output: + with open(args.output, "w", encoding="utf-8") as file: + json.dump(failed_projects, file, indent=2) + return 0 if len(failed_projects) == 0 else 1 if __name__ == "__main__":