diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 643ec3657b..81196fb20f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,32 @@ on: jobs: + check_urls: + runs-on: ubuntu-latest + steps: + - name: Git checkout + uses: actions/checkout@v4.1.1 + + - name: Manual dispatch, get project name from input + if: github.event_name == 'workflow_dispatch' + run: | + echo '["cmake/projects/${{ github.event.inputs.project }}/hunter.cmake"]' > ${HOME}/files.json + + - name: Get changed files and save them to ${HOME}/files.json + if: github.event_name != 'workflow_dispatch' + id: files + uses: lots0logs/gh-action-get-changed-files@2.2.2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install dependencies + run: | + sudo apt-get install -yq python3-pycurl + + - name: Check changed projects for working URLs + run: | + python maintenance/check-urls.py + set_matrix: runs-on: ubuntu-latest outputs: diff --git a/maintenance/check-urls.py b/maintenance/check-urls.py old mode 100644 new mode 100755 index 1b201ee781..af004d75e5 --- a/maintenance/check-urls.py +++ b/maintenance/check-urls.py @@ -1,9 +1,37 @@ -from os.path import dirname, abspath, join +#!/usr/bin/env python3 +# # -*- coding: utf-8 -*- + +import argparse from glob import glob +import json +import pathlib +import os +from os.path import dirname, abspath, join import re +import signal import sys import pycurl +# failed projects, global to print on early abort with Ctrl+C +failed_projects: dict[str, list[str]] = dict() + + +def print_failed_projects(): + if failed_projects: + print("failed projects:") + print(json.dumps(failed_projects, indent=2)) + else: + print("all clear! No project with failing URL found") + + +def signal_handler(signal, frame): + # force exit as pycurl.error after KeyboardInterrupt is caught + # and then the next url is checked + print("You pressed Ctrl+C!") + print_failed_projects() + sys.exit(1) + + def getResponseStausCode(url): try: c = pycurl.Curl() @@ -18,36 +46,91 @@ def getResponseStausCode(url): except pycurl.error: return 999 -hunterDir = dirname(dirname(abspath(__file__))) -projectsDir = join(hunterDir, 'cmake', 'projects') -project = '' -if len(sys.argv) > 1: - project = sys.argv[1] +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "projects", + help="project names to process, used for local debugging", + nargs="*", + ) + parser.add_argument( + "-o", + "--output", + help="specify file to write failed projects and URLs to, default write to stdout", + type=str, + default="", + ) + args = parser.parse_args() -projectsFiles = join(projectsDir, project, '**', '*.cmake') + repo_root = pathlib.Path(__file__).parent.parent + projects_dir = repo_root / "cmake" / "projects" -checkedFile = join(hunterDir, 'maintenance', 'checked.txt') -try: - checkedStream = open(checkedFile, "r+") - checked = checkedStream.readlines() -except FileNotFoundError: - checkedStream = open(checkedFile, "w") - checked = [] + projects = set() + if args.projects: + if "all" in args.projects: + print("project 'all' specified, checking all projects") + project_hunter_files = projects_dir / "*" / "hunter.cmake" + for hunter_file in glob(project_hunter_files.as_posix(), recursive=False): + project = pathlib.Path(hunter_file).parent.name + projects.add(project) + else: + for project in args.projects: + if (projects_dir / project).is_dir(): + projects.add(project) + else: + raise RuntimeError( + f"provided project doesn't exist: {project}: expected dir: {projects_dir / project}" + ) + else: + try: + with open(os.environ.get("HOME") + "/files.json") as json_files: + files = json.load(json_files) + except IOError: + raise RuntimeError("Can't read changed files from files.json") -projects = dict() + p = re.compile("cmake/projects/([^/]+)") + for file in files: + if p.match(file): + project = p.match(file).group(1) + if (projects_dir / project).is_dir(): + projects.add(project) -for projectFile in glob(projectsFiles, recursive=True): - with open(projectFile, "r") as file: - content = file.read() + # override signal handler to make it possible to hard exit at Ctrl+C + # and print a status if we've found a failing URL or not yet + signal.signal(signal.SIGINT, signal_handler) - entries = re.findall(r'hunter_add_version\s*\(\s*PACKAGE_NAME\s+"*(.*?)"*\s+VERSION\s+"*(.*?)"*\s+URL\s+"*(.*?)"*\s+SHA1\s+"*(.*?)"*\s+.*?\)', content, re.MULTILINE | re.DOTALL) - if len(entries): + for project in sorted(projects): + print() + print(f"checking project: {project}") + hunter_file = projects_dir / project / "hunter.cmake" + if not hunter_file.is_file(): + raise RuntimeError(f"hunter.cmake file not found: {hunter_file}") + with open(hunter_file, "r", encoding="utf-8") as file: + content = file.read() + + entries = re.findall( + r'hunter_add_version\s*\(\s*PACKAGE_NAME\s+"*(.*?)"*\s+VERSION\s+"*(.*?)"*\s+URL\s+"*(.*?)"*\s+SHA1\s+"*(.*?)"*\s+.*?\)', + content, + re.MULTILINE | re.DOTALL, + ) + if len(entries) == 0: + raise RuntimeError( + f"no URLs found for project '{project}' in file: {hunter_file}" + ) for name, version, url, _ in entries: - if not any(url == x.rstrip('\n') for x in checked): - statusCode = getResponseStausCode(url) - print(str(statusCode) + ' ' + url) - if statusCode > 200: - checkedStream.write(str(statusCode) + ' ' + url + '\n') - -checkedStream.close() + status_code = getResponseStausCode(url) + print(f"{status_code} {url}") + if status_code > 200: + if project not in failed_projects: + failed_projects[project] = [] + failed_projects[project].append(f"{status_code} {url}") + print_failed_projects() + if len(failed_projects) > 0 and args.output: + with open(args.output, "w", encoding="utf-8") as file: + json.dump(failed_projects, file, indent=2) + return 0 if len(failed_projects) == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main())