From 109ea2a70a7299bd9fe6bb60ed673cb3f2acdef0 Mon Sep 17 00:00:00 2001 From: Sangjoon Bob Lee Date: Tue, 24 Sep 2024 13:37:08 -0400 Subject: [PATCH 1/5] Add flake8, gitignore, precommit config, isort --- .flake8 | 11 +++++ .gitignore | 99 +++++++++++++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 46 +++++++++++++++++++ isort.cfg | 4 ++ 4 files changed, 160 insertions(+) create mode 100644 .flake8 create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 isort.cfg diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..076b992 --- /dev/null +++ b/.flake8 @@ -0,0 +1,11 @@ +[flake8] +exclude = + .git, + __pycache__, + build, + dist, + doc/source/conf.py +max-line-length = 115 +# Ignore some style 'errors' produced while formatting by 'black' +# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#labels-why-pycodestyle-warnings +extend-ignore = E203 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c73d66a --- /dev/null +++ b/.gitignore @@ -0,0 +1,99 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +venv/ +*.egg-info/ +.installed.cfg +*.egg +bin/ +temp/ +tags/ +errors.err + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt +MANIFEST + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Django stuff: +*.log + +# Sphinx documentation +docs/build/ +docs/source/generated/ + +# pytest +.pytest_cache/ + +# PyBuilder +target/ + +# Editor files +# mac +.DS_Store +*~ + +# vim +*.swp +*.swo + +# pycharm +.idea/ + +# VSCode +.vscode/ + +# Ipython Notebook +.ipynb_checkpoints + +# version information +setup.cfg +/src/{{ cookiecutter.github_org }}/*/version.cfg + +# Rever +rever/ \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..217902f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,46 @@ +default_language_version: + python: python3 +ci: + autofix_commit_msg: | + [pre-commit.ci] auto fixes from pre-commit hooks + autofix_prs: true + autoupdate_branch: 'pre-commit-autoupdate' + autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' + autoupdate_schedule: monthly + skip: [no-commit-to-branch] + submodules: false +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-case-conflict + - id: check-merge-conflict + - id: check-toml + - id: check-added-large-files + - repo: https://github.com/psf/black + rev: 24.4.2 + hooks: + - id: black + - repo: https://github.com/pycqa/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + args: ["--profile", "black"] + - repo: https://github.com/kynan/nbstripout + rev: 0.7.1 + hooks: + - id: nbstripout + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: no-commit-to-branch + name: Prevent Commit to Main Branch + args: ["--branch", "main"] + stages: [pre-commit] \ No newline at end of file diff --git a/isort.cfg b/isort.cfg new file mode 100644 index 0000000..96f6cda --- /dev/null +++ b/isort.cfg @@ -0,0 +1,4 @@ +[settings] +line_length = 115 +multi_line_output = 3 +include_trailing_comma = True \ No newline at end of file From d60ab8aaa2c8b4035a070b47c40a5318bc8785e8 Mon Sep 17 00:00:00 2001 From: Sangjoon Bob Lee Date: Tue, 24 Sep 2024 13:53:46 -0400 Subject: [PATCH 2/5] Fix flake8 problem --- .flake8 | 2 +- .github/workflows/check-news.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.flake8 b/.flake8 index 076b992..2d2cb16 100644 --- a/.flake8 +++ b/.flake8 @@ -8,4 +8,4 @@ exclude = max-line-length = 115 # Ignore some style 'errors' produced while formatting by 'black' # https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#labels-why-pycodestyle-warnings -extend-ignore = E203 \ No newline at end of file +extend-ignore = E203 diff --git a/.github/workflows/check-news.py b/.github/workflows/check-news.py index f62fc82..1ee06f5 100644 --- a/.github/workflows/check-news.py +++ b/.github/workflows/check-news.py @@ -17,7 +17,9 @@ def get_added_files(pr: PullRequest.PullRequest): def check_news_file(pr): - return any(map(lambda file_name: fnmatch(file_name, "news/*.rst"), get_added_files(pr))) + return any( + map(lambda file_name: fnmatch(file_name, "news/*.rst"), get_added_files(pr)) + ) def get_pr_number(): @@ -29,7 +31,9 @@ def get_pr_number(): def get_old_comment(pr: PullRequest.PullRequest): for comment in pr.get_issue_comments(): - if ("github-actions" in comment.user.login) and ("No news item is found" in comment.body): + if ("github-actions" in comment.user.login) and ( + "No news item is found" in comment.body + ): return comment @@ -56,7 +60,7 @@ def main(): **Warning!** No news item is found for this PR. If this is a user-facing change/feature/fix, please add a news item by copying the format from `news/TEMPLATE.rst`. """ - ) + ) assert False From 3921dac5e4585248d1cd188cb74e8abfcfc16486 Mon Sep 17 00:00:00 2001 From: Sangjoon Bob Lee Date: Tue, 24 Sep 2024 13:54:11 -0400 Subject: [PATCH 3/5] Add gitignore, precomit, isort --- .gitignore | 2 +- .pre-commit-config.yaml | 2 +- isort.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index c73d66a..fed8d34 100644 --- a/.gitignore +++ b/.gitignore @@ -96,4 +96,4 @@ setup.cfg /src/{{ cookiecutter.github_org }}/*/version.cfg # Rever -rever/ \ No newline at end of file +rever/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 217902f..3070e19 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,4 +43,4 @@ repos: - id: no-commit-to-branch name: Prevent Commit to Main Branch args: ["--branch", "main"] - stages: [pre-commit] \ No newline at end of file + stages: [pre-commit] diff --git a/isort.cfg b/isort.cfg index 96f6cda..e0926f4 100644 --- a/isort.cfg +++ b/isort.cfg @@ -1,4 +1,4 @@ [settings] line_length = 115 multi_line_output = 3 -include_trailing_comma = True \ No newline at end of file +include_trailing_comma = True From 7b6627b09a3b94b6a77d14327e3e961b4c062de7 Mon Sep 17 00:00:00 2001 From: Sangjoon Bob Lee Date: Tue, 24 Sep 2024 13:56:58 -0400 Subject: [PATCH 4/5] Isort cf release --- cf_release.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cf_release.py b/cf_release.py index c38b507..0484177 100644 --- a/cf_release.py +++ b/cf_release.py @@ -1,9 +1,10 @@ -import sys -import requests import subprocess -from os.path import join, exists, dirname +import sys +from os.path import dirname, exists, join + import click -from click import prompt, confirm, Choice +import requests +from click import confirm, prompt """ This script streamlines the process of updating Python package versions and From 424758c8ab3f5367d5eedfdf66d87c7738098fc6 Mon Sep 17 00:00:00 2001 From: Sangjoon Bob Lee Date: Tue, 24 Sep 2024 13:57:29 -0400 Subject: [PATCH 5/5] Apply black, isort, fix line lenght --- .pre-commit-config.yaml | 1 + auto_api.py | 42 +++--- basic_release.sh | 2 +- release.py | 279 ++++++++++++++++++++++------------------ snake_nest.py | 131 +++++++++++-------- snake_nest.txt | 2 +- update_workflow.py | 21 +-- 7 files changed, 272 insertions(+), 206 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3070e19..f9940da 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,7 @@ repos: rev: v4.6.0 hooks: - id: check-yaml + exclude: '.github/' - id: end-of-file-fixer - id: trailing-whitespace - id: check-case-conflict diff --git a/auto_api.py b/auto_api.py index 1cd5f02..8bcabd6 100644 --- a/auto_api.py +++ b/auto_api.py @@ -1,10 +1,10 @@ #!/usr/bin/env python -from pathlib import Path import optparse -import re +import shlex import subprocess import sys +from pathlib import Path def call(cmd, cwd, capture_output=False): @@ -20,16 +20,16 @@ def __init__(self, prog_name, *args, **kwargs): super(Parser, self).__init__(*args, **kwargs) self.prog_name = prog_name - def detailed_error(self, msg): - self.exit(2, f"{prog_name}: error: {msg}\n") - - parser = Parser(prog_name=prog_name_short, - usage='\n'.join([ - "%prog ", - "Automatically populate the API directory for a package.", - "This only handles packages with single-depth submodules.", - ]), - epilog="Please report bugs on https://github.com/Billingegroup/release-scripts/issues." + parser = Parser( + prog_name=prog_name_short, + usage="\n".join( + [ + "%prog ", + "Automatically populate the API directory for a package.", + "This only handles packages with single-depth submodules.", + ] + ), + epilog="Please report bugs on https://github.com/Billingegroup/release-scripts/issues.", ) return parser @@ -50,17 +50,17 @@ def main(opts, pargs): # Populate API directory def gen_package_files(package_dir, package_name): - """ Generate package files. - + """Generate package files. + Parameters ---------- - + package_dir: Path The package directory (e.g. /src/diffpy/pdfmorph). package_name: str The name of the package (e.g. diffpy.pdfmorph). """ - eq_spacing = "="*len(f"{package_name} package") + eq_spacing = "=" * len(f"{package_name} package") s = f""":tocdepth: -1 {package_name} package @@ -96,7 +96,11 @@ def gen_package_files(package_dir, package_name): sm_names = [] skip_files = ["__init__", "version"] for child in package_dir.iterdir(): - if child.is_file() and child.suffix == ".py" and child.stem not in skip_files: + if ( + child.is_file() + and child.suffix == ".py" + and child.stem not in skip_files + ): sm_names.append(f"{package_name}.{child.stem}") if len(sm_names) > 0: s += """ @@ -104,7 +108,7 @@ def gen_package_files(package_dir, package_name): ---------- """ for sm_name in sm_names: - dsh_spacing = "^"*len(f"{sm_name} module") + dsh_spacing = "^" * len(f"{sm_name} module") s += f""" {sm_name} module {dsh_spacing} @@ -114,7 +118,7 @@ def gen_package_files(package_dir, package_name): :undoc-members: :show-inheritance: """ - + s += "\n" package_file = api_dir / f"{package_name}.rst" with open(package_file, "w") as pfile: diff --git a/basic_release.sh b/basic_release.sh index 1ba0a58..065bd85 100755 --- a/basic_release.sh +++ b/basic_release.sh @@ -48,7 +48,7 @@ mkdir "$tmp_release_dir" project_path="$(pwd)" project="${project_path##*/}" tgz_name="$project-$version.tar.gz" -tar --exclude="./$tmp_release_dir" -zcf "./$tmp_release_dir/$tgz_name" . +tar --exclude="./$tmp_release_dir" -zcf "./$tmp_release_dir/$tgz_name" . # GitHub Release git tag $version $(git rev-parse HEAD) diff --git a/release.py b/release.py index af56e64..d36fa01 100755 --- a/release.py +++ b/release.py @@ -1,193 +1,194 @@ #!/usr/bin/env python +import optparse import re +import shlex import subprocess - -import optparse import sys -import shlex +import warnings +from hashlib import sha256 from pathlib import Path -import warnings import requests -from hashlib import sha256 gh_release_notes = None + def call(cmd, cwd, capture_output=False): cmd_list = shlex.split(cmd) return subprocess.run(cmd_list, cwd=cwd, capture_output=capture_output, text=True) + def create_option_parser(): prog_name_short = Path(sys.argv[0]).name # Program name - + class Parser(optparse.OptionParser): def __init__(self, prog_name, *args, **kwargs): super(Parser, self).__init__(*args, **kwargs) self.prog_name = prog_name - - def detailed_error(self, msg): - self.exit(2, f"{prog_name}: error: {msg}\n") - - parser = Parser(prog_name=prog_name_short, - usage='\n'.join([ - "%prog OPTIONS", - "Release a particular version of a directory.", - ]), - epilog="Please report bugs on https://github.com/Billingegroup/release-scripts/issues." + + parser = Parser( + prog_name=prog_name_short, + usage="\n".join( + [ + "%prog OPTIONS", + "Release a particular version of a directory.", + ] + ), + epilog="Please report bugs on https://github.com/Billingegroup/release-scripts/issues.", ) - + vsn_group = optparse.OptionGroup( parser, "Version Control", - "Update local version numbers, changelog information, and remote tag versions." + "Update local version numbers, changelog information, and remote tag versions.", ) parser.add_option_group(vsn_group) - - vsn_group.add_option( - "--version-bump", - metavar="VBREGEX", - dest="vb_regex" - ) - + + vsn_group.add_option("--version-bump", metavar="VBREGEX", dest="vb_regex") + vsn_group.add_option( "--changelog", action="store_true", - help="Combine all update files in the news directory into a single changelog file." + help="Combine all update files in the news directory into a single changelog file.", ) - + vsn_group.add_option( "--cl-file", metavar="FILEPATH", - help="Name (and path if not in root) of changelog file. Default \'CHANGELOG.rst\'" + help="Name (and path if not in root) of changelog file. Default 'CHANGELOG.rst'", ) - + vsn_group.add_option( "--cl-news", metavar="NEWSDIR", - help="Location of news directory. Default is \'news\' in the root directory." + help="Location of news directory. Default is 'news' in the root directory.", ) - + vsn_group.add_option( "--cl-template", metavar="TEMPLATE", - help="Name of template file. One will be auto-generated if one does not exist. Default \'TEMPLATE.rst\'." + help="Name of template file. One will be auto-generated if one does not exist. Default 'TEMPLATE.rst'.", ) - + vsn_group.add_option( "--cl-categories", metavar="CATLIST", - help="List of categories to include in the changelog. Default \'Added, Changed, Deprecated, Removed, Fixed, Security\'." + help=( + "List of categories to include in the changelog. " + "Default 'Added, Changed, Deprecated, Removed, Fixed, Security'." + ), ) - + vsn_group.add_option( "--cl-ignore", metavar="FILELIST", - help="List of files in news to ignore. The template file is always ignored." + help="List of files in news to ignore. The template file is always ignored.", ) - + vsn_group.add_option( "--cl-access-point", metavar="APSTRING", - help="All changes will be put in the change log after the access point. Default \'.. current developments\'." + help=( + "All changes will be put in the change log after the access point. " + "Default '.. current developments'.", + ), ) - + vsn_group.add_option( - "--tag", - action="store_true", - help="Create and push a version tag to GitHub." + "--tag", action="store_true", help="Create and push a version tag to GitHub." ) - + vsn_group.add_option( - "--upstream", - action="store_true", - help="Push to upstream rather than origin." + "--upstream", action="store_true", help="Push to upstream rather than origin." ) - + rel_group = optparse.OptionGroup( parser, "Release Targets", - "Make sure you have bumped the version, updated the changelog, and pushed the tag." + "Make sure you have bumped the version, updated the changelog, and pushed the tag.", ) parser.add_option_group(rel_group) - + rel_group.add_option( "--release", action="store_true", - help="Update the changelog, push the tag, upload to Github, and upload to PyPi." + help="Update the changelog, push the tag, upload to Github, and upload to PyPi.", ) - + rel_group.add_option( "--pre-release", action="store_true", - help="Push the tag, upload to Github, and upload to PyPi." + help="Push the tag, upload to Github, and upload to PyPi.", ) - - rel_group.add_option( - "--github", - action="store_true", - help="Initiate a release on GitHub." - ) - + rel_group.add_option( - "--gh-title", - metavar="TITLE", - help="Title of GitHub release." + "--github", action="store_true", help="Initiate a release on GitHub." ) - + + rel_group.add_option("--gh-title", metavar="TITLE", help="Title of GitHub release.") + rel_group.add_option( "--gh-notes", metavar="NOTES", - help="Release notes to be posted for GitHub release." + help="Release notes to be posted for GitHub release.", ) - + rel_group.add_option( "--pypi", action="store_true", - help="Initiate a release on PyPi. Default is to upload the source distribution. Use the --wheel option to also build a wheel using python-build." + help=( + "Initiate a release on PyPi. Default is to upload the source distribution. " + "Use the --wheel option to also build a wheel using python-build.", + ), ) rel_group.add_option( - "-c", "--c", + "-c", + "--c", action="store_true", - help="Configure to release a project with C/C++ extension" + help="Configure to release a project with C/C++ extension", ) rel_group.add_option( "--no-wheel", action="store_true", - help="Provide source distribution only for the PyPi release." + help="Provide source distribution only for the PyPi release.", ) rel_group.add_option( "--cf-hash", metavar="HASHFILE", dest="forge", - help="Generate the SHA256 Hash for a conda-forge release in HASHFILE." + help="Generate the SHA256 Hash for a conda-forge release in HASHFILE.", ) - + return parser + # TODO: Implement automatic version bumping def version_bump(opts, pargs): pass + def update_changelog(opts, pargs): release_dir = pargs[0] version = pargs[1] - + # Get name of default branch remote = "origin" if opts.upstream: remote = "upstream" - remote_listing = call(f"git remote show {remote}", release_dir, capture_output=True).stdout + remote_listing = call( + f"git remote show {remote}", release_dir, capture_output=True + ).stdout head_name = re.search("HEAD branch.*: (.+)", remote_listing).group(1) call(f"git checkout {head_name}", release_dir) - + # Categories of changes - categories = ['Added', 'Changed', 'Deprecated', 'Removed', 'Fixed', 'Security'] + categories = ["Added", "Changed", "Deprecated", "Removed", "Fixed", "Security"] if opts.cl_categories is not None: - categories = list(map(str.strip, opts.cl_categories.split(','))) + categories = list(map(str.strip, opts.cl_categories.split(","))) # Find news directory news = "news" @@ -196,26 +197,26 @@ def update_changelog(opts, pargs): news = (release_dir / news).resolve() if not news.exists(): call(f"mkdir {news.name}", release_dir) - + # Files to ignore ignore = [] if opts.cl_ignore is not None: - ignore = list(map(str.strip, opts.cl_ignore.split(','))) - + ignore = list(map(str.strip, opts.cl_ignore.split(","))) + # Add template to ignore list template = "TEMPLATE.rst" if opts.cl_template is not None: template = opts.cl_template ignore.append(template) template = (news / template).resolve() - + # Generate template if one does not exist if not template.exists(): - with open(template, 'w') as tf: + with open(template, "w") as tf: entries = [f"**{cat}:**\n\n* \n" for cat in categories] generated_template = "\n".join(entries) tf.write(f"{generated_template}\n") - + # Create a changelog if one does not exist changelog = "CHANGELOG.rst" if opts.cl_file is not None: @@ -223,33 +224,33 @@ def update_changelog(opts, pargs): ignore.append(changelog) changelog = (release_dir / changelog).resolve() if not changelog.exists(): - with open(changelog, 'w') as cf: + with open(changelog, "w") as cf: generated_changelog = "=============\nRelease Notes\n=============\n\n.. current developments\n" cf.write(f"{generated_changelog}\n") - + # Compile all changes changes = {cat: [] for cat in categories} for change_file in news.iterdir(): if change_file.name in ignore: continue key = None - with open(change_file, 'r') as cf: + with open(change_file, "r") as cf: for row in cf: # New key rx = re.search(r"\*\*(.+):\*\*", row) if rx: key = rx.group(1) continue - + # New entry if key is not None and "" not in row and row.strip() != "": - changes[key].append(row) - + changes[key].append(row) + # Write to changelog access_point = ".. current developments" if opts.cl_access_point is not None: access_point = opts.cl_access_point - with open(changelog, 'r+') as cf: + with open(changelog, "r+") as cf: # Generate text based on changes sep = "" for i in range(len(str(version))): @@ -260,7 +261,7 @@ def update_changelog(opts, pargs): generated_update += f"\n**{key}:**\n" key_updates = "".join(changes[key]) generated_update += f"\n{key_updates}" - + # Write update after access point line = cf.readline() ptr = cf.tell() @@ -280,26 +281,27 @@ def update_changelog(opts, pargs): cf.seek(ptr) cf.write(f"\n{generated_update}{spc}") cf.write(f"{prev_updates}") - + # Set Github release notes global gh_release_notes - gh_release_notes = generated_update[len(f"{version}\n{sep}\n"):] - + gh_release_notes = generated_update[len(f"{version}\n{sep}\n") :] + # Remove used files for change_file in news.iterdir(): if change_file.name in ignore: continue change_file.unlink() - + # Push changes to GitHub call(f"git add {news} {changelog}", release_dir) - call(f"git commit -m \"Update {changelog.name}\" --no-verify", release_dir) + call(f'git commit -m "Update {changelog.name}" --no-verify', release_dir) call(f"git push {remote} {head_name}", release_dir) + def push_tag(opts, pargs): release_dir = pargs[0] version = pargs[1] - + # Create and push tag to origin (must have origin release access) call(f"git tag {version}", release_dir) if opts.upstream is not None: @@ -307,10 +309,12 @@ def push_tag(opts, pargs): else: call(f"git push origin {version}", release_dir) + # TODO: Implement environment and permissions checks def check(): pass + def github_release(opts, pargs): release_dir = pargs[0] version = pargs[1] @@ -320,12 +324,12 @@ def github_release(opts, pargs): while (Path(release_dir) / tmp_dir).exists(): tmp_dir += "_prime" call(f"mkdir {tmp_dir}", release_dir) - + # Build tar project = Path(release_dir).name tgz_name = f"{project}-{version}.tar.gz" - call(f"tar --exclude=\"./{tmp_dir}\" -zcf \"./{tmp_dir}/{tgz_name}\" . ", release_dir) - + call(f'tar --exclude="./{tmp_dir}" -zcf "./{tmp_dir}/{tgz_name}" . ', release_dir) + # Set notes and title if user has not provided any gh_notes = "--generate-notes" if gh_release_notes is not None and gh_release_notes.strip() != "": @@ -335,24 +339,33 @@ def github_release(opts, pargs): gh_title = f"-t {version}" if opts.gh_title is not None: gh_title = f"-t {opts.gh_title}" - + # Release through gh - call(f"gh release create \"{version}\" \"./{tmp_dir}/{tgz_name}\" \"{gh_title}\" \"{gh_notes}\"", release_dir) - + call( + f'gh release create "{version}" "./{tmp_dir}/{tgz_name}" "{gh_title}" "{gh_notes}"', + release_dir, + ) + # Cleanup call(f"rm -rf {tmp_dir}", release_dir) - + def pypi_release(opts, pargs): release_dir = pargs[0] version = pargs[1] - db_warning = "Warning: No new distribution build. This occurs when there are no new changes to the source code since the previous release. Please check for any untracked changes and update your package changelog/release-history to reflect the newest version." + db_warning = ( + "Warning: No new distribution build. This occurs when there are no new changes " + "to the source code since the previous release. Please check for any untracked " + "changes and update your package changelog/release-history to reflect the " + "newest version." + ) + build_wheel = not opts.no_wheel - + # Build wheel and source if build_wheel: - call("python -m build", release_dir) + call("python -m build", release_dir) # Upload using twine no_tar = True no_whl = True @@ -362,11 +375,14 @@ def pypi_release(opts, pargs): if re.search(f".*{version}.*.whl", file.name): no_whl = False if no_tar: - call(f"echo \"{db_warning}\"", release_dir) + call(f'echo "{db_warning}"', release_dir) elif no_whl: - call(f"echo \"Warning: No wheel found.\"", release_dir) + call('echo "Warning: No wheel found."', release_dir) else: - call(f"twine upload dist/*{version}*.tar.gz dist/*{version}*.whl", release_dir) + call( + f"twine upload dist/*{version}*.tar.gz dist/*{version}*.whl", + release_dir, + ) # Only upload source elif not build_wheel: @@ -377,10 +393,11 @@ def pypi_release(opts, pargs): if re.search(f".*{version}.*.tar.gz", file.name): no_tar = False if no_tar: - call(f"echo \"{db_warning}\"", release_dir) + call(f'echo "{db_warning}"', release_dir) else: call(f"twine upload dist/*{version}*.tar.gz", release_dir) + # Generate SHA256 Hash for a Conda-Forge Release def cf_hash(opts, pargs): version = pargs[1] @@ -396,7 +413,9 @@ def cf_hash(opts, pargs): source_message += f"[{i+1}] {source}\n" source_message += "Choose a distribution source (name or number): " dist_source = "" - while dist_source.lower() not in sources and dist_source not in [str(i) for i in range(1, len(sources)+1)]: + while dist_source.lower() not in sources and dist_source not in [ + str(i) for i in range(1, len(sources) + 1) + ]: dist_source = input(source_message) # Get the download link for the .tar.gz @@ -405,24 +424,35 @@ def cf_hash(opts, pargs): source_url = f"https://pypi.io/packages/source/{pep_name[0]}/{pep_name}/{pep_name}-{version}.tar.gz" dist_source = "PyPi" if dist_source == "2" or dist_source == "github": - github_org = input(f"What organization/user's version of {module_name} are you looking for: ") - source_url = f"https://github.com/{github_org}/{module_name}/archive/{version}.tar.gz" + github_org = input( + f"What organization/user's version of {module_name} are you looking for: " + ) + source_url = ( + f"https://github.com/{github_org}/{module_name}/archive/{version}.tar.gz" + ) dist_source = "GitHub" tar_gz_dist = requests.get(source_url) # Hash and ensure hash is not of an empty file sha256_hash = sha256(tar_gz_dist.content).hexdigest().strip() if ( - sha256_hash == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - or sha256_hash == "0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5" - or sha256_hash == "d5558cd419c8d46bdc958064cb97f963d1ea793866414c025906ec15033512ed" - ): - warnings.warn("SHA256 Hash Returned Emtpy File Hash. " - f"Make sure your .tar.gz is uploaded to {dist_source}!") + sha256_hash + == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + or sha256_hash + == "0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5" + or sha256_hash + == "d5558cd419c8d46bdc958064cb97f963d1ea793866414c025906ec15033512ed" + ): + warnings.warn( + "SHA256 Hash Returned Emtpy File Hash. " + f"Make sure your .tar.gz is uploaded to {dist_source}!" + ) # Print hash and source to a file - with open(opts.forge, 'w') as hash_dump: - hash_dump.write(f"Distribution downloaded from: {source_url}\nSHA256: {sha256_hash}\n") + with open(opts.forge, "w") as hash_dump: + hash_dump.write( + f"Distribution downloaded from: {source_url}\nSHA256: {sha256_hash}\n" + ) print(f"SHA256 Hash written to file: {opts.forge}") @@ -436,8 +466,11 @@ def cf_hash(opts, pargs): parser.error("Improper usage. Too many arguments!") if opts.release and opts.pre_release: - parser.error("Both release and pre-release specified. Please re-run the command specifying either release or pre_release.") - + parser.error( + "Both release and pre-release specified. " + "Please re-run the command specifying either release or pre_release." + ) + # Linking --c to set --no-wheel for project with C/C++ extension if opts.c: opts.no_wheel = True diff --git a/snake_nest.py b/snake_nest.py index dd839af..c5307d0 100755 --- a/snake_nest.py +++ b/snake_nest.py @@ -1,76 +1,73 @@ #!/usr/bin/env python -import re -import subprocess - import optparse -import sys import os - +import subprocess +import sys import warnings def create_option_parser(): prog_name_short = os.path.basename(sys.argv[0]) # Program name - + class Parser(optparse.OptionParser): def __init__(self, prog_name, *args, **kwargs): super(Parser, self).__init__(*args, **kwargs) self.prog_name = prog_name - - def detailed_error(self, msg): - self.exit(2, f"{prog_name}: error: {msg}\n") - - parser = Parser(prog_name=prog_name_short, - usage='\n'.join([ - "python %prog ", - "For example: python %prog \"3.10, 3.11, 3.12\"" - ]), - epilog="Please report bugs on https://github.com/Billingegroup/release-scripts/issues." + + parser = Parser( + prog_name=prog_name_short, + usage="\n".join( + [ + "python %prog ", + 'For example: python %prog "3.10, 3.11, 3.12"', + ] + ), + epilog="Please report bugs on https://github.com/Billingegroup/release-scripts/issues.", ) parser.add_option( - '-m', - '--mamba', + "-m", + "--mamba", action="store_true", dest="mamba", - help="""Use mamba instead of conda. Without this option enabled, conda will be used.""" + help="""Use mamba instead of conda. Without this option enabled, conda will be used.""", ) parser.add_option( - '-p', - '--prefix', + "-p", + "--prefix", metavar="PREFIX", dest="prefix", help="""Set the environment name prefix. Environments will be named . The default prefix is \"py-\".""", ) parser.add_option( - '-s', - '--suffix', + "-s", + "--suffix", metavar="SUFFIX", dest="suffix", help="""Set the environment name suffix. Environments will be named . The default suffix is \"-env\".""", ) parser.add_option( - '--vreqs', + "--vreqs", action="store_true", dest="vreqs", help="""Indicate that the requirements to be installed are version-specific. This changes the behavior of the inputs for --requirements and --pip-requirements. Each now takes in an expression of the form \"[vsn]\". The program will replace \"[vsn]\" with a version number (e.g. 3.10). -For example, the input \"reqs/py-[vsn]-reqs.txt\" corresponds to files like \"reqs/py-3.10-reqs.txt\".""" +For example, the input \"reqs/py-[vsn]-reqs.txt\" corresponds to files like \"reqs/py-3.10-reqs.txt\".""", ) parser.add_option( - '-r', - '--requirements', + "-r", + "--requirements", metavar="REQFILE", dest="reqs", help="""A file containing requirements to install in each environment. These requirements must be installable through conda/mamba.""", ) parser.add_option( - '--pip-requirements', + "--pip-requirements", metavar="PREQFILE", dest="pip_reqs", help="""A file containing requirements to install in each environment. @@ -78,45 +75,48 @@ def detailed_error(self, msg): It is recommended to install through conda/mamba unless the packages are available only on PyPi.""", ) parser.add_option( - '-d', - '--dev-mode', + "-d", + "--dev-mode", metavar="DEVDIR", dest="dev_dir", help="""Install the specified directory in developer mode (pip install -e DEVDIR). This command will be run in each environment created.""", ) parser.add_option( - '--nest', - '--make-snake-nest', + "--nest", + "--make-snake-nest", metavar="NESTDIR", dest="sn_dir", - help="""Create a snake nest (a directory containing symbolic links to each version-specific Python executable.) -The user should input the path to the desired directory and the program will create that directory if it does not exist. -A warning is thrown if there are existing files in the given directory and no snake nest will be generated.""", + help=( + "Create a snake nest (a dir containing symbolic links to each version-specific Python executable). " + "The user should input the path to the desired dir and the program will create that directory if it " + "does not exist. A warning is thrown if there are existing files in the given dir, and no snake nest " + "will be generated." + ), ) parser.add_option( - '--clean', + "--clean", action="store_true", dest="clean", help="""Remove version environments. -Environments removed are specified the same way as creating environments.""" +Environments removed are specified the same way as creating environments.""", ) parser.add_option( - '--run-script', + "--run-script", metavar="SCRIPTFILE", dest="script", help="""Run a script in each environment. The script file is user-specified. -You can include [vsn] in the script and it will be replaced with the proper version number.""" +You can include [vsn] in the script and it will be replaced with the proper version number.""", ) return parser - + def create_snake_nest(opts, pargs): if len(pargs) != 1: parser.error("Improper usage.") versions = list(map(str.strip, pargs[0].split(","))) - + # Switch environment manager to mamba if chosen env_manager = "conda" if opts.mamba: @@ -129,8 +129,10 @@ def create_snake_nest(opts, pargs): prefix = opts.prefix if opts.suffix is not None: suffix = opts.suffix + def get_env_name(prefix, suffix, version): return prefix + version + suffix + env_names = list(map(lambda v: get_env_name(prefix, suffix, v), versions)) # Create environments @@ -139,9 +141,15 @@ def get_env_name(prefix, suffix, version): if opts.vreqs and vers_spec_req is not None: vers_spec_req = opts.reqs.replace("[vsn]", versions[i]) if vers_spec_req is not None: - subprocess.run(f"{env_manager} create -n {env_name} python={versions[i]} --file={vers_spec_req} --yes", shell=True) + subprocess.run( + f"{env_manager} create -n {env_name} python={versions[i]} --file={vers_spec_req} --yes", + shell=True, + ) else: - subprocess.run(f"{env_manager} create -n {env_name} python={versions[i]} --yes", shell=True) + subprocess.run( + f"{env_manager} create -n {env_name} python={versions[i]} --yes", + shell=True, + ) # Create snake-nest directory if it does not exist sn_dir = opts.sn_dir @@ -173,17 +181,26 @@ def get_env_name(prefix, suffix, version): if opts.vreqs and vers_spec_preq is not None: vers_spec_preq = vers_spec_preq.replace("[vsn]", versions[i]) if vers_spec_preq is not None: - subprocess.run(f"{env_manager} run -n {env_names[i]} pip install -r {vers_spec_preq}", shell=True) + subprocess.run( + f"{env_manager} run -n {env_names[i]} pip install -r {vers_spec_preq}", + shell=True, + ) if opts.dev_dir is not None: - subprocess.run(f"{env_manager} run -n {env_names[i]} pip install -e {opts.dev_dir}", shell=True) - + subprocess.run( + f"{env_manager} run -n {env_names[i]} pip install -e {opts.dev_dir}", + shell=True, + ) + # Setup the snake nest if sn_dir is not None: - subprocess.run(f"{env_manager} run -n {env_names[i]} which python >> {sn_file}", shell=True) - + subprocess.run( + f"{env_manager} run -n {env_names[i]} which python >> {sn_file}", + shell=True, + ) + # Setup symbolic links in snake nest if sn_dir is not None: - with open(sn_file, 'r') as snf: + with open(sn_file, "r") as snf: i = 0 for sym_link in snf: if "python" not in sym_link: @@ -199,7 +216,7 @@ def cleanup(opts, pargs): if len(pargs) != 1: parser.error("Improper usage.") versions = list(map(str.strip, pargs[0].split(","))) - + # Switch environment manager to mamba if chosen env_manager = "conda" if opts.mamba: @@ -212,8 +229,10 @@ def cleanup(opts, pargs): prefix = opts.prefix if opts.suffix is not None: suffix = opts.suffix + def get_env_name(prefix, suffix, version): return prefix + version + suffix + env_names = list(map(lambda v: get_env_name(prefix, suffix, v), versions)) # Delete environments @@ -225,7 +244,7 @@ def run_script(opts, pargs): if len(pargs) != 1: parser.error("Improper usage.") versions = list(map(str.strip, pargs[0].split(","))) - + # Switch environment manager to mamba if chosen env_manager = "conda" if opts.mamba: @@ -238,17 +257,21 @@ def run_script(opts, pargs): prefix = opts.prefix if opts.suffix is not None: suffix = opts.suffix + def get_env_name(prefix, suffix, version): return prefix + version + suffix + env_names = list(map(lambda v: get_env_name(prefix, suffix, v), versions)) # Perform operations specified by a file for i, env_name in enumerate(env_names): - with open(opts.script, 'r') as rf: + with open(opts.script, "r") as rf: for command in rf: command = command.replace("[vsn]", versions[i]) - - subprocess.run(f"{env_manager} run -n {env_names[i]} {command}", shell=True) + + subprocess.run( + f"{env_manager} run -n {env_names[i]} {command}", shell=True + ) if __name__ == "__main__": diff --git a/snake_nest.txt b/snake_nest.txt index e0a11db..b41d617 100644 --- a/snake_nest.txt +++ b/snake_nest.txt @@ -25,7 +25,7 @@ Options (taken from --help): -s SUFFIX, --suffix=SUFFIX Set the environment name suffix. Environments will be named . The default suffix is "-env". - --vreqs Indicate that the requirements to be installed are version-specific. This changes the behavior of the inputs for + --vreqs Indicate that the requirements to be installed are version-specific. This changes the behavior of the inputs for --requirements and --pip-requirements. Each now takes in an expression of the form "[vsn]". The program will replace "[vsn]" with a version number (e.g. 3.10). For example, the input "reqs/py-[vsn]-reqs.txt" corresponds to files like "reqs/py-3.10-reqs.txt". diff --git a/update_workflow.py b/update_workflow.py index 1f1d547..4aa926f 100644 --- a/update_workflow.py +++ b/update_workflow.py @@ -1,22 +1,23 @@ #!/usr/bin/env python """ -This script fetches the latest workflows from the central repository 'release-scripts' -and updates the local dummy workflows. Before running the script, install the required +This script fetches the latest workflows from the central repository 'release-scripts' +and updates the local dummy workflows. Before running the script, install the required packages using the following command: conda install requests -This script assumes the package repository has the same parent directory as 'release-scripts'. +This script assumes the package repository has the same parent directory as 'release-scripts'. You can change this by modifying the 'LOCAL_WORKFLOW_DIR' variable. -Sometimes there would be timeout errors while fetching the workflows from the central repository. +Sometimes there would be timeout errors while fetching the workflows from the central repository. In such cases, you can try running the script again. """ import os import re from pathlib import Path + import requests proj = ( @@ -33,18 +34,22 @@ user_input_cache = {"PROJECT": proj} + def get_central_workflows(): - base_url = f"https://api.github.com/repos/{CENTRAL_REPO_ORG}/{CENTRAL_REPO_NAME}/contents/{CENTRAL_WORKFLOW_DIR}" + base_url = ( + f"https://api.github.com/repos/{CENTRAL_REPO_ORG}/" + f"{CENTRAL_REPO_NAME}/contents/{CENTRAL_WORKFLOW_DIR}" + ) response = requests.get(base_url, timeout=5) if response.status_code != 200: raise Exception(f"Failed to fetch central workflows: {response.status_code}") workflows = {} for file in response.json(): - if file['type'] == 'file' and file['name'].endswith('.yml'): - content_response = requests.get(file['download_url'], timeout=5) + if file["type"] == "file" and file["name"].endswith(".yml"): + content_response = requests.get(file["download_url"], timeout=5) if content_response.status_code == 200: - workflows[file['name']] = content_response.text + workflows[file["name"]] = content_response.text return workflows