From 8920b93fc4a1edb19c200b99c274668418646dd7 Mon Sep 17 00:00:00 2001 From: Amy Wooding Date: Wed, 25 Jan 2023 10:37:53 -0500 Subject: [PATCH 01/13] handle arbitraty conda channels --- {{ cookiecutter.repo_name }}/Makefile.envs | 25 +++--- .../scripts/split_pip.py | 86 ++++++++++++------- 2 files changed, 63 insertions(+), 48 deletions(-) diff --git a/{{ cookiecutter.repo_name }}/Makefile.envs b/{{ cookiecutter.repo_name }}/Makefile.envs index 4c65eb7..5723f76 100644 --- a/{{ cookiecutter.repo_name }}/Makefile.envs +++ b/{{ cookiecutter.repo_name }}/Makefile.envs @@ -4,28 +4,22 @@ include Makefile.include -$(LOCKFILE): check_installation .make.bootstrap .make.pip-requirements.txt .make.environment-default.yml .make.conda-forge-requirements.txt +$(LOCKFILE): check_installation .make.bootstrap split_environment_files ifeq (conda, $(VIRTUALENV)) - $(CONDA_EXE) env update -n $(PROJECT_NAME) -f .make.environment-default.yml --prune - $(CONDA_EXE) install -n $(PROJECT_NAME) --file .make.conda-forge-requirements.txt --channel defaults --channel conda-forge --strict-channel-priority --yes + for channel in $(shell $(CAT) .make.channel-order.include); do\ + $(ECHO) installing from .make.$$channel-environment.txt;\ + $(CONDA_EXE) install -n $(PROJECT_NAME) --file .make.$$channel-environment.txt --channel defaults --channel $$channel --strict-channel-priority --yes ;\ + done $(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture pip install -r .make.pip-requirements.txt $(CONDA_EXE) env export -n $(PROJECT_NAME) -f $(LOCKFILE) else $(error Unsupported Environment `$(VIRTUALENV)`. Use conda) endif -# extract multi-phase dependencies from environment.yml -.make.environment-pip.yml: environment.yml .make.bootstrap - $(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture $(PYTHON_INTERPRETER) scripts/split_pip.py pip-yaml $(PROJECT_DIR)environment.yml > $@ - -.make.pip-requirements.txt: environment.yml .make.bootstrap - $(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture $(PYTHON_INTERPRETER) scripts/split_pip.py pip $(PROJECT_DIR)environment.yml > $@ - -.make.conda-forge-requirements.txt: environment.yml .make.bootstrap - $(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture $(PYTHON_INTERPRETER) scripts/split_pip.py conda-forge $(PROJECT_DIR)environment.yml > $@ - -.make.environment-default.yml: environment.yml .make.bootstrap - $(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture $(PYTHON_INTERPRETER) scripts/split_pip.py default $(PROJECT_DIR)environment.yml > $@ +.PHONY: split_environment_files +## extract multi-phase dependencies from environment.yml and create ordering file +split_environment_files: environment.yml .make.bootstrap + $(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture $(PYTHON_INTERPRETER) scripts/split_pip.py $(PROJECT_DIR)environment.yml .make.bootstrap: scripts/bootstrap.yml $(CONDA_EXE) env update -n $(PROJECT_NAME) -f scripts/bootstrap.yml @@ -69,6 +63,7 @@ endif # Checks that the conda environment is active environment_enabled: ifeq (conda,$(VIRTUALENV)) + $(CONDA_EXE) config --env --set channel_priority strict ifneq ($(notdir ${CONDA_DEFAULT_ENV}), $(PROJECT_NAME)) $(error Run "$(VIRTUALENV) activate $(PROJECT_NAME)" before proceeding...) endif diff --git a/{{ cookiecutter.repo_name }}/scripts/split_pip.py b/{{ cookiecutter.repo_name }}/scripts/split_pip.py index ecdc987..53e04fc 100644 --- a/{{ cookiecutter.repo_name }}/scripts/split_pip.py +++ b/{{ cookiecutter.repo_name }}/scripts/split_pip.py @@ -2,13 +2,19 @@ import json import sys import yaml +from collections import defaultdict -ACCEPTABLE_FORMATS = ["default", "pip", "pip-yaml", "conda-forge"] -def env_split(conda_env, kind="default"): - """Given a conda_environment dict, split into pip/nonpip versions +def env_split(conda_env, channel_order): + """Given a conda_environment dict, and a channel order, split into versions for each channel. + + Returns: + + conda_env: (list) + remaining setup bits of the environment.yml file + channel_dict: (dict) + dict containing the list of dependencies by channel name - conda_env: dict Python object corresponding to environment.yml""" # Cheater way to make deep Copies json_copy = json.dumps(conda_env) @@ -17,49 +23,63 @@ def env_split(conda_env, kind="default"): pipdeps = None deplist = conda_env.pop('dependencies') - conda_forge_list = [] + channel_dict = defaultdict(list) for k, dep in enumerate(deplist[:]): # Note: copy list, as we mutate it if isinstance(dep, dict): # nested yaml if dep.get('pip', None): - pipdeps = ["pip", deplist.pop(k)] + channel_dict['pip'] = deplist.pop(k) else: - prefix = 'conda-forge::' - if dep.startswith(prefix): - conda_forge_list.append(dep[len(prefix):]) + prefix_check = dep.split('::') + if len(prefix_check) > 1: + channel = prefix_check[0] + if not channel in channel_order: + raise Exception(f'the channel {channel} required for {dep} is not specified in a channel-order section of the environment file') + channel_dict[f'{channel}'].append(prefix_check[1]) deplist.remove(dep) - conda_env['dependencies'] = deplist - pip_env['dependencies'] = pipdeps - return conda_env, pip_env, conda_forge_list + channel_dict['defaults'] = deplist + conda_env.pop('channel-order') + return conda_env, channel_dict + +def get_channel_order(conda_env): + """ + Given a conda_environment dict, get the channels from the channel order. + """ + channel_order = conda_env.get('channel-order') + + if channel_order is None: + channel_order = ['defaults'] + if not 'defaults' in channel_order: + channel_order.insert(0, 'defaults') + channel_order.append('pip') + return channel_order def usage(): print(f""" -Usage: split_pip.py [{"|".join(ACCEPTABLE_FORMATS)}] path/to/environment.yml +Usage: split_pip.py path/to/environment.yml """) if __name__ == '__main__': - if len(sys.argv) != 3: - usage() - exit(1) - - kind = sys.argv[1] - if kind not in ACCEPTABLE_FORMATS: + if len(sys.argv) != 2: usage() exit(1) - with open(sys.argv[2], 'r') as yamlfile: + with open(sys.argv[1], 'r') as yamlfile: conda_env = yaml.safe_load(yamlfile) - cenv, penv, forgelist = env_split(conda_env) - if kind == "pip-yaml": - _ = yaml.dump(penv, sys.stdout, allow_unicode=True, default_flow_style=False) - elif kind == "pip": - print("\n".join(penv["dependencies"].pop(-1)["pip"])) - elif kind == "pip-yaml": - _ = yaml.dump(penv, sys.stdout, allow_unicode=True, default_flow_style=False) - elif kind == "default": - _ = yaml.dump(cenv, sys.stdout, allow_unicode=True, default_flow_style=False) - elif kind == "conda-forge": - print("\n".join(forgelist)) - else: - raise Exception(f"Invalid Kind: {kind}") + #check for acceptable formats + channel_order = get_channel_order(conda_env) + with open('.make.channel-order.include', 'w') as f: + f. write(' '.join(channel_order[:-1])) #exclude pip as a channel here + + cenv, channel_dict = env_split(conda_env, channel_order) + + for kind in channel_order: + if kind == "pip": + filename = '.make.pip-requirements.txt' + with open(filename, 'w') as f: + f.write("\n".join(channel_dict['pip']['pip'])) + else: + filename = f'.make.{kind}-environment.txt' + with open(filename, 'w') as f: + f.write("\n".join(channel_dict[kind])) From baa9fa2fa54e8ba5a0a03ea2fdd82019fc3ec305 Mon Sep 17 00:00:00 2001 From: Amy Wooding Date: Wed, 25 Jan 2023 10:38:14 -0500 Subject: [PATCH 02/13] use the template python version --- {{ cookiecutter.repo_name }}/scripts/bootstrap.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/{{ cookiecutter.repo_name }}/scripts/bootstrap.yml b/{{ cookiecutter.repo_name }}/scripts/bootstrap.yml index c52f026..4997352 100644 --- a/{{ cookiecutter.repo_name }}/scripts/bootstrap.yml +++ b/{{ cookiecutter.repo_name }}/scripts/bootstrap.yml @@ -1,5 +1,13 @@ +{% macro pyver() -%} +{% if cookiecutter.python_version == 'latest' -%} + - python=3 +{% else -%} + - python={{ cookiecutter.python_version }} +{% endif -%} +{% endmacro -%} +name: {{ cookiecutter.repo_name }} channels: - defaults dependencies: - - python=3.7 - pyyaml +{{ pyver()|indent(2, true) }} From 09c1fd4dfa3507535822baa1bce9434fb70e446b Mon Sep 17 00:00:00 2001 From: Amy Wooding Date: Wed, 25 Jan 2023 10:51:59 -0500 Subject: [PATCH 03/13] add test of the new environment code --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6c79e63..3bf0b03 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,12 +40,13 @@ jobs: pwd which make cookiecutter --config-file .cookiecutter-easydata-test-circleci.yml . -f --no-input - + - run: name: Create test-env environment and contrive to always use it command: | conda activate cookiecutter cd test-env + python scripts/tests/add-extra-channel-dependency.py export CONDA_EXE=/opt/conda/bin/conda make create_environment conda activate test-env From ceb636b39eb37eb719f0915fd8dc633b270b38b4 Mon Sep 17 00:00:00 2001 From: Amy Wooding Date: Wed, 25 Jan 2023 10:56:31 -0500 Subject: [PATCH 04/13] add missing file --- .../scripts/tests/add-extra-channel-dependency.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 {{ cookiecutter.repo_name }}/scripts/tests/add-extra-channel-dependency.py diff --git a/{{ cookiecutter.repo_name }}/scripts/tests/add-extra-channel-dependency.py b/{{ cookiecutter.repo_name }}/scripts/tests/add-extra-channel-dependency.py new file mode 100644 index 0000000..c615a4e --- /dev/null +++ b/{{ cookiecutter.repo_name }}/scripts/tests/add-extra-channel-dependency.py @@ -0,0 +1,14 @@ +import sys +import yaml + + +if __name__ == "__main__": + channel_order = ['defaults', 'pytorch'] + dependency_new = "pytorch::cpuonly" + + with open("environment.yml", "rt", encoding="utf-8") as file_env: + env = yaml.safe_load(file_env) + env["dependencies"].append(dependency_new) + env["channel_order"] = channel_order + with open("environment.yml", "wt", encoding="utf-8") as file_env: + yaml.safe_dump(env, file_env) From 9739191f857dff2bc8d420b98dbe449e556df651 Mon Sep 17 00:00:00 2001 From: Amy Wooding Date: Wed, 25 Jan 2023 11:08:37 -0500 Subject: [PATCH 05/13] fix indentation --- .../scripts/bootstrap.yml | 6 ++-- .../test-environment.yml | 33 +++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 {{ cookiecutter.repo_name }}/test-environment.yml diff --git a/{{ cookiecutter.repo_name }}/scripts/bootstrap.yml b/{{ cookiecutter.repo_name }}/scripts/bootstrap.yml index 4997352..20cd12d 100644 --- a/{{ cookiecutter.repo_name }}/scripts/bootstrap.yml +++ b/{{ cookiecutter.repo_name }}/scripts/bootstrap.yml @@ -7,7 +7,7 @@ {% endmacro -%} name: {{ cookiecutter.repo_name }} channels: - - defaults + - defaults dependencies: - - pyyaml -{{ pyver()|indent(2, true) }} + - pyyaml +{{ pyver()|indent(3, true) }} diff --git a/{{ cookiecutter.repo_name }}/test-environment.yml b/{{ cookiecutter.repo_name }}/test-environment.yml new file mode 100644 index 0000000..9845bc6 --- /dev/null +++ b/{{ cookiecutter.repo_name }}/test-environment.yml @@ -0,0 +1,33 @@ +channel_order: +- defaults +- pytorch +channels: +- defaults +dependencies: +- pip +- pip: + - -e . + - python-dotenv>=0.5.1 + - nbval + - nbdime + - gdown +- setuptools +- wheel +- git>=2.5 +- sphinx +- bokeh +- click +- colorcet +- coverage +- coveralls +- matplotlib +- jupyter +- scikit-learn +- scipy +- joblib +- nb_conda_kernels +- pandas +- requests +- pathlib +- fsspec +- pytorch::cpuonly From 25e41e37a0d59fa7241e8f35807b9aefe8a0dd21 Mon Sep 17 00:00:00 2001 From: Amy Wooding Date: Wed, 25 Jan 2023 11:12:04 -0500 Subject: [PATCH 06/13] fix typo and remove test file --- .../scripts/tests/add-extra-channel-dependency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/{{ cookiecutter.repo_name }}/scripts/tests/add-extra-channel-dependency.py b/{{ cookiecutter.repo_name }}/scripts/tests/add-extra-channel-dependency.py index c615a4e..8c41a6b 100644 --- a/{{ cookiecutter.repo_name }}/scripts/tests/add-extra-channel-dependency.py +++ b/{{ cookiecutter.repo_name }}/scripts/tests/add-extra-channel-dependency.py @@ -9,6 +9,6 @@ with open("environment.yml", "rt", encoding="utf-8") as file_env: env = yaml.safe_load(file_env) env["dependencies"].append(dependency_new) - env["channel_order"] = channel_order + env["channel-order"] = channel_order with open("environment.yml", "wt", encoding="utf-8") as file_env: yaml.safe_dump(env, file_env) From f0f548134adedc03b6fdd38ffabe7478b177e25c Mon Sep 17 00:00:00 2001 From: Amy Wooding Date: Wed, 25 Jan 2023 11:25:17 -0500 Subject: [PATCH 07/13] for latest, defautl to conda latest python version and remove test file --- {{ cookiecutter.repo_name }}/environment.yml | 2 +- .../scripts/bootstrap.yml | 2 +- .../test-environment.yml | 33 ------------------- 3 files changed, 2 insertions(+), 35 deletions(-) delete mode 100644 {{ cookiecutter.repo_name }}/test-environment.yml diff --git a/{{ cookiecutter.repo_name }}/environment.yml b/{{ cookiecutter.repo_name }}/environment.yml index 6749871..5982a14 100644 --- a/{{ cookiecutter.repo_name }}/environment.yml +++ b/{{ cookiecutter.repo_name }}/environment.yml @@ -1,6 +1,6 @@ {% macro pyver() -%} {% if cookiecutter.python_version == 'latest' -%} - - python=3 + - python {% else -%} - python={{ cookiecutter.python_version }} {% endif -%} diff --git a/{{ cookiecutter.repo_name }}/scripts/bootstrap.yml b/{{ cookiecutter.repo_name }}/scripts/bootstrap.yml index 20cd12d..d0e5cc0 100644 --- a/{{ cookiecutter.repo_name }}/scripts/bootstrap.yml +++ b/{{ cookiecutter.repo_name }}/scripts/bootstrap.yml @@ -1,6 +1,6 @@ {% macro pyver() -%} {% if cookiecutter.python_version == 'latest' -%} - - python=3 + - python {% else -%} - python={{ cookiecutter.python_version }} {% endif -%} diff --git a/{{ cookiecutter.repo_name }}/test-environment.yml b/{{ cookiecutter.repo_name }}/test-environment.yml deleted file mode 100644 index 9845bc6..0000000 --- a/{{ cookiecutter.repo_name }}/test-environment.yml +++ /dev/null @@ -1,33 +0,0 @@ -channel_order: -- defaults -- pytorch -channels: -- defaults -dependencies: -- pip -- pip: - - -e . - - python-dotenv>=0.5.1 - - nbval - - nbdime - - gdown -- setuptools -- wheel -- git>=2.5 -- sphinx -- bokeh -- click -- colorcet -- coverage -- coveralls -- matplotlib -- jupyter -- scikit-learn -- scipy -- joblib -- nb_conda_kernels -- pandas -- requests -- pathlib -- fsspec -- pytorch::cpuonly From b22f548c11593b291a2920933d5c62e2e3385a34 Mon Sep 17 00:00:00 2001 From: Amy Wooding Date: Wed, 25 Jan 2023 11:41:25 -0500 Subject: [PATCH 08/13] handle situation where channel-order doesn't exist --- .circleci/config.yml | 2 +- {{ cookiecutter.repo_name }}/scripts/split_pip.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3bf0b03..788c38a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,9 +46,9 @@ jobs: command: | conda activate cookiecutter cd test-env - python scripts/tests/add-extra-channel-dependency.py export CONDA_EXE=/opt/conda/bin/conda make create_environment + python scripts/tests/add-extra-channel-dependency.py conda activate test-env conda install -c anaconda make touch environment.yml diff --git a/{{ cookiecutter.repo_name }}/scripts/split_pip.py b/{{ cookiecutter.repo_name }}/scripts/split_pip.py index 53e04fc..62d059c 100644 --- a/{{ cookiecutter.repo_name }}/scripts/split_pip.py +++ b/{{ cookiecutter.repo_name }}/scripts/split_pip.py @@ -39,7 +39,7 @@ def env_split(conda_env, channel_order): deplist.remove(dep) channel_dict['defaults'] = deplist - conda_env.pop('channel-order') + conda_env.pop('channel-order', None) return conda_env, channel_dict def get_channel_order(conda_env): From 756485e283d90af3cc140913bf0e9a80cf2180c0 Mon Sep 17 00:00:00 2001 From: Amy Wooding Date: Mon, 30 Jan 2023 21:54:11 -0500 Subject: [PATCH 09/13] use a windows friendly loop --- {{ cookiecutter.repo_name }}/Makefile.envs | 6 ++---- {{ cookiecutter.repo_name }}/Makefile.include | 1 + {{ cookiecutter.repo_name }}/Makefile.win32 | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/{{ cookiecutter.repo_name }}/Makefile.envs b/{{ cookiecutter.repo_name }}/Makefile.envs index 5723f76..02feb89 100644 --- a/{{ cookiecutter.repo_name }}/Makefile.envs +++ b/{{ cookiecutter.repo_name }}/Makefile.envs @@ -6,10 +6,8 @@ include Makefile.include $(LOCKFILE): check_installation .make.bootstrap split_environment_files ifeq (conda, $(VIRTUALENV)) - for channel in $(shell $(CAT) .make.channel-order.include); do\ - $(ECHO) installing from .make.$$channel-environment.txt;\ - $(CONDA_EXE) install -n $(PROJECT_NAME) --file .make.$$channel-environment.txt --channel defaults --channel $$channel --strict-channel-priority --yes ;\ - done + $(foreach channel, $(shell $(CAT) .make.channel-order.include),\ + $(CONDA_EXE) install -n $(PROJECT_NAME) --file .make.$(channel)-environment.txt --channel defaults --channel $(channel) --strict-channel-priority --yes $(CMDSEP)) $(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture pip install -r .make.pip-requirements.txt $(CONDA_EXE) env export -n $(PROJECT_NAME) -f $(LOCKFILE) else diff --git a/{{ cookiecutter.repo_name }}/Makefile.include b/{{ cookiecutter.repo_name }}/Makefile.include index e8486ca..fc65727 100644 --- a/{{ cookiecutter.repo_name }}/Makefile.include +++ b/{{ cookiecutter.repo_name }}/Makefile.include @@ -19,5 +19,6 @@ CAT ?= cat SET ?= export WHICH ?= which DEVNULL ?= /dev/null +CMDSEP ?= ; $(warning From here on, using SHELL = $(SHELL)) diff --git a/{{ cookiecutter.repo_name }}/Makefile.win32 b/{{ cookiecutter.repo_name }}/Makefile.win32 index 92d8800..de046eb 100644 --- a/{{ cookiecutter.repo_name }}/Makefile.win32 +++ b/{{ cookiecutter.repo_name }}/Makefile.win32 @@ -5,6 +5,7 @@ CAT = type SET = set WHICH = where DEVNULL = nul +CMDSEP = & # Some UNIXish packages force the installation of a Bourne-compatible shell, and Make # prefers using this when it sees it. We thus force the usage of the good ole Batch From 6e69de1eff50e7931978ac951916e26fdb18d776 Mon Sep 17 00:00:00 2001 From: Amy Wooding Date: Wed, 1 Feb 2023 11:01:34 -0500 Subject: [PATCH 10/13] update environment management instructions to include channel order and arbitrary channel use --- .../reference/easydata/conda-environments.md | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/{{ cookiecutter.repo_name }}/reference/easydata/conda-environments.md b/{{ cookiecutter.repo_name }}/reference/easydata/conda-environments.md index 724d131..60a9a9f 100644 --- a/{{ cookiecutter.repo_name }}/reference/easydata/conda-environments.md +++ b/{{ cookiecutter.repo_name }}/reference/easydata/conda-environments.md @@ -81,6 +81,7 @@ When adding packages to your python environment, **do not `pip install` or `cond Your `environment.yml` file will look something like this: ``` name: {{ cookiecutter.repo_name }} +dependencies: - pip - pip: - -e . # conda >= 4.4 only @@ -106,7 +107,7 @@ name: {{ cookiecutter.repo_name }} ``` To add any package available from conda, add it to the end of the list. If you have a PYPI dependency that's not avaible via conda, add it to the list of pip installable dependencies under ` - pip:`. -You can include any {{ cookiecutter.upstream_location }} python-based project in the `pip` section via `git+https://{{ cookiecutter.upstream_location }}//`. +You can include any `{{ cookiecutter.upstream_location }}` python-based project in the `pip` section via `git+https://{{ cookiecutter.upstream_location }}//`. In particular, if you're working off of a fork or a work in progress branch of a repo in {{ cookiecutter.upstream_location }} (say, your personal version of ), you can change `git+https://{{ cookiecutter.upstream_location }}//` to @@ -117,6 +118,43 @@ Once you're done your edits, run `make update_environment` and voila, you're upd To share your updated environment, check in your `environment.yml` file. (More on this in [Sharing your Work](sharing-your-work.md)) +#### Adding packages from other conda channels +Say we want to add a package only available from the `conda-forge` conda channel and not the default conda channel. (The conda channel is what follows `-c` when using `conda install -c my-channel my-package`. Suppose we want to use `make` on windows. Then we need to use `conda-forge` since the default conda channel only has linux and macOS installations of `make`. To normally conda install this, we would use `conda install -c conda-forge make`. **We won't do that here**. + +Instead, we add a `channel-order` section that starts with `defaults` and lists the other channels we want to use in the order we want to install from them (note that this is a custom EasyData section to the `environment.yml`). Then we add our package in the dependency list in the form `channel-name::package-name`, for example, `conda-forge::make`. + +In this case an updated `environment.yml` file looks like this: +``` +name: {{ cookiecutter.repo_name }} +channel-order: + - defaults + - conda-forge +dependencies: + - pip + - pip: + - -e . # conda >= 4.4 only + - python-dotenv>=0.5.1 + - nbval + - nbdime + - umap-learn + - gdown + - setuptools + - wheel + - git>=2.5 # for git worktree template updating + - sphinx + - bokeh + - click + - colorcet + - coverage + - coveralls + - datashader + - holoviews + - matplotlib + - jupyter + - conda-forge::make +... +``` + #### Lock files Now, we'll admit that this workflow isn't perfectly reproducible in the sense that conda still has to resolve versions from the `environment.yml`. To make it more reproducible, running either `make create_environment` or `make update_environment` will generate an `environment.{$ARCH}.lock.yml` (e.g. `environment.i386.lock.yml`). This file keeps a record of the exact environment that is currently installed in your conda environment `{{ cookiecutter.repo_name }}`. If you ever need to reproduce an environment exactly, you can install from the `.lock.yml` file. (Note: These are architecture dependent). From d233bfbbf03b162054aeca515d64cf9397a079c6 Mon Sep 17 00:00:00 2001 From: Kjell Wooding Date: Wed, 1 Feb 2023 11:50:55 -0500 Subject: [PATCH 11/13] fix help messages. Some of these should not display --- {{ cookiecutter.repo_name }}/Makefile | 2 +- {{ cookiecutter.repo_name }}/Makefile.envs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/{{ cookiecutter.repo_name }}/Makefile b/{{ cookiecutter.repo_name }}/Makefile index addf322..2533593 100644 --- a/{{ cookiecutter.repo_name }}/Makefile +++ b/{{ cookiecutter.repo_name }}/Makefile @@ -75,7 +75,7 @@ test: update_environment $(if $(CI_RUNNING),--ignore=$(TESTS_NO_CI)) \ $(MODULE_NAME) -## Run all Unit Tests with coverage +## Run all Unit and code coverage tests test_with_coverage: update_environment $(SET) LOGLEVEL=DEBUG; coverage run -m pytest --pyargs --doctest-modules --doctest-continue-on-failure --verbose \ $(if $(CI_RUNNING),--ignore=$(TESTS_NO_CI)) \ diff --git a/{{ cookiecutter.repo_name }}/Makefile.envs b/{{ cookiecutter.repo_name }}/Makefile.envs index 02feb89..43396df 100644 --- a/{{ cookiecutter.repo_name }}/Makefile.envs +++ b/{{ cookiecutter.repo_name }}/Makefile.envs @@ -15,7 +15,7 @@ else endif .PHONY: split_environment_files -## extract multi-phase dependencies from environment.yml and create ordering file +# extract multi-phase dependencies from environment.yml and create ordering file split_environment_files: environment.yml .make.bootstrap $(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture $(PYTHON_INTERPRETER) scripts/split_pip.py $(PROJECT_DIR)environment.yml From 7c6b736d80fd081c55c4d1e58a19044f5d24e294 Mon Sep 17 00:00:00 2001 From: Kjell Wooding Date: Wed, 1 Feb 2023 11:51:10 -0500 Subject: [PATCH 12/13] remove lint target. We don't currently use this --- {{ cookiecutter.repo_name }}/Makefile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/{{ cookiecutter.repo_name }}/Makefile b/{{ cookiecutter.repo_name }}/Makefile index 2533593..40d2c52 100644 --- a/{{ cookiecutter.repo_name }}/Makefile +++ b/{{ cookiecutter.repo_name }}/Makefile @@ -81,11 +81,6 @@ test_with_coverage: update_environment $(if $(CI_RUNNING),--ignore=$(TESTS_NO_CI)) \ $(MODULE_NAME) -.PHONY: lint -## Lint using flake8 -lint: - flake8 $(MODULE_NAME) - .phony: help_update_easydata help_update_easydata: @$(PYTHON_INTERPRETER) scripts/help-update.py From 223c1fb62b9e8e149fef21ef00ff16d00e7baca1 Mon Sep 17 00:00:00 2001 From: Kjell Wooding Date: Wed, 1 Feb 2023 11:51:28 -0500 Subject: [PATCH 13/13] change this warning to a variable on the standard help page --- {{ cookiecutter.repo_name }}/Makefile | 2 +- {{ cookiecutter.repo_name }}/Makefile.include | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/{{ cookiecutter.repo_name }}/Makefile b/{{ cookiecutter.repo_name }}/Makefile index 40d2c52..15ba76e 100644 --- a/{{ cookiecutter.repo_name }}/Makefile +++ b/{{ cookiecutter.repo_name }}/Makefile @@ -100,7 +100,7 @@ debug: # Self Documenting Commands # ################################################################################# -HELP_VARS := PROJECT_NAME DEBUG_FILE ARCH PLATFORM +HELP_VARS := PROJECT_NAME DEBUG_FILE ARCH PLATFORM SHELL .DEFAULT_GOAL := show-help .PHONY: show-help diff --git a/{{ cookiecutter.repo_name }}/Makefile.include b/{{ cookiecutter.repo_name }}/Makefile.include index fc65727..85854ee 100644 --- a/{{ cookiecutter.repo_name }}/Makefile.include +++ b/{{ cookiecutter.repo_name }}/Makefile.include @@ -20,5 +20,3 @@ SET ?= export WHICH ?= which DEVNULL ?= /dev/null CMDSEP ?= ; - -$(warning From here on, using SHELL = $(SHELL))