diff --git a/.circleci/config.yml b/.circleci/config.yml index 6c79e63..788c38a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,7 +40,7 @@ jobs: pwd which make cookiecutter --config-file .cookiecutter-easydata-test-circleci.yml . -f --no-input - + - run: name: Create test-env environment and contrive to always use it command: | @@ -48,6 +48,7 @@ jobs: cd test-env export CONDA_EXE=/opt/conda/bin/conda make create_environment + python scripts/tests/add-extra-channel-dependency.py conda activate test-env conda install -c anaconda make touch environment.yml diff --git a/{{ cookiecutter.repo_name }}/Makefile b/{{ cookiecutter.repo_name }}/Makefile index addf322..15ba76e 100644 --- a/{{ cookiecutter.repo_name }}/Makefile +++ b/{{ cookiecutter.repo_name }}/Makefile @@ -75,17 +75,12 @@ test: update_environment $(if $(CI_RUNNING),--ignore=$(TESTS_NO_CI)) \ $(MODULE_NAME) -## Run all Unit Tests with coverage +## Run all Unit and code coverage tests test_with_coverage: update_environment $(SET) LOGLEVEL=DEBUG; coverage run -m pytest --pyargs --doctest-modules --doctest-continue-on-failure --verbose \ $(if $(CI_RUNNING),--ignore=$(TESTS_NO_CI)) \ $(MODULE_NAME) -.PHONY: lint -## Lint using flake8 -lint: - flake8 $(MODULE_NAME) - .phony: help_update_easydata help_update_easydata: @$(PYTHON_INTERPRETER) scripts/help-update.py @@ -105,7 +100,7 @@ debug: # Self Documenting Commands # ################################################################################# -HELP_VARS := PROJECT_NAME DEBUG_FILE ARCH PLATFORM +HELP_VARS := PROJECT_NAME DEBUG_FILE ARCH PLATFORM SHELL .DEFAULT_GOAL := show-help .PHONY: show-help diff --git a/{{ cookiecutter.repo_name }}/Makefile.envs b/{{ cookiecutter.repo_name }}/Makefile.envs index 4c65eb7..43396df 100644 --- a/{{ cookiecutter.repo_name }}/Makefile.envs +++ b/{{ cookiecutter.repo_name }}/Makefile.envs @@ -4,28 +4,20 @@ include Makefile.include -$(LOCKFILE): check_installation .make.bootstrap .make.pip-requirements.txt .make.environment-default.yml .make.conda-forge-requirements.txt +$(LOCKFILE): check_installation .make.bootstrap split_environment_files ifeq (conda, $(VIRTUALENV)) - $(CONDA_EXE) env update -n $(PROJECT_NAME) -f .make.environment-default.yml --prune - $(CONDA_EXE) install -n $(PROJECT_NAME) --file .make.conda-forge-requirements.txt --channel defaults --channel conda-forge --strict-channel-priority --yes + $(foreach channel, $(shell $(CAT) .make.channel-order.include),\ + $(CONDA_EXE) install -n $(PROJECT_NAME) --file .make.$(channel)-environment.txt --channel defaults --channel $(channel) --strict-channel-priority --yes $(CMDSEP)) $(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture pip install -r .make.pip-requirements.txt $(CONDA_EXE) env export -n $(PROJECT_NAME) -f $(LOCKFILE) else $(error Unsupported Environment `$(VIRTUALENV)`. Use conda) endif -# extract multi-phase dependencies from environment.yml -.make.environment-pip.yml: environment.yml .make.bootstrap - $(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture $(PYTHON_INTERPRETER) scripts/split_pip.py pip-yaml $(PROJECT_DIR)environment.yml > $@ - -.make.pip-requirements.txt: environment.yml .make.bootstrap - $(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture $(PYTHON_INTERPRETER) scripts/split_pip.py pip $(PROJECT_DIR)environment.yml > $@ - -.make.conda-forge-requirements.txt: environment.yml .make.bootstrap - $(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture $(PYTHON_INTERPRETER) scripts/split_pip.py conda-forge $(PROJECT_DIR)environment.yml > $@ - -.make.environment-default.yml: environment.yml .make.bootstrap - $(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture $(PYTHON_INTERPRETER) scripts/split_pip.py default $(PROJECT_DIR)environment.yml > $@ +.PHONY: split_environment_files +# extract multi-phase dependencies from environment.yml and create ordering file +split_environment_files: environment.yml .make.bootstrap + $(CONDA_EXE) run -n $(PROJECT_NAME) --no-capture $(PYTHON_INTERPRETER) scripts/split_pip.py $(PROJECT_DIR)environment.yml .make.bootstrap: scripts/bootstrap.yml $(CONDA_EXE) env update -n $(PROJECT_NAME) -f scripts/bootstrap.yml @@ -69,6 +61,7 @@ endif # Checks that the conda environment is active environment_enabled: ifeq (conda,$(VIRTUALENV)) + $(CONDA_EXE) config --env --set channel_priority strict ifneq ($(notdir ${CONDA_DEFAULT_ENV}), $(PROJECT_NAME)) $(error Run "$(VIRTUALENV) activate $(PROJECT_NAME)" before proceeding...) endif diff --git a/{{ cookiecutter.repo_name }}/Makefile.include b/{{ cookiecutter.repo_name }}/Makefile.include index e8486ca..85854ee 100644 --- a/{{ cookiecutter.repo_name }}/Makefile.include +++ b/{{ cookiecutter.repo_name }}/Makefile.include @@ -19,5 +19,4 @@ CAT ?= cat SET ?= export WHICH ?= which DEVNULL ?= /dev/null - -$(warning From here on, using SHELL = $(SHELL)) +CMDSEP ?= ; diff --git a/{{ cookiecutter.repo_name }}/Makefile.win32 b/{{ cookiecutter.repo_name }}/Makefile.win32 index 92d8800..de046eb 100644 --- a/{{ cookiecutter.repo_name }}/Makefile.win32 +++ b/{{ cookiecutter.repo_name }}/Makefile.win32 @@ -5,6 +5,7 @@ CAT = type SET = set WHICH = where DEVNULL = nul +CMDSEP = & # Some UNIXish packages force the installation of a Bourne-compatible shell, and Make # prefers using this when it sees it. We thus force the usage of the good ole Batch diff --git a/{{ cookiecutter.repo_name }}/environment.yml b/{{ cookiecutter.repo_name }}/environment.yml index 6749871..5982a14 100644 --- a/{{ cookiecutter.repo_name }}/environment.yml +++ b/{{ cookiecutter.repo_name }}/environment.yml @@ -1,6 +1,6 @@ {% macro pyver() -%} {% if cookiecutter.python_version == 'latest' -%} - - python=3 + - python {% else -%} - python={{ cookiecutter.python_version }} {% endif -%} diff --git a/{{ cookiecutter.repo_name }}/reference/easydata/conda-environments.md b/{{ cookiecutter.repo_name }}/reference/easydata/conda-environments.md index 724d131..60a9a9f 100644 --- a/{{ cookiecutter.repo_name }}/reference/easydata/conda-environments.md +++ b/{{ cookiecutter.repo_name }}/reference/easydata/conda-environments.md @@ -81,6 +81,7 @@ When adding packages to your python environment, **do not `pip install` or `cond Your `environment.yml` file will look something like this: ``` name: {{ cookiecutter.repo_name }} +dependencies: - pip - pip: - -e . # conda >= 4.4 only @@ -106,7 +107,7 @@ name: {{ cookiecutter.repo_name }} ``` To add any package available from conda, add it to the end of the list. If you have a PYPI dependency that's not avaible via conda, add it to the list of pip installable dependencies under ` - pip:`. -You can include any {{ cookiecutter.upstream_location }} python-based project in the `pip` section via `git+https://{{ cookiecutter.upstream_location }}//`. +You can include any `{{ cookiecutter.upstream_location }}` python-based project in the `pip` section via `git+https://{{ cookiecutter.upstream_location }}//`. In particular, if you're working off of a fork or a work in progress branch of a repo in {{ cookiecutter.upstream_location }} (say, your personal version of ), you can change `git+https://{{ cookiecutter.upstream_location }}//` to @@ -117,6 +118,43 @@ Once you're done your edits, run `make update_environment` and voila, you're upd To share your updated environment, check in your `environment.yml` file. (More on this in [Sharing your Work](sharing-your-work.md)) +#### Adding packages from other conda channels +Say we want to add a package only available from the `conda-forge` conda channel and not the default conda channel. (The conda channel is what follows `-c` when using `conda install -c my-channel my-package`. Suppose we want to use `make` on windows. Then we need to use `conda-forge` since the default conda channel only has linux and macOS installations of `make`. To normally conda install this, we would use `conda install -c conda-forge make`. **We won't do that here**. + +Instead, we add a `channel-order` section that starts with `defaults` and lists the other channels we want to use in the order we want to install from them (note that this is a custom EasyData section to the `environment.yml`). Then we add our package in the dependency list in the form `channel-name::package-name`, for example, `conda-forge::make`. + +In this case an updated `environment.yml` file looks like this: +``` +name: {{ cookiecutter.repo_name }} +channel-order: + - defaults + - conda-forge +dependencies: + - pip + - pip: + - -e . # conda >= 4.4 only + - python-dotenv>=0.5.1 + - nbval + - nbdime + - umap-learn + - gdown + - setuptools + - wheel + - git>=2.5 # for git worktree template updating + - sphinx + - bokeh + - click + - colorcet + - coverage + - coveralls + - datashader + - holoviews + - matplotlib + - jupyter + - conda-forge::make +... +``` + #### Lock files Now, we'll admit that this workflow isn't perfectly reproducible in the sense that conda still has to resolve versions from the `environment.yml`. To make it more reproducible, running either `make create_environment` or `make update_environment` will generate an `environment.{$ARCH}.lock.yml` (e.g. `environment.i386.lock.yml`). This file keeps a record of the exact environment that is currently installed in your conda environment `{{ cookiecutter.repo_name }}`. If you ever need to reproduce an environment exactly, you can install from the `.lock.yml` file. (Note: These are architecture dependent). diff --git a/{{ cookiecutter.repo_name }}/scripts/bootstrap.yml b/{{ cookiecutter.repo_name }}/scripts/bootstrap.yml index c52f026..d0e5cc0 100644 --- a/{{ cookiecutter.repo_name }}/scripts/bootstrap.yml +++ b/{{ cookiecutter.repo_name }}/scripts/bootstrap.yml @@ -1,5 +1,13 @@ +{% macro pyver() -%} +{% if cookiecutter.python_version == 'latest' -%} + - python +{% else -%} + - python={{ cookiecutter.python_version }} +{% endif -%} +{% endmacro -%} +name: {{ cookiecutter.repo_name }} channels: - - defaults + - defaults dependencies: - - python=3.7 - - pyyaml + - pyyaml +{{ pyver()|indent(3, true) }} diff --git a/{{ cookiecutter.repo_name }}/scripts/split_pip.py b/{{ cookiecutter.repo_name }}/scripts/split_pip.py index ecdc987..62d059c 100644 --- a/{{ cookiecutter.repo_name }}/scripts/split_pip.py +++ b/{{ cookiecutter.repo_name }}/scripts/split_pip.py @@ -2,13 +2,19 @@ import json import sys import yaml +from collections import defaultdict -ACCEPTABLE_FORMATS = ["default", "pip", "pip-yaml", "conda-forge"] -def env_split(conda_env, kind="default"): - """Given a conda_environment dict, split into pip/nonpip versions +def env_split(conda_env, channel_order): + """Given a conda_environment dict, and a channel order, split into versions for each channel. + + Returns: + + conda_env: (list) + remaining setup bits of the environment.yml file + channel_dict: (dict) + dict containing the list of dependencies by channel name - conda_env: dict Python object corresponding to environment.yml""" # Cheater way to make deep Copies json_copy = json.dumps(conda_env) @@ -17,49 +23,63 @@ def env_split(conda_env, kind="default"): pipdeps = None deplist = conda_env.pop('dependencies') - conda_forge_list = [] + channel_dict = defaultdict(list) for k, dep in enumerate(deplist[:]): # Note: copy list, as we mutate it if isinstance(dep, dict): # nested yaml if dep.get('pip', None): - pipdeps = ["pip", deplist.pop(k)] + channel_dict['pip'] = deplist.pop(k) else: - prefix = 'conda-forge::' - if dep.startswith(prefix): - conda_forge_list.append(dep[len(prefix):]) + prefix_check = dep.split('::') + if len(prefix_check) > 1: + channel = prefix_check[0] + if not channel in channel_order: + raise Exception(f'the channel {channel} required for {dep} is not specified in a channel-order section of the environment file') + channel_dict[f'{channel}'].append(prefix_check[1]) deplist.remove(dep) - conda_env['dependencies'] = deplist - pip_env['dependencies'] = pipdeps - return conda_env, pip_env, conda_forge_list + channel_dict['defaults'] = deplist + conda_env.pop('channel-order', None) + return conda_env, channel_dict + +def get_channel_order(conda_env): + """ + Given a conda_environment dict, get the channels from the channel order. + """ + channel_order = conda_env.get('channel-order') + + if channel_order is None: + channel_order = ['defaults'] + if not 'defaults' in channel_order: + channel_order.insert(0, 'defaults') + channel_order.append('pip') + return channel_order def usage(): print(f""" -Usage: split_pip.py [{"|".join(ACCEPTABLE_FORMATS)}] path/to/environment.yml +Usage: split_pip.py path/to/environment.yml """) if __name__ == '__main__': - if len(sys.argv) != 3: - usage() - exit(1) - - kind = sys.argv[1] - if kind not in ACCEPTABLE_FORMATS: + if len(sys.argv) != 2: usage() exit(1) - with open(sys.argv[2], 'r') as yamlfile: + with open(sys.argv[1], 'r') as yamlfile: conda_env = yaml.safe_load(yamlfile) - cenv, penv, forgelist = env_split(conda_env) - if kind == "pip-yaml": - _ = yaml.dump(penv, sys.stdout, allow_unicode=True, default_flow_style=False) - elif kind == "pip": - print("\n".join(penv["dependencies"].pop(-1)["pip"])) - elif kind == "pip-yaml": - _ = yaml.dump(penv, sys.stdout, allow_unicode=True, default_flow_style=False) - elif kind == "default": - _ = yaml.dump(cenv, sys.stdout, allow_unicode=True, default_flow_style=False) - elif kind == "conda-forge": - print("\n".join(forgelist)) - else: - raise Exception(f"Invalid Kind: {kind}") + #check for acceptable formats + channel_order = get_channel_order(conda_env) + with open('.make.channel-order.include', 'w') as f: + f. write(' '.join(channel_order[:-1])) #exclude pip as a channel here + + cenv, channel_dict = env_split(conda_env, channel_order) + + for kind in channel_order: + if kind == "pip": + filename = '.make.pip-requirements.txt' + with open(filename, 'w') as f: + f.write("\n".join(channel_dict['pip']['pip'])) + else: + filename = f'.make.{kind}-environment.txt' + with open(filename, 'w') as f: + f.write("\n".join(channel_dict[kind])) diff --git a/{{ cookiecutter.repo_name }}/scripts/tests/add-extra-channel-dependency.py b/{{ cookiecutter.repo_name }}/scripts/tests/add-extra-channel-dependency.py new file mode 100644 index 0000000..8c41a6b --- /dev/null +++ b/{{ cookiecutter.repo_name }}/scripts/tests/add-extra-channel-dependency.py @@ -0,0 +1,14 @@ +import sys +import yaml + + +if __name__ == "__main__": + channel_order = ['defaults', 'pytorch'] + dependency_new = "pytorch::cpuonly" + + with open("environment.yml", "rt", encoding="utf-8") as file_env: + env = yaml.safe_load(file_env) + env["dependencies"].append(dependency_new) + env["channel-order"] = channel_order + with open("environment.yml", "wt", encoding="utf-8") as file_env: + yaml.safe_dump(env, file_env)