From 443352a01cfdaf273ab5ff3b4fbee525cbadca10 Mon Sep 17 00:00:00 2001 From: Thomas Riccardi Date: Thu, 3 Oct 2019 15:40:50 +0200 Subject: [PATCH 1/2] Fix .dockerignore --- .dockerignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 1abcd93..7cfb35e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,7 +2,7 @@ dmake.yml .dmake/ DMakefile -Dockerfile +deploy/Dockerfile .git **/.gitignore .dockerignore From 48d3a8408f8e2394518a3dce8aa3cddfd0ed06af Mon Sep 17 00:00:00 2001 From: Thomas Riccardi Date: Thu, 3 Oct 2019 15:43:19 +0200 Subject: [PATCH 2/2] Fix code coverage It was previously broken because it tested on the installed egg. We now split unit tests and egg tests (with demo.py) so code coverage works on source code (and still test somehow the egg). Also, move dev dependencies to requirements.dev.txt & bump them. Also, store egg on host ./dist Now the full workflow is split into 3 types of dmake services: - client:: - dmake build: linter - dmake test: unit test - build-egg: one service to build the universal egg - dmake build: build egg - dmake test: copy egg to host ./dist - test-egg-py{2,3}: - dmake test: test egg from host ./dist with demo.py --- .dockerignore | 1 + .gitignore | 1 + .travis.yml | 2 +- Makefile | 11 +++---- deploy/Dockerfile | 29 ++++++++++++++----- deploy/install.sh | 7 +---- dmake.yml | 69 +++++++++++++++++++++++++++++++++++++++++++- requirements.dev.txt | 5 ++++ 8 files changed, 105 insertions(+), 20 deletions(-) create mode 100644 requirements.dev.txt diff --git a/.dockerignore b/.dockerignore index 7cfb35e..d435ae9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,6 +16,7 @@ venv* build/ dist/ +*.egg-info/ *-coverage.xml *-junit.xml diff --git a/.gitignore b/.gitignore index 59b6c3c..3a336c0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ build/ dist/ +*.egg-info/ *-coverage.xml *-junit.xml diff --git a/.travis.yml b/.travis.yml index f7c19f4..a85761c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,6 @@ install: # command to run tests script: - - pip install pytest==4.0.2 pytest-voluptuous==1.1.0 httpretty==0.9.6 # for testing + - pip install -r requirements.dev.txt - LOG_LEVEL=DEBUG python -m pytest --junit-xml=junit.xml -vv app/tests - LOG_LEVEL=DEBUG python app/demo.py diff --git a/Makefile b/Makefile index 48cd18a..9510e1a 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,17 @@ all: release -release: clean - python3 setup.py sdist bdist_wheel +build: clean + # lint + unit tests + build egg (available in host ./dist) + test egg on python 2 and 3 + dmake test test-egg-py2 test-egg-py3 clean: - rm -rf build dist *.egg-info + rm -rf dist -publish-test: release +publish-test: build # For testing, note that once one version is uploaded, you have to increment the version number or make a post release to re-upload # https://www.python.org/dev/peps/pep-0440/#post-releases twine upload --repository-url https://test.pypi.org/legacy/ dist/* -publish: release +publish: build # More info here https://packaging.python.org/tutorials/packaging-projects/ twine upload dist/* diff --git a/deploy/Dockerfile b/deploy/Dockerfile index ddd5e64..7cb1700 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,21 +1,36 @@ ARG BASE_IMAGE -FROM ${BASE_IMAGE} as builder +FROM ${BASE_IMAGE} as dev WORKDIR /app COPY . . # lint check RUN flake8 --statistics --verbose +# prepare local dev environment, for tests execution +RUN pip install -e . + + +FROM dev as build + +# build egg RUN python setup.py bdist_wheel -FROM ${BASE_IMAGE} as runtime +# ideally use ROOT_IMAGE but it's not yet doable in dmake +# => manually force root image here +FROM python:2.7 as runtime-py2 WORKDIR /app +# don't copy egg there: use universal egg from `build-egg` service at runtime +# prepare egg test: demo.py +COPY --from=build /app/demo.py /app/ -# copy egg -COPY --from=builder /app/dist/deepomatic_api-*.whl /tmp/ -COPY --from=builder /app/demo.py /app/ -COPY --from=builder /app/tests /app/tests -RUN pip install /tmp/deepomatic_api-*.whl +# ideally use ROOT_IMAGE but it's not yet doable in dmake +# => manually force root image here +FROM python:3.6 as runtime-py3 + +WORKDIR /app +# don't copy egg there: use universal egg from `build-egg` service at runtime +# prepare egg test: demo.py +COPY --from=build /app/demo.py /app/ diff --git a/deploy/install.sh b/deploy/install.sh index 494bdbc..576944b 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -2,9 +2,4 @@ set -e -apt-get update && apt-get install -y build-essential -pip install -r requirements.txt -pip install pytest==4.0.2 \ - pytest-cov==2.6.1 \ - pytest-voluptuous==1.1.0 \ - httpretty==0.9.6 # for testing +pip install -r requirements.dev.txt diff --git a/dmake.yml b/dmake.yml index 3e41d5c..0d32741 100644 --- a/dmake.yml +++ b/dmake.yml @@ -23,6 +23,7 @@ docker: root_image: python:2.7 copy_files: - requirements.txt + - requirements.dev.txt install_scripts: - deploy/install.sh # Nothing changes comparing to above, except 'variant' and 'root_image' @@ -44,6 +45,7 @@ services: build: context: . dockerfile: deploy/Dockerfile + target: dev base_image_variant: - 2.7 - 3.4 @@ -52,9 +54,74 @@ services: tests: commands: - LOG_LEVEL=DEBUG pytest --junit-xml=junit.xml --cov=. --cov-report=xml:coverage.xml --cov-report html:cover -vv /app/tests - - LOG_LEVEL=DEBUG python /app/demo.py junit_report: junit.xml cobertura_report: coverage.xml html_report: directory: cover title: HTML coverage report + + - service_name: build-egg + # unit tests must have passed on all supported platforms to build the universal egg + needed_services: + - client:2.7 + - client:3.4 + - client:3.5 + - client:3.6 + config: + docker_image: + build: + context: . + dockerfile: deploy/Dockerfile + target: build + base_image_variant: 3.6 + tests: + # TODO follow up on https://github.com/Deepomatic/dmake/issues/311 + data_volumes: + - container_volume: /dist/ + source: ./dist/ + commands: + # copy universal egg + - rm -rf /dist/* + - cp /app/dist/* /dist + - chmod -R a+w /dist + + - service_name: test-egg-py2 + needed_services: + - build-egg + config: + docker_image: + build: + context: . + dockerfile: deploy/Dockerfile + target: runtime-py2 + # only used by dmake shell; bypassed by runtime-py2 Dockerfile target + base_image_variant: + - 2.7 + tests: + data_volumes: + - container_volume: /dist/ + source: ./dist/ + commands: + - pip install /dist/deepomatic_api-*.whl + - LOG_LEVEL=DEBUG python /app/demo.py + + + - service_name: test-egg-py3 + needed_services: + - build-egg + config: + docker_image: + build: + context: . + dockerfile: deploy/Dockerfile + target: runtime-py3 + # only used by dmake shell; bypassed by runtime-py3 Dockerfile target + base_image_variant: + - 3.6 + tests: + data_volumes: + - container_volume: /dist/ + source: ./dist/ + commands: + - pip install /dist/deepomatic_api-*.whl + - LOG_LEVEL=DEBUG python /app/demo.py diff --git a/requirements.dev.txt b/requirements.dev.txt new file mode 100644 index 0000000..6a2542d --- /dev/null +++ b/requirements.dev.txt @@ -0,0 +1,5 @@ +-r requirements.txt +pytest==4.6.5 +pytest-cov==2.7.1 +pytest-voluptuous==1.1.0 +httpretty==0.9.6