diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml new file mode 100644 index 00000000000..b85ee058e2e --- /dev/null +++ b/.github/workflows/pr_checks.yml @@ -0,0 +1,158 @@ +name: PR Checks + +on: + pull_request: + branches: + - main + paths: + - fidesctl/** + - .github/workflows/pr_checks.yaml + +env: + CONTAINER: fidesctl-local + IMAGE: ethyca/fidesctl:local + +jobs: + Build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + + - name: Build fidesctl container + uses: docker/build-push-action@v2 + with: + builder: ${{ steps.buildx.outputs.name }} + context: ./fidesctl + file: ./fidesctl/Dockerfile + outputs: type=docker,dest=/tmp/${{ env.CONTAINER }}.tar + push: false + tags: ${{ env.IMAGE }} + + - name: Upload fidesctl container + uses: actions/upload-artifact@v2 + with: + name: ${{ env.CONTAINER }} + path: /tmp/${{ env.CONTAINER }}.tar + retention-days: 1 + + Fidesctl: + needs: Build + runs-on: ubuntu-latest + steps: + - name: Download fidesctl container + uses: actions/download-artifact@v2 + with: + name: ${{ env.CONTAINER }} + path: /tmp/ + + - name: Load fidesctl image + run: docker load --input /tmp/${{ env.CONTAINER }}.tar + + - name: Checkout + uses: actions/checkout@v2 + + - name: Check fidesctl installation + run: make check-install + + - name: Run fidesctl evaluation + run: make fidesctl + + Black: + needs: Build + runs-on: ubuntu-latest + steps: + - name: Download fidesctl container + uses: actions/download-artifact@v2 + with: + name: ${{ env.CONTAINER }} + path: /tmp/ + + - name: Load fidesctl image + run: docker load --input /tmp/${{ env.CONTAINER }}.tar + + - name: Checkout + uses: actions/checkout@v2 + + - name: Run formatter + run: make black + + Pylint: + needs: Build + runs-on: ubuntu-latest + steps: + - name: Download fidesctl container + uses: actions/download-artifact@v2 + with: + name: ${{ env.CONTAINER }} + path: /tmp/ + + - name: Load fidesctl image + run: docker load --input /tmp/${{ env.CONTAINER }}.tar + + - name: Checkout + uses: actions/checkout@v2 + + - name: Run linter + run: make pylint + + Mypy: + needs: Build + runs-on: ubuntu-latest + steps: + - name: Download fidesctl container + uses: actions/download-artifact@v2 + with: + name: ${{ env.CONTAINER }} + path: /tmp/ + + - name: Load fidesctl image + run: docker load --input /tmp/${{ env.CONTAINER }}.tar + + - name: Checkout + uses: actions/checkout@v2 + + - name: Run typechecker + run: make mypy + + Xenon: + needs: Build + runs-on: ubuntu-latest + steps: + - name: Download fidesctl container + uses: actions/download-artifact@v2 + with: + name: ${{ env.CONTAINER }} + path: /tmp/ + + - name: Load fidesctl image + run: docker load --input /tmp/${{ env.CONTAINER }}.tar + + - name: Checkout + uses: actions/checkout@v2 + + - name: Run cyclomatic complexity check + run: make xenon + + Test: + needs: Build + runs-on: ubuntu-latest + steps: + - name: Download fidesctl container + uses: actions/download-artifact@v2 + with: + name: ${{ env.CONTAINER }} + path: /tmp/ + + - name: Load fidesctl image + run: docker load --input /tmp/${{ env.CONTAINER }}.tar + + - name: Checkout + uses: actions/checkout@v2 + + - name: Run test suite + run: make pytest diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml deleted file mode 100644 index 9b2089bf752..00000000000 --- a/.github/workflows/test.yaml +++ /dev/null @@ -1,40 +0,0 @@ -name: Tests - -# Only test on Pull Requests that target main -on: - pull_request: - branches: - - main - paths: # Only run tests when certain paths change - - fidesctl/** - - .github/workflows/test.yaml - -jobs: - Tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Build the Docker Images - run: make compose-build - - - name: Check fidesctl installed - run: make check-install - - - name: Fidesctl Evaluation - run: make fidesctl - - - name: Format - run: make black - - - name: Lint - run: make pylint - - - name: TypeCheck - run: make mypy - - - name: Complexity Check - run: make xenon - - - name: Test - run: make pytest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 04c8c644d4c..b7643ad5af9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,6 +3,13 @@ minimum_pre_commit_version: "2" repos: - repo: local hooks: + - id: docker + name: docker + entry: make build-local + files: "^fidesctl/" + types_or: [file, python] + language: system + - id: black name: black entry: make black diff --git a/Makefile b/Makefile index cb79f0d9b04..d51bc4726b1 100644 --- a/Makefile +++ b/Makefile @@ -3,16 +3,19 @@ #################### # CONSTANTS #################### -RUN = docker-compose run --rm $(IMAGE_NAME) -RUN_NO_DEPS = docker-compose run --no-deps --rm $(IMAGE_NAME) - REGISTRY := ethyca IMAGE_TAG := $(shell git fetch --force --tags && git describe --tags --dirty --always) +# Image Names & Tags IMAGE_NAME := fidesctl IMAGE := $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG) +IMAGE_LOCAL := $(REGISTRY)/$(IMAGE_NAME):local IMAGE_LATEST := $(REGISTRY)/$(IMAGE_NAME):latest +# Run in Compose +RUN = docker compose run --rm $(IMAGE_NAME) +RUN_NO_DEPS = docker compose run --no-deps --rm $(IMAGE_NAME) + .PHONY: help help: @echo -------------------- @@ -37,30 +40,23 @@ help: # Dev #################### -.PHONY: init-db -init-db: compose-build - @echo "Checking for new migrations to run..." - @docker-compose up -d $(IMAGE_NAME) - @$(RUN) fidesctl init-db - @make teardown - .PHONY: reset-db -reset-db: compose-build +reset-db: build-local @echo "Reset the database..." - @docker-compose up -d $(IMAGE_NAME) + @docker compose up -d $(IMAGE_NAME) @$(RUN) fidesctl reset-db -y @make teardown .PHONY: api -api: compose-build +api: build-local @echo "Spinning up the webserver..." - @docker-compose up $(IMAGE_NAME) + @docker compose up $(IMAGE_NAME) @make teardown .PHONY: cli -cli: compose-build +cli: build-local @echo "Setting up a local development shell... (press CTRL-D to exit)" - @docker-compose up -d $(IMAGE_NAME) + @docker compose up -d $(IMAGE_NAME) @$(RUN) /bin/bash @make teardown @@ -71,6 +67,9 @@ cli: compose-build build: docker build --tag $(IMAGE) fidesctl/ +build-local: + docker build --tag $(IMAGE_LOCAL) fidesctl/ + push: build docker tag $(IMAGE) $(IMAGE_LATEST) docker push $(IMAGE) @@ -80,30 +79,32 @@ push: build # CI #################### -black: compose-build +black: @$(RUN_NO_DEPS) black --check src/ -check-all: check-install fidesctl black pylint mypy xenon pytest +# The order of dependent targets here is intentional +check-all: build-local check-install fidesctl black pylint mypy xenon pytest @echo "Running formatter, linter, typechecker and tests..." check-install: @echo "Checking that fidesctl is installed..." @$(RUN_NO_DEPS) fidesctl -fidesctl: compose-build +.PHONY: fidesctl +fidesctl: @$(RUN_NO_DEPS) fidesctl --local evaluate fides_resources/ -mypy: compose-build +mypy: @$(RUN_NO_DEPS) mypy -pylint: compose-build +pylint: @$(RUN_NO_DEPS) pylint src/ -pytest: compose-build - @docker-compose up -d $(IMAGE_NAME) +pytest: + @docker compose up -d $(IMAGE_NAME) @$(RUN) pytest -x -xenon: compose-build +xenon: @$(RUN_NO_DEPS) xenon src \ --max-absolute B \ --max-modules B \ @@ -124,22 +125,16 @@ clean: .PHONY: teardown teardown: @echo "Tearing down the dev environment..." - @docker-compose down + @docker compose down @echo "Teardown complete" -.PHONY: compose-build -compose-build: - @echo "Build the images required in the docker-compose file..." - @docker-compose down - @docker-compose build - .PHONY: docs-build -docs-build: compose-build - @docker-compose run --rm $(IMAGE_NAME) \ +docs-build: build-local + @docker compose run --rm $(IMAGE_NAME) \ python generate_openapi.py ../docs/fides/docs/api/openapi.json .PHONY: docs-serve docs-serve: docs-build - @docker-compose build docs - @docker-compose run --rm --service-ports docs \ + @docker compose build docs + @docker compose run --rm --service-ports docs \ /bin/bash -c "pip install -e /fidesctl && mkdocs serve --dev-addr=0.0.0.0:8000" diff --git a/docker-compose.yml b/docker-compose.yml index 156b41ffc19..1683dd00405 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,6 @@ services: fidesctl: - build: - context: fidesctl - dockerfile: Dockerfile + image: ethyca/fidesctl:local command: uvicorn --host 0.0.0.0 --port 8080 --reload fidesapi.main:app healthcheck: test: [ "CMD", "curl", "-f", "http://0.0.0.0:8080/health" ] @@ -43,7 +41,6 @@ services: docs: build: context: docs/fides/ - dockerfile: Dockerfile volumes: - ./docs/fides:/docs - ./fidesctl:/fidesctl diff --git a/fidesctl/Dockerfile b/fidesctl/Dockerfile index 57335a964c6..7f13c07b602 100644 --- a/fidesctl/Dockerfile +++ b/fidesctl/Dockerfile @@ -28,4 +28,5 @@ RUN pip install -e ".[all]" # Immediately flush to stdout, globally ENV PYTHONUNBUFFERED=TRUE +EXPOSE 8080 CMD ["fidesctl", "webserver"]