From 7ff4146110418f22855d71754472cf27a84a99a1 Mon Sep 17 00:00:00 2001 From: Matthew McGowan Date: Wed, 26 Jul 2023 15:01:18 -0700 Subject: [PATCH 1/5] chore: refactor workflows to use a common shared workflow --- .github/workflows/manual-run.yml | 64 ++------------------ .github/workflows/python-ci.yml | 90 ++++++++++++++++++++++++++++ .github/workflows/python-package.yml | 66 ++------------------ 3 files changed, 101 insertions(+), 119 deletions(-) create mode 100644 .github/workflows/python-ci.yml diff --git a/.github/workflows/manual-run.yml b/.github/workflows/manual-run.yml index 07903b7..2d8b295 100644 --- a/.github/workflows/manual-run.yml +++ b/.github/workflows/manual-run.yml @@ -7,62 +7,8 @@ on: workflow_dispatch jobs: build: - - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: [3.8, 3.9] - - steps: - - name: Send building notification - run: | - curl --request POST \ - --url 'https://api.notefile.net/?product=${{ secrets.NOTEHUB_PRODUCT_UID }}&device=${{ secrets.NOTECARD_DEVICE_ID }}' \ - --header 'Content-Type: application/json' \ - --header 'X-Session-Token: ${{ secrets.NOTEHUB_SESSION_TOKEN }}' \ - --data '{"req":"note.add","file":"build_results.qi","body":{"result":"building"}}' - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - make flake8 - # flake8 test/ notecard/ examples/ --count --ignore=E722,F401,F403,W503,E501 --show-source --statistics - - name: Lint Docs with Pydocstyle - run: | - make docstyle - # pydocstyle notecard/ examples/ - - name: Send running tests notification - run: | - curl --request POST \ - --url 'https://api.notefile.net/?product=${{ secrets.NOTEHUB_PRODUCT_UID }}&device=${{ secrets.NOTECARD_DEVICE_ID }}' \ - --header 'Content-Type: application/json' \ - --header 'X-Session-Token: ${{ secrets.NOTEHUB_SESSION_TOKEN }}' \ - --data '{"req":"note.add","file":"build_results.qi","body":{"result":"running_tests"}}' - - name: Test with pytest - run: | - make test - - name: Check if the job has succeeded - if: ${{ success() }} - run: | - curl --request POST \ - --url 'https://api.notefile.net/?product=${{ secrets.NOTEHUB_PRODUCT_UID }}&device=${{ secrets.NOTECARD_DEVICE_ID }}' \ - --header 'Content-Type: application/json' \ - --header 'X-Session-Token: ${{ secrets.NOTEHUB_SESSION_TOKEN }}' \ - --data '{"req":"note.add","file":"build_results.qi","body":{"result":"success"}}' - - name: Check if the job has failed - if: ${{ failure() }} - run: | - curl --request POST \ - --url 'https://api.notefile.net/?product=${{ secrets.NOTEHUB_PRODUCT_UID }}&device=${{ secrets.NOTECARD_DEVICE_ID }}' \ - --header 'Content-Type: application/json' \ - --header 'X-Session-Token: ${{ secrets.NOTEHUB_SESSION_TOKEN }}' \ - --data '{"req":"note.add","file":"build_results.qi","body":{"result":"tests_failed"}}' + uses: ./.github/workflows/python-ci.yml + secrets: inherit + with: + notehub_notify: false + coveralls: false \ No newline at end of file diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml new file mode 100644 index 0000000..75a188e --- /dev/null +++ b/.github/workflows/python-ci.yml @@ -0,0 +1,90 @@ +# Reusable workflow to run the python CI, which covers +# Optional notificiation to notehub (requires secrets NOTEHUB_SESSION_TOKEN, NOTEHUB_DEVICE_ID and NOTEHUB_PRODUCT_UID) +# Python installation and dependencies +# Linting (flake8 and docstyle) +# Testing and coverage with pytest +# Optionally publish coverage to coveralls (requires secrets.GITHUB_TOKEN) + +on: + workflow_call: + secrets: + NOTEHUB_SESSION_TOKEN: + NOTEHUB_PRODUCT_UID: + NOTECARD_DEVICE_ID: + inputs: + coveralls: + type: boolean + required: false + default: false + notehub_notify: + type: boolean + required: false + default: false + +jobs: + build: + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: [3.9] + + steps: + - name: Send building notification + if: ${{ inputs.notehub_notify }} + run: | + curl --request POST \ + --url 'https://api.notefile.net/?product=${{ secrets.NOTEHUB_PRODUCT_UID }}&device=${{ secrets.NOTECARD_DEVICE_ID }}' \ + --header 'Content-Type: application/json' \ + --header 'X-Session-Token: ${{ secrets.NOTEHUB_SESSION_TOKEN }}' \ + --data '{"req":"note.add","file":"build_results.qi","body":{"result":"building"}}' + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest coveralls + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + make flake8 + - name: Lint Docs with Pydocstyle + run: | + make docstyle + - name: Send running tests notification + if: ${{ inputs.notehub_notify }} + run: | + curl --request POST \ + --url 'https://api.notefile.net/?product=${{ secrets.NOTEHUB_PRODUCT_UID }}&device=${{ secrets.NOTECARD_DEVICE_ID }}' \ + --header 'Content-Type: application/json' \ + --header 'X-Session-Token: ${{ secrets.NOTEHUB_SESSION_TOKEN }}' \ + --data '{"req":"note.add","file":"build_results.qi","body":{"result":"running_tests"}}' + - name: Test with pytest + run: | + coverage run -m pytest + - name: Publish to Coveralls + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + if: ${{ inputs.coveralls }} + run: | + coveralls --service=github + + - name: Check if the job has succeeded + if: ${{ success() && inputs.notehub_notify }} + run: | + curl --request POST \ + --url 'https://api.notefile.net/?product=${{ secrets.NOTEHUB_PRODUCT_UID }}&device=${{ secrets.NOTECARD_DEVICE_ID }}' \ + --header 'Content-Type: application/json' \ + --header 'X-Session-Token: ${{ secrets.NOTEHUB_SESSION_TOKEN }}' \ + --data '{"req":"note.add","file":"build_results.qi","body":{"result":"success"}}' + - name: Check if the job has failed + if: ${{ failure() && inputs.notehub_notify }} + run: | + curl --request POST \ + --url 'https://api.notefile.net/?product=${{ secrets.NOTEHUB_PRODUCT_UID }}&device=${{ secrets.NOTECARD_DEVICE_ID }}' \ + --header 'Content-Type: application/json' \ + --header 'X-Session-Token: ${{ secrets.NOTEHUB_SESSION_TOKEN }}' \ + --data '{"req":"note.add","file":"build_results.qi","body":{"result":"tests_failed"}}' diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 93652de..add01da 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -8,66 +8,12 @@ on: branches: [ main ] pull_request: branches: [ main ] + workflow_dispatch: jobs: build: - - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: [3.6, 3.7, 3.8, 3.9] - - steps: - - name: Send building notification - run: | - curl --request POST \ - --url 'https://api.notefile.net/?product=${{ secrets.NOTEHUB_PRODUCT_UID }}&device=${{ secrets.NOTECARD_DEVICE_ID }}' \ - --header 'Content-Type: application/json' \ - --header 'X-Session-Token: ${{ secrets.NOTEHUB_SESSION_TOKEN }}' \ - --data '{"req":"note.add","file":"build_results.qi","body":{"result":"building"}}' - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest coveralls - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - make flake8 - - name: Lint Docs with Pydocstyle - run: | - make docstyle - - name: Send running tests notification - run: | - curl --request POST \ - --url 'https://api.notefile.net/?product=${{ secrets.NOTEHUB_PRODUCT_UID }}&device=${{ secrets.NOTECARD_DEVICE_ID }}' \ - --header 'Content-Type: application/json' \ - --header 'X-Session-Token: ${{ secrets.NOTEHUB_SESSION_TOKEN }}' \ - --data '{"req":"note.add","file":"build_results.qi","body":{"result":"running_tests"}}' - - name: Test with pytest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - coverage run -m pytest - coveralls --service=github - - name: Check if the job has succeeded - if: ${{ success() }} - run: | - curl --request POST \ - --url 'https://api.notefile.net/?product=${{ secrets.NOTEHUB_PRODUCT_UID }}&device=${{ secrets.NOTECARD_DEVICE_ID }}' \ - --header 'Content-Type: application/json' \ - --header 'X-Session-Token: ${{ secrets.NOTEHUB_SESSION_TOKEN }}' \ - --data '{"req":"note.add","file":"build_results.qi","body":{"result":"success"}}' - - name: Check if the job has failed - if: ${{ failure() }} - run: | - curl --request POST \ - --url 'https://api.notefile.net/?product=${{ secrets.NOTEHUB_PRODUCT_UID }}&device=${{ secrets.NOTECARD_DEVICE_ID }}' \ - --header 'Content-Type: application/json' \ - --header 'X-Session-Token: ${{ secrets.NOTEHUB_SESSION_TOKEN }}' \ - --data '{"req":"note.add","file":"build_results.qi","body":{"result":"tests_failed"}}' + uses: ./.github/workflows/python-ci.yml + secrets: inherit + with: + notehub_notify: true + coveralls: true \ No newline at end of file From 193839bbc79c912aa3361f709a0ebf03410ad41a Mon Sep 17 00:00:00 2001 From: Matthew McGowan Date: Wed, 26 Jul 2023 15:31:09 -0700 Subject: [PATCH 2/5] chore: adds test run to datadog when the api key is defined --- .github/workflows/python-ci.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 75a188e..10223bf 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -4,6 +4,7 @@ # Linting (flake8 and docstyle) # Testing and coverage with pytest # Optionally publish coverage to coveralls (requires secrets.GITHUB_TOKEN) +# Reports test coverage to DataDog if secrets.DD_API_KEY is defined. on: workflow_call: @@ -26,7 +27,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [3.9] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - name: Send building notification @@ -45,7 +46,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest coveralls + pip install -U flake8 pytest coveralls ddtrace if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Lint with flake8 run: | @@ -63,8 +64,13 @@ jobs: --header 'X-Session-Token: ${{ secrets.NOTEHUB_SESSION_TOKEN }}' \ --data '{"req":"note.add","file":"build_results.qi","body":{"result":"running_tests"}}' - name: Test with pytest + env: + DD_API_KEY: ${{ secrets.DD_API_KEY }} + DD_CIVISIBILITY_AGENTLESS_ENABLED: ${{ !!secrets.DD_API_KEY }} + DD_SERVICE: note-python + DD_ENV: ci run: | - coverage run -m pytest + coverage run -m pytest --ddtrace - name: Publish to Coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 6dcf5af73430b53250eccd5f5cf38671cd00a800 Mon Sep 17 00:00:00 2001 From: Matthew McGowan Date: Wed, 26 Jul 2023 15:38:15 -0700 Subject: [PATCH 3/5] chore: warn if DD_API_KEY is not defined. --- .github/workflows/python-ci.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 10223bf..14373cb 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -29,6 +29,9 @@ jobs: matrix: python-version: [3.6, 3.7, 3.8, 3.9] + env: + DD_API_KEY: ${{ secrets.DD_API_KEY }} + steps: - name: Send building notification if: ${{ inputs.notehub_notify }} @@ -63,10 +66,15 @@ jobs: --header 'Content-Type: application/json' \ --header 'X-Session-Token: ${{ secrets.NOTEHUB_SESSION_TOKEN }}' \ --data '{"req":"note.add","file":"build_results.qi","body":{"result":"running_tests"}}' + + - name: Check DD API Key + if: ${{ !env.DD_API_KEY }} + run: | + echo Test run will NOT be collected by DD + - name: Test with pytest env: - DD_API_KEY: ${{ secrets.DD_API_KEY }} - DD_CIVISIBILITY_AGENTLESS_ENABLED: ${{ !!secrets.DD_API_KEY }} + DD_CIVISIBILITY_AGENTLESS_ENABLED: ${{ !!env.DD_API_KEY }} DD_SERVICE: note-python DD_ENV: ci run: | From 8426e7a4b3f6045bb800f8395c64b8b0b32250f7 Mon Sep 17 00:00:00 2001 From: Matthew McGowan Date: Wed, 26 Jul 2023 16:26:53 -0700 Subject: [PATCH 4/5] chore: try the `--ddtrace-patch-all` flag --- .github/workflows/python-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 14373cb..4fa74f9 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -78,7 +78,7 @@ jobs: DD_SERVICE: note-python DD_ENV: ci run: | - coverage run -m pytest --ddtrace + coverage run -m pytest --ddtrace --ddtrace-patch-all - name: Publish to Coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 52c3d9f7ed5afdf1db90caa4021de39368d8e988 Mon Sep 17 00:00:00 2001 From: Matthew McGowan Date: Fri, 28 Jul 2023 12:21:53 -0700 Subject: [PATCH 5/5] style: line endings --- .github/workflows/manual-run.yml | 2 +- .github/workflows/python-package.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual-run.yml b/.github/workflows/manual-run.yml index 2d8b295..2ead9e4 100644 --- a/.github/workflows/manual-run.yml +++ b/.github/workflows/manual-run.yml @@ -11,4 +11,4 @@ jobs: secrets: inherit with: notehub_notify: false - coveralls: false \ No newline at end of file + coveralls: false diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index add01da..5d8027f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,4 +16,4 @@ jobs: secrets: inherit with: notehub_notify: true - coveralls: true \ No newline at end of file + coveralls: true