From 9c3153882c95237495adb44637d68ab7cf4424cf Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Thu, 23 Mar 2023 13:35:29 +0100 Subject: [PATCH 01/19] FIX: Workflow file for persistence performance - wrong name - trying to fix install --- .github/workflows/persistence-performance.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/persistence-performance.yml b/.github/workflows/persistence-performance.yml index 6cc6aa7c..913ba4a7 100644 --- a/.github/workflows/persistence-performance.yml +++ b/.github/workflows/persistence-performance.yml @@ -1,4 +1,4 @@ -name: HF integration tests +name: Test performance of skops persistence on: schedule: @@ -19,7 +19,7 @@ jobs: uses: actions/setup-python@v4 - name: Install requirements run: | - pip install .[tests] + pip install -e .[tests] pip --version pip list - name: Run persistence performance checks From b154775b229e25657a530c3d37b89d5acae07841 Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Thu, 23 Mar 2023 16:11:39 +0100 Subject: [PATCH 02/19] Try to trigger workflow on PR --- .github/workflows/persistence-performance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/persistence-performance.yml b/.github/workflows/persistence-performance.yml index 913ba4a7..1e4cfcc4 100644 --- a/.github/workflows/persistence-performance.yml +++ b/.github/workflows/persistence-performance.yml @@ -1,6 +1,7 @@ name: Test performance of skops persistence on: + pull_request: schedule: - cron: '0 9 * * 0' # every sunday at 9:00 UTC workflow_dispatch: From de603180904392c6e2e0ab351f45cb2425145716 Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Thu, 23 Mar 2023 16:33:11 +0100 Subject: [PATCH 03/19] Revert pip install -e, use same cmd as elsewhere --- .github/workflows/persistence-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/persistence-performance.yml b/.github/workflows/persistence-performance.yml index 1e4cfcc4..1119e744 100644 --- a/.github/workflows/persistence-performance.yml +++ b/.github/workflows/persistence-performance.yml @@ -20,7 +20,7 @@ jobs: uses: actions/setup-python@v4 - name: Install requirements run: | - pip install -e .[tests] + pip install .[docs,tests] pip --version pip list - name: Run persistence performance checks From ad43ec4e305a2e250f4ce33ec2ac53626d925189 Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Thu, 23 Mar 2023 16:52:02 +0100 Subject: [PATCH 04/19] More bugfixing: define packages in setup --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 3cc6aa6a..fba0a56d 100644 --- a/setup.py +++ b/setup.py @@ -80,6 +80,7 @@ def setup_package(): "rich": min_deps.tag_to_packages["rich"], }, include_package_data=True, + packages=["skops"], ) setup(**package_data, **metadata) From 826e2b52c37cae067bee320d89a7230f2840a8d0 Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Thu, 23 Mar 2023 17:21:29 +0100 Subject: [PATCH 05/19] More debugging --- .github/workflows/persistence-performance.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/persistence-performance.yml b/.github/workflows/persistence-performance.yml index 1119e744..902b3f0b 100644 --- a/.github/workflows/persistence-performance.yml +++ b/.github/workflows/persistence-performance.yml @@ -23,5 +23,9 @@ jobs: pip install .[docs,tests] pip --version pip list + python -c "import skops;path=skops.__path__;print(path);import os;print(os.listdir(path[0]));print(os.listdir(path[0]+'/io'))" + cd scripts + python -c "import skops;path=skops.__path__;print(path);import os;print(os.listdir(path[0]));print(os.listdir(path[0]+'/io'))" + cd .. - name: Run persistence performance checks run: python scripts/check_persistence_performance.py From e1dcfb0850e11aa9d4c765764537a25f2972bc7f Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Fri, 24 Mar 2023 14:27:05 +0100 Subject: [PATCH 06/19] Revert some changes made for testing --- .github/workflows/persistence-performance.yml | 2 +- setup.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/persistence-performance.yml b/.github/workflows/persistence-performance.yml index 902b3f0b..3218d5cd 100644 --- a/.github/workflows/persistence-performance.yml +++ b/.github/workflows/persistence-performance.yml @@ -20,7 +20,7 @@ jobs: uses: actions/setup-python@v4 - name: Install requirements run: | - pip install .[docs,tests] + pip install .[tests] pip --version pip list python -c "import skops;path=skops.__path__;print(path);import os;print(os.listdir(path[0]));print(os.listdir(path[0]+'/io'))" diff --git a/setup.py b/setup.py index fba0a56d..3cc6aa6a 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,6 @@ def setup_package(): "rich": min_deps.tag_to_packages["rich"], }, include_package_data=True, - packages=["skops"], ) setup(**package_data, **metadata) From 92f94e593207ac76d5f07876781f138f8dd50ea2 Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Fri, 24 Mar 2023 14:27:41 +0100 Subject: [PATCH 07/19] Check older versions of github actions --- .github/workflows/build-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 1dfa0e30..85f524f9 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -39,7 +39,7 @@ jobs: # message and make it available in later steps. This is because we want to # check the content of the commit message, but on PRs, it's replaced by an # artificial commit message. See https://github.com/skops-dev/skops/pull/147 - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 with: fetch-depth: 0 ref: ${{github.event.after}} @@ -49,7 +49,7 @@ jobs: shell: bash - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python }} From 63b623ac80bd600eaae060b35dfc1b334e0e8d5d Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Fri, 24 Mar 2023 14:33:11 +0100 Subject: [PATCH 08/19] More testing of GH action versions --- .github/workflows/build-test.yml | 4 ++-- .github/workflows/persistence-performance.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 85f524f9..1dfa0e30 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -39,7 +39,7 @@ jobs: # message and make it available in later steps. This is because we want to # check the content of the commit message, but on PRs, it's replaced by an # artificial commit message. See https://github.com/skops-dev/skops/pull/147 - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 ref: ${{github.event.after}} @@ -49,7 +49,7 @@ jobs: shell: bash - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} diff --git a/.github/workflows/persistence-performance.yml b/.github/workflows/persistence-performance.yml index 3218d5cd..5f5e2626 100644 --- a/.github/workflows/persistence-performance.yml +++ b/.github/workflows/persistence-performance.yml @@ -15,9 +15,9 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3.5 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v4.5 - name: Install requirements run: | pip install .[tests] From 54e1cacfb807e5a75cd97417cd6feb4118cd3aab Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Fri, 24 Mar 2023 14:37:14 +0100 Subject: [PATCH 09/19] More testing of GH action versions --- .github/workflows/persistence-performance.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/persistence-performance.yml b/.github/workflows/persistence-performance.yml index 5f5e2626..588e3d97 100644 --- a/.github/workflows/persistence-performance.yml +++ b/.github/workflows/persistence-performance.yml @@ -15,9 +15,9 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v3.5 + - uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v4.5 + uses: actions/setup-python@v3 - name: Install requirements run: | pip install .[tests] From 10154ba8c82a53acccbf66e6b56103943eb20e25 Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Fri, 24 Mar 2023 14:39:55 +0100 Subject: [PATCH 10/19] More testing of GH action versions --- .github/workflows/persistence-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/persistence-performance.yml b/.github/workflows/persistence-performance.yml index 588e3d97..854417e5 100644 --- a/.github/workflows/persistence-performance.yml +++ b/.github/workflows/persistence-performance.yml @@ -15,7 +15,7 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v3 - name: Install requirements From 99ce8a5b85d001483d4ea5bb6308eec88ac511f3 Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Fri, 24 Mar 2023 14:42:34 +0100 Subject: [PATCH 11/19] More testing of GH action versions --- .github/workflows/persistence-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/persistence-performance.yml b/.github/workflows/persistence-performance.yml index 854417e5..3218d5cd 100644 --- a/.github/workflows/persistence-performance.yml +++ b/.github/workflows/persistence-performance.yml @@ -17,7 +17,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 - name: Install requirements run: | pip install .[tests] From 73d470e66dd3640bc06e5524e9fd8fc70dd07654 Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Fri, 24 Mar 2023 16:38:20 +0100 Subject: [PATCH 12/19] Try fixing by copying script to root --- .github/workflows/persistence-performance.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/persistence-performance.yml b/.github/workflows/persistence-performance.yml index 3218d5cd..6ab0f6a1 100644 --- a/.github/workflows/persistence-performance.yml +++ b/.github/workflows/persistence-performance.yml @@ -23,9 +23,7 @@ jobs: pip install .[tests] pip --version pip list - python -c "import skops;path=skops.__path__;print(path);import os;print(os.listdir(path[0]));print(os.listdir(path[0]+'/io'))" - cd scripts - python -c "import skops;path=skops.__path__;print(path);import os;print(os.listdir(path[0]));print(os.listdir(path[0]+'/io'))" - cd .. - name: Run persistence performance checks - run: python scripts/check_persistence_performance.py + run: | + cp scripts/check_persistence_performance.py . + python check_persistence_performance.py From 3c1e09529f49a201ff314de2163f09993a802603 Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Mon, 27 Mar 2023 11:01:42 +0200 Subject: [PATCH 13/19] Remove PR hook, add comment --- .github/workflows/persistence-performance.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/persistence-performance.yml b/.github/workflows/persistence-performance.yml index 6ab0f6a1..685ef18e 100644 --- a/.github/workflows/persistence-performance.yml +++ b/.github/workflows/persistence-performance.yml @@ -1,7 +1,6 @@ name: Test performance of skops persistence on: - pull_request: schedule: - cron: '0 9 * * 0' # every sunday at 9:00 UTC workflow_dispatch: @@ -24,6 +23,9 @@ jobs: pip --version pip list - name: Run persistence performance checks + # TODO: once https://github.com/actions/setup-python/issues/634 is + # resolved copying the script is no longer necessary, so change to: + # python scripts/check_persistence_performance.py . run: | cp scripts/check_persistence_performance.py . python check_persistence_performance.py From c32c12c923dd3be9f69bca1b3db8fff22b3142ca Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Mon, 27 Mar 2023 11:54:40 +0200 Subject: [PATCH 14/19] Try running with Python 3.11 --- .github/workflows/persistence-performance.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/persistence-performance.yml b/.github/workflows/persistence-performance.yml index 685ef18e..62b326b0 100644 --- a/.github/workflows/persistence-performance.yml +++ b/.github/workflows/persistence-performance.yml @@ -17,15 +17,13 @@ jobs: - uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v4 + with: + python-version: "3.11" - name: Install requirements run: | pip install .[tests] pip --version pip list - name: Run persistence performance checks - # TODO: once https://github.com/actions/setup-python/issues/634 is - # resolved copying the script is no longer necessary, so change to: - # python scripts/check_persistence_performance.py . run: | - cp scripts/check_persistence_performance.py . - python check_persistence_performance.py + python3.11 scripts/check_persistence_performance.py From 81a9443866711fe65bc3f928a34960e9c668e6a6 Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Mon, 27 Mar 2023 11:55:02 +0200 Subject: [PATCH 15/19] Add PR hook (don't merge) --- .github/workflows/persistence-performance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/persistence-performance.yml b/.github/workflows/persistence-performance.yml index 62b326b0..d9c70c01 100644 --- a/.github/workflows/persistence-performance.yml +++ b/.github/workflows/persistence-performance.yml @@ -1,6 +1,7 @@ name: Test performance of skops persistence on: + pull_request: schedule: - cron: '0 9 * * 0' # every sunday at 9:00 UTC workflow_dispatch: From 0e9c4c9c06dc6e0d0e3f0a9b9674f7b9e6d7c793 Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Mon, 27 Mar 2023 12:07:05 +0200 Subject: [PATCH 16/19] Try calling with just python --- .github/workflows/persistence-performance.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/persistence-performance.yml b/.github/workflows/persistence-performance.yml index d9c70c01..3a852592 100644 --- a/.github/workflows/persistence-performance.yml +++ b/.github/workflows/persistence-performance.yml @@ -26,5 +26,4 @@ jobs: pip --version pip list - name: Run persistence performance checks - run: | - python3.11 scripts/check_persistence_performance.py + run: python scripts/check_persistence_performance.py From 6d921598b389cc5cfaff42a97c84f3ca37c7ec3a Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Mon, 27 Mar 2023 12:07:22 +0200 Subject: [PATCH 17/19] Try printing only 3 columns All columns together exceed GH column width. --- scripts/check_persistence_performance.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/check_persistence_performance.py b/scripts/check_persistence_performance.py index 5f6d2ca1..9df7aa2e 100644 --- a/scripts/check_persistence_performance.py +++ b/scripts/check_persistence_performance.py @@ -99,6 +99,7 @@ def format_result(results: dict[str, list[Any]], topk: int) -> None: df = df.sort_values(["abs_diff"], ascending=False).reset_index(drop=True) print(f"{topk} largest differences:") print(df.head(10)) + print(df[["name", "abs_diff", "rel_diff"]].head(10)) df_slow = df.query("too_slow") if df_slow.empty: From 9cae1768a7e63ab8d515d91ed2bf12ea9865e05e Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Mon, 27 Mar 2023 12:13:47 +0200 Subject: [PATCH 18/19] Print only class name of estimator ...in an attempt to get the column width down --- scripts/check_persistence_performance.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/check_persistence_performance.py b/scripts/check_persistence_performance.py index 9df7aa2e..868f32b5 100644 --- a/scripts/check_persistence_performance.py +++ b/scripts/check_persistence_performance.py @@ -53,9 +53,10 @@ def check_persist_performance() -> None: estimator.fit(X) name = _get_check_estimator_ids(estimator) + cls_name, _, _ = name.partition("(") time_pickle, time_skops = run_check(estimator, number=NUM_REPS) - results["name"].append(name) + results["name"].append(cls_name) results["pickle (s)"].append(time_pickle) results["skops (s)"].append(time_skops) @@ -98,8 +99,7 @@ def format_result(results: dict[str, list[Any]], topk: int) -> None: df = df.sort_values(["abs_diff"], ascending=False).reset_index(drop=True) print(f"{topk} largest differences:") - print(df.head(10)) - print(df[["name", "abs_diff", "rel_diff"]].head(10)) + print(df[["name", "pickle (s)", "skops (s)", "abs_diff", "rel_diff"]].head(10)) df_slow = df.query("too_slow") if df_slow.empty: From ba6cc8c01d469c3f4173ef1a84586c14ecc3d15d Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Mon, 27 Mar 2023 12:16:42 +0200 Subject: [PATCH 19/19] Remove PR hook again --- .github/workflows/persistence-performance.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/persistence-performance.yml b/.github/workflows/persistence-performance.yml index 3a852592..0d82f6c9 100644 --- a/.github/workflows/persistence-performance.yml +++ b/.github/workflows/persistence-performance.yml @@ -1,7 +1,6 @@ name: Test performance of skops persistence on: - pull_request: schedule: - cron: '0 9 * * 0' # every sunday at 9:00 UTC workflow_dispatch: