From 9dc3d05a82b90b118f22627cce6f5844ac790633 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:06:50 +0200 Subject: [PATCH 01/44] [Recording Oracle] Add ruff --- .../ci-lint-cvat-recording-oracle.yaml | 33 +++++ .../recording-oracle/.pre-commit-config.yaml | 24 ++-- .../cvat/recording-oracle/alembic.ini | 14 +- .../cvat/recording-oracle/poetry.lock | 135 +++++++----------- .../cvat/recording-oracle/pyproject.toml | 82 +++++++++-- 5 files changed, 180 insertions(+), 108 deletions(-) create mode 100644 .github/workflows/ci-lint-cvat-recording-oracle.yaml diff --git a/.github/workflows/ci-lint-cvat-recording-oracle.yaml b/.github/workflows/ci-lint-cvat-recording-oracle.yaml new file mode 100644 index 0000000000..8c91ac49ec --- /dev/null +++ b/.github/workflows/ci-lint-cvat-recording-oracle.yaml @@ -0,0 +1,33 @@ +name: CVAT Recording Oracle Lint + +on: + push: + paths: + - 'packages/examples/cvat/recording-oracle/**' + - '.github/workflows/ci-lint-cvat-recording-oracle.yaml' + +env: + WORKING_DIR: ./packages/examples/cvat/recording-oracle + +defaults: + run: + working-directory: ./packages/examples/cvat/recording-oracle + +jobs: + cvat-exo-lint: + name: CVAT Recording Oracle Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + cache: 'pip' + cache-dependency-path: ${{ env.WORKING_DIR }}/poetry.lock + - run: python -m pip install poetry + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'poetry' + cache-dependency-path: ${{ env.WORKING_DIR }}/poetry.lock + - run: poetry install --no-root --only lint + - run: poetry run pre-commit run --all-files diff --git a/packages/examples/cvat/recording-oracle/.pre-commit-config.yaml b/packages/examples/cvat/recording-oracle/.pre-commit-config.yaml index 211285f89c..395275745a 100644 --- a/packages/examples/cvat/recording-oracle/.pre-commit-config.yaml +++ b/packages/examples/cvat/recording-oracle/.pre-commit-config.yaml @@ -1,11 +1,17 @@ repos: - - repo: https://github.com/ambv/black - rev: 22.6.0 + - repo: local hooks: - - id: black - language_version: python3.10 - - repo: https://github.com/PyCQA/isort - rev: 5.12.0 - hooks: - - id: isort - language_version: python3.11 \ No newline at end of file + - id: lint + name: lint + entry: ruff check --fix --unsafe-fixes --show-fixes + language: system + require_serial: true + files: "^packages/examples/cvat/recording-oracle/.*" + types: [python] + - id: format + name: format + entry: ruff format + require_serial: true + language: system + files: "^packages/examples/cvat/recording-oracle/.*" + types: [python] diff --git a/packages/examples/cvat/recording-oracle/alembic.ini b/packages/examples/cvat/recording-oracle/alembic.ini index d9f94e0c5a..e5d0ea3454 100644 --- a/packages/examples/cvat/recording-oracle/alembic.ini +++ b/packages/examples/cvat/recording-oracle/alembic.ini @@ -68,11 +68,15 @@ sqlalchemy.url = driver://user:pass@localhost/dbname # on newly generated revision scripts. See the documentation for further # detail and examples -# format using "black" - use the console_scripts runner, against the "black" entrypoint -# hooks = black -# black.type = console_scripts -# black.entrypoint = black -# black.options = -l 79 REVISION_SCRIPT_FILENAME +hooks=ruff, ruff_format, types_update + +ruff.type = exec +ruff.executable = ruff +ruff.options = check --fix --unsafe-fixes REVISION_SCRIPT_FILENAME + +ruff_format.type = exec +ruff_format.executable = ruff +ruff_format.options = format REVISION_SCRIPT_FILENAME # Logging configuration [loggers] diff --git a/packages/examples/cvat/recording-oracle/poetry.lock b/packages/examples/cvat/recording-oracle/poetry.lock index e8fb4542ef..8d5acb95b5 100644 --- a/packages/examples/cvat/recording-oracle/poetry.lock +++ b/packages/examples/cvat/recording-oracle/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiohttp" @@ -397,52 +397,6 @@ files = [ {file = "bitarray-2.9.2.tar.gz", hash = "sha256:a8f286a51a32323715d77755ed959f94bef13972e9a2fe71b609e40e6d27957e"}, ] -[[package]] -name = "black" -version = "23.12.1" -description = "The uncompromising code formatter." -optional = false -python-versions = ">=3.8" -files = [ - {file = "black-23.12.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e0aaf6041986767a5e0ce663c7a2f0e9eaf21e6ff87a5f95cbf3675bfd4c41d2"}, - {file = "black-23.12.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c88b3711d12905b74206227109272673edce0cb29f27e1385f33b0163c414bba"}, - {file = "black-23.12.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a920b569dc6b3472513ba6ddea21f440d4b4c699494d2e972a1753cdc25df7b0"}, - {file = "black-23.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:3fa4be75ef2a6b96ea8d92b1587dd8cb3a35c7e3d51f0738ced0781c3aa3a5a3"}, - {file = "black-23.12.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8d4df77958a622f9b5a4c96edb4b8c0034f8434032ab11077ec6c56ae9f384ba"}, - {file = "black-23.12.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:602cfb1196dc692424c70b6507593a2b29aac0547c1be9a1d1365f0d964c353b"}, - {file = "black-23.12.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c4352800f14be5b4864016882cdba10755bd50805c95f728011bcb47a4afd59"}, - {file = "black-23.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:0808494f2b2df923ffc5723ed3c7b096bd76341f6213989759287611e9837d50"}, - {file = "black-23.12.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:25e57fd232a6d6ff3f4478a6fd0580838e47c93c83eaf1ccc92d4faf27112c4e"}, - {file = "black-23.12.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2d9e13db441c509a3763a7a3d9a49ccc1b4e974a47be4e08ade2a228876500ec"}, - {file = "black-23.12.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d1bd9c210f8b109b1762ec9fd36592fdd528485aadb3f5849b2740ef17e674e"}, - {file = "black-23.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:ae76c22bde5cbb6bfd211ec343ded2163bba7883c7bc77f6b756a1049436fbb9"}, - {file = "black-23.12.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1fa88a0f74e50e4487477bc0bb900c6781dbddfdfa32691e780bf854c3b4a47f"}, - {file = "black-23.12.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a4d6a9668e45ad99d2f8ec70d5c8c04ef4f32f648ef39048d010b0689832ec6d"}, - {file = "black-23.12.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b18fb2ae6c4bb63eebe5be6bd869ba2f14fd0259bda7d18a46b764d8fb86298a"}, - {file = "black-23.12.1-cp38-cp38-win_amd64.whl", hash = "sha256:c04b6d9d20e9c13f43eee8ea87d44156b8505ca8a3c878773f68b4e4812a421e"}, - {file = "black-23.12.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3e1b38b3135fd4c025c28c55ddfc236b05af657828a8a6abe5deec419a0b7055"}, - {file = "black-23.12.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4f0031eaa7b921db76decd73636ef3a12c942ed367d8c3841a0739412b260a54"}, - {file = "black-23.12.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97e56155c6b737854e60a9ab1c598ff2533d57e7506d97af5481141671abf3ea"}, - {file = "black-23.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:dd15245c8b68fe2b6bd0f32c1556509d11bb33aec9b5d0866dd8e2ed3dba09c2"}, - {file = "black-23.12.1-py3-none-any.whl", hash = "sha256:78baad24af0f033958cad29731e27363183e140962595def56423e626f4bee3e"}, - {file = "black-23.12.1.tar.gz", hash = "sha256:4ce3ef14ebe8d9509188014d96af1c456a910d5b5cbf434a09fef7e024b3d0d5"}, -] - -[package.dependencies] -click = ">=8.0.0" -mypy-extensions = ">=0.4.3" -packaging = ">=22.0" -pathspec = ">=0.9.0" -platformdirs = ">=2" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} - -[package.extras] -colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] -jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -uvloop = ["uvloop (>=0.15.2)"] - [[package]] name = "boto3" version = "1.34.30" @@ -1870,20 +1824,6 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] -[[package]] -name = "isort" -version = "5.13.2" -description = "A Python utility / library to sort Python imports." -optional = false -python-versions = ">=3.8.0" -files = [ - {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, - {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, -] - -[package.extras] -colors = ["colorama (>=0.4.6)"] - [[package]] name = "jmespath" version = "1.0.1" @@ -2466,17 +2406,6 @@ files = [ {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, ] -[[package]] -name = "mypy-extensions" -version = "1.0.0" -description = "Type system extensions for programs checked with the mypy type checker." -optional = false -python-versions = ">=3.5" -files = [ - {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, - {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, -] - [[package]] name = "networkx" version = "3.2.1" @@ -2762,17 +2691,6 @@ files = [ [package.dependencies] regex = ">=2022.3.15" -[[package]] -name = "pathspec" -version = "0.12.1" -description = "Utility library for gitignore style pattern matching of file paths." -optional = false -python-versions = ">=3.8" -files = [ - {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, - {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, -] - [[package]] name = "pgpy" version = "0.6.0" @@ -2951,6 +2869,8 @@ files = [ {file = "psycopg2-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3"}, {file = "psycopg2-2.9.9-cp311-cp311-win32.whl", hash = "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372"}, {file = "psycopg2-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981"}, + {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, + {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, {file = "psycopg2-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa"}, {file = "psycopg2-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a"}, {file = "psycopg2-2.9.9-cp38-cp38-win32.whl", hash = "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c"}, @@ -3597,35 +3517,82 @@ files = [ {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b42169467c42b692c19cf539c38d4602069d8c1505e97b86387fcf7afb766e1d"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:07238db9cbdf8fc1e9de2489a4f68474e70dffcb32232db7c08fa61ca0c7c462"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:fff3573c2db359f091e1589c3d7c5fc2f86f5bdb6f24252c2d8e539d4e45f412"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:aa2267c6a303eb483de8d02db2871afb5c5fc15618d894300b88958f729ad74f"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:840f0c7f194986a63d2c2465ca63af8ccbbc90ab1c6001b1978f05119b5e7334"}, + {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:024cfe1fc7c7f4e1aff4a81e718109e13409767e4f871443cbff3dba3578203d"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win32.whl", hash = "sha256:c69212f63169ec1cfc9bb44723bf2917cbbd8f6191a00ef3410f5a7fe300722d"}, {file = "ruamel.yaml.clib-0.2.8-cp310-cp310-win_amd64.whl", hash = "sha256:cabddb8d8ead485e255fe80429f833172b4cadf99274db39abc080e068cbcc31"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bef08cd86169d9eafb3ccb0a39edb11d8e25f3dae2b28f5c52fd997521133069"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:b16420e621d26fdfa949a8b4b47ade8810c56002f5389970db4ddda51dbff248"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:25c515e350e5b739842fc3228d662413ef28f295791af5e5110b543cf0b57d9b"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_24_aarch64.whl", hash = "sha256:1707814f0d9791df063f8c19bb51b0d1278b8e9a2353abbb676c2f685dee6afe"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:46d378daaac94f454b3a0e3d8d78cafd78a026b1d71443f4966c696b48a6d899"}, + {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:09b055c05697b38ecacb7ac50bdab2240bfca1a0c4872b0fd309bb07dc9aa3a9"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win32.whl", hash = "sha256:53a300ed9cea38cf5a2a9b069058137c2ca1ce658a874b79baceb8f892f915a7"}, {file = "ruamel.yaml.clib-0.2.8-cp311-cp311-win_amd64.whl", hash = "sha256:c2a72e9109ea74e511e29032f3b670835f8a59bbdc9ce692c5b4ed91ccf1eedb"}, {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ebc06178e8821efc9692ea7544aa5644217358490145629914d8020042c24aa1"}, {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:edaef1c1200c4b4cb914583150dcaa3bc30e592e907c01117c08b13a07255ec2"}, {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d176b57452ab5b7028ac47e7b3cf644bcfdc8cacfecf7e71759f7f51a59e5c92"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_24_aarch64.whl", hash = "sha256:1dc67314e7e1086c9fdf2680b7b6c2be1c0d8e3a8279f2e993ca2a7545fecf62"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3213ece08ea033eb159ac52ae052a4899b56ecc124bb80020d9bbceeb50258e9"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aab7fd643f71d7946f2ee58cc88c9b7bfc97debd71dcc93e03e2d174628e7e2d"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-win32.whl", hash = "sha256:5c365d91c88390c8d0a8545df0b5857172824b1c604e867161e6b3d59a827eaa"}, + {file = "ruamel.yaml.clib-0.2.8-cp312-cp312-win_amd64.whl", hash = "sha256:1758ce7d8e1a29d23de54a16ae867abd370f01b5a69e1a3ba75223eaa3ca1a1b"}, {file = "ruamel.yaml.clib-0.2.8-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a5aa27bad2bb83670b71683aae140a1f52b0857a2deff56ad3f6c13a017a26ed"}, {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c58ecd827313af6864893e7af0a3bb85fd529f862b6adbefe14643947cfe2942"}, {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-macosx_12_0_arm64.whl", hash = "sha256:f481f16baec5290e45aebdc2a5168ebc6d35189ae6fea7a58787613a25f6e875"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_24_aarch64.whl", hash = "sha256:77159f5d5b5c14f7c34073862a6b7d34944075d9f93e681638f6d753606c6ce6"}, {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7f67a1ee819dc4562d444bbafb135832b0b909f81cc90f7aa00260968c9ca1b3"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4ecbf9c3e19f9562c7fdd462e8d18dd902a47ca046a2e64dba80699f0b6c09b7"}, + {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:87ea5ff66d8064301a154b3933ae406b0863402a799b16e4a1d24d9fbbcbe0d3"}, {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win32.whl", hash = "sha256:75e1ed13e1f9de23c5607fe6bd1aeaae21e523b32d83bb33918245361e9cc51b"}, {file = "ruamel.yaml.clib-0.2.8-cp37-cp37m-win_amd64.whl", hash = "sha256:3f215c5daf6a9d7bbed4a0a4f760f3113b10e82ff4c5c44bec20a68c8014f675"}, {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1b617618914cb00bf5c34d4357c37aa15183fa229b24767259657746c9077615"}, {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:a6a9ffd280b71ad062eae53ac1659ad86a17f59a0fdc7699fd9be40525153337"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_24_aarch64.whl", hash = "sha256:305889baa4043a09e5b76f8e2a51d4ffba44259f6b4c72dec8ca56207d9c6fe1"}, {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:700e4ebb569e59e16a976857c8798aee258dceac7c7d6b50cab63e080058df91"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e2b4c44b60eadec492926a7270abb100ef9f72798e18743939bdbf037aab8c28"}, + {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e79e5db08739731b0ce4850bed599235d601701d5694c36570a99a0c5ca41a9d"}, {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win32.whl", hash = "sha256:955eae71ac26c1ab35924203fda6220f84dce57d6d7884f189743e2abe3a9fbe"}, {file = "ruamel.yaml.clib-0.2.8-cp38-cp38-win_amd64.whl", hash = "sha256:56f4252222c067b4ce51ae12cbac231bce32aee1d33fbfc9d17e5b8d6966c312"}, {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:03d1162b6d1df1caa3a4bd27aa51ce17c9afc2046c31b0ad60a0a96ec22f8001"}, {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bba64af9fa9cebe325a62fa398760f5c7206b215201b0ec825005f1b18b9bccf"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_24_aarch64.whl", hash = "sha256:a1a45e0bb052edf6a1d3a93baef85319733a888363938e1fc9924cb00c8df24c"}, {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:da09ad1c359a728e112d60116f626cc9f29730ff3e0e7db72b9a2dbc2e4beed5"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:184565012b60405d93838167f425713180b949e9d8dd0bbc7b49f074407c5a8b"}, + {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a75879bacf2c987c003368cf14bed0ffe99e8e85acfa6c0bfffc21a090f16880"}, {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win32.whl", hash = "sha256:84b554931e932c46f94ab306913ad7e11bba988104c5cff26d90d03f68258cd5"}, {file = "ruamel.yaml.clib-0.2.8-cp39-cp39-win_amd64.whl", hash = "sha256:25ac8c08322002b06fa1d49d1646181f0b2c72f5cbc15a85e80b4c30a544bb15"}, {file = "ruamel.yaml.clib-0.2.8.tar.gz", hash = "sha256:beb2e0404003de9a4cab9753a8805a8fe9320ee6673136ed7f04255fe60bb512"}, ] +[[package]] +name = "ruff" +version = "0.6.1" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.6.1-py3-none-linux_armv6l.whl", hash = "sha256:b4bb7de6a24169dc023f992718a9417380301b0c2da0fe85919f47264fb8add9"}, + {file = "ruff-0.6.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:45efaae53b360c81043e311cdec8a7696420b3d3e8935202c2846e7a97d4edae"}, + {file = "ruff-0.6.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:bc60c7d71b732c8fa73cf995efc0c836a2fd8b9810e115be8babb24ae87e0850"}, + {file = "ruff-0.6.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c7477c3b9da822e2db0b4e0b59e61b8a23e87886e727b327e7dcaf06213c5cf"}, + {file = "ruff-0.6.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a0af7ab3f86e3dc9f157a928e08e26c4b40707d0612b01cd577cc84b8905cc9"}, + {file = "ruff-0.6.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:392688dbb50fecf1bf7126731c90c11a9df1c3a4cdc3f481b53e851da5634fa5"}, + {file = "ruff-0.6.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5278d3e095ccc8c30430bcc9bc550f778790acc211865520f3041910a28d0024"}, + {file = "ruff-0.6.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fe6d5f65d6f276ee7a0fc50a0cecaccb362d30ef98a110f99cac1c7872df2f18"}, + {file = "ruff-0.6.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2e0dd11e2ae553ee5c92a81731d88a9883af8db7408db47fc81887c1f8b672e"}, + {file = "ruff-0.6.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d812615525a34ecfc07fd93f906ef5b93656be01dfae9a819e31caa6cfe758a1"}, + {file = "ruff-0.6.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:faaa4060f4064c3b7aaaa27328080c932fa142786f8142aff095b42b6a2eb631"}, + {file = "ruff-0.6.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:99d7ae0df47c62729d58765c593ea54c2546d5de213f2af2a19442d50a10cec9"}, + {file = "ruff-0.6.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9eb18dfd7b613eec000e3738b3f0e4398bf0153cb80bfa3e351b3c1c2f6d7b15"}, + {file = "ruff-0.6.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:c62bc04c6723a81e25e71715aa59489f15034d69bf641df88cb38bdc32fd1dbb"}, + {file = "ruff-0.6.1-py3-none-win32.whl", hash = "sha256:9fb4c4e8b83f19c9477a8745e56d2eeef07a7ff50b68a6998f7d9e2e3887bdc4"}, + {file = "ruff-0.6.1-py3-none-win_amd64.whl", hash = "sha256:c2ebfc8f51ef4aca05dad4552bbcf6fe8d1f75b2f6af546cc47cc1c1ca916b5b"}, + {file = "ruff-0.6.1-py3-none-win_arm64.whl", hash = "sha256:3bc81074971b0ffad1bd0c52284b22411f02a11a012082a76ac6da153536e014"}, + {file = "ruff-0.6.1.tar.gz", hash = "sha256:af3ffd8c6563acb8848d33cd19a69b9bfe943667f0419ca083f8ebe4224a3436"}, +] + [[package]] name = "s3transfer" version = "0.10.0" @@ -4208,4 +4175,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.10, <3.13" -content-hash = "cedfa7b5254c80e53fb8a6a51fed164273319b165c45f085501eb4559db5787e" +content-hash = "eba0dfb48ad8e340536c8dc20a91fb5f1511e15d6c45438197d99ee79a72e85a" diff --git a/packages/examples/cvat/recording-oracle/pyproject.toml b/packages/examples/cvat/recording-oracle/pyproject.toml index b3a5b5180d..82fbbf1b4f 100644 --- a/packages/examples/cvat/recording-oracle/pyproject.toml +++ b/packages/examples/cvat/recording-oracle/pyproject.toml @@ -24,21 +24,83 @@ google-cloud-storage = "^2.14.0" datumaro = {git = "https://github.com/cvat-ai/datumaro.git", rev = "ff83c00c2c1bc4b8fdfcc55067fcab0a9b5b6b11"} [tool.poetry.group.dev.dependencies] -black = "^23.3.0" -pre-commit = "^3.3.3" hypothesis = "^6.82.6" -isort = "^5.12.0" -[tool.isort] -profile = "black" -forced_separate = ["tests"] -line_length = 100 -skip_gitignore = true # align tool behavior with Black +[tool.poetry.group.lint.dependencies] +pre-commit = "^3.3.3" +ruff = "^0.6.0" -[tool.black] +[tool.ruff] line-length = 100 -target-version = ['py310'] +target-version = "py310" + + +[tool.ruff.lint] +select = ["ALL"] +unfixable = [ + "RUF005", # messes up concantenation with numpy structures +] +ignore = [ + "ANN101", # method args annotations (mypy will take care of that) + "ANN001", # | + "ANN202", # | + "ANN201", # | + "ANN401", # | + "ANN102", # | + "RUF001", # Allow cyrillic letters in comments + "COM812", # Trailing comas are handled by ruff format + "ISC001", # Recommended to be disabled when using ruff format + "B904", # Raise from: modern pythons preserve previous exceptions + "EM", # Forbids using literal strings in exceptions. I think it's ok + "TRY003", # Forbids long messages in exceptions, again, it's ok + "G004", # Logging statement uses f-string — it's ok, f-strings are fast + "A003", # Class attribute `id` is shadowing a Python builtin — it's ok in class body + "FIX001", # Forbids using TODOs, but TODOs are useful + "FIX002", # | + "TD001", # | + "TD002", # | + "TD003", # | + "E711", # Allow == None comparisons for sqlalchemy queries + "E712", # Allow == True comparisons for sqlalchemy queries + "PERF203", # Noisy microoptimisation + # Want to resolve eventually, but not now: + "S101", # Allow asserts (there are too many of them right now to fix) + "TRY401", # Checks for excesive logging of exception objects + "E501", # Too long lines + "PTH118", # Prefer pathlib instead of os.path + "G001", # Forbid str.format for logging + "PTH123", # Checks for uses of `os.path.splitext` + "D", # Docstrings +] + + +[tool.ruff.lint.per-file-ignores] +"tests/*" = [ + "PLR0913", # Annotations and args + "ANN202", # | + "ANN201", # | + "ANN001", # | + "ANN003", # | + "ARG001", # | + "SLF001", # Allow private attrs access + "PLR2004", # Allow magic values + "S", # security + "DTZ005", # allow datetimes without timezones +] + +[tool.ruff.lint.pep8-naming] +classmethod-decorators = [ + "pydantic.validator", +] + +[tool.ruff.lint.pylint] +max-args = 9 # in the name of stupidity and lazyness, please god forgive me + +[tool.ruff.lint.isort] +forced-separate = ["tests"] +[tool.ruff.lint.flake8-tidy-imports] +ban-relative-imports = "all" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" From 63cfe311c786a2d3a72cb40ff95e3f9095394aae Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:30 +0200 Subject: [PATCH 02/44] [Recording Oracle] Autofix F401: unused-import ## What it does Checks for unused imports. ## Why is this bad? Unused imports add a performance overhead at runtime, and risk creating import cycles. They also increase the cognitive load of reading the code. If an import statement is used to check for the availability or existence of a module, consider using `importlib.util.find_spec` instead. If an import statement is used to re-export a symbol as part of a module's public interface, consider using a "redundant" import alias, which instructs Ruff (and other tools) to respect the re-export, and avoid marking it as unused, as in: ```python from module import member as member ``` Alternatively, you can use `__all__` to declare a symbol as part of the module's interface, as in: ```python # __init__.py import some_module __all__ = ["some_module"] ``` ## Fix safety Fixes to remove unused imports are safe, except in `__init__.py` files. Applying fixes to `__init__.py` files is currently in preview. The fix offered depends on the type of the unused import. Ruff will suggest a safe fix to export first-party imports with either a redundant alias or, if already present in the file, an `__all__` entry. If multiple `__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix to remove third-party and standard library imports -- the fix is unsafe because the module's interface changes. ## Example ```python import numpy as np # unused import def area(radius): return 3.14 * radius**2 ``` Use instead: ```python def area(radius): return 3.14 * radius**2 ``` To check the availability of a module, use `importlib.util.find_spec`: ```python from importlib.util import find_spec if find_spec("numpy") is not None: print("numpy is installed") else: print("numpy is not installed") ``` ## Options - `lint.ignore-init-module-imports` ## References - [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement) - [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec) - [Typing documentation: interface conventions](https://typing.readthedocs.io/en/latest/source/libraries.html#library-interface-public-and-private-symbols) --- .../00271dfae3b1_add_task_iterations.py | 9 +++---- .../versions/a0c5c3a4c13f_add_gt_stats.py | 24 ++++++++++--------- .../alembic/versions/ca93dce1a618_init.py | 3 ++- .../cvat/recording-oracle/src/core/config.py | 3 ++- .../src/endpoints/__init__.py | 3 ++- .../src/endpoints/error_handlers.py | 3 ++- .../src/handlers/error_handlers.py | 3 ++- .../examples/cvat/recording-oracle/src/log.py | 3 ++- .../recording-oracle/src/schemas/__init__.py | 2 +- .../src/services/cloud/client.py | 15 ++++-------- .../src/validation/dataset_comparison.py | 6 ++--- .../src/validators/__init__.py | 2 +- .../src/validators/validation.py | 2 +- .../tests/integration/chain/test_escrow.py | 14 ++++++----- .../tests/integration/chain/test_web3.py | 18 +++++++++----- ...test_process_reputation_oracle_webhooks.py | 16 +++++++------ 16 files changed, 69 insertions(+), 57 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/alembic/versions/00271dfae3b1_add_task_iterations.py b/packages/examples/cvat/recording-oracle/alembic/versions/00271dfae3b1_add_task_iterations.py index d16700325f..3a5e4d9ea0 100644 --- a/packages/examples/cvat/recording-oracle/alembic/versions/00271dfae3b1_add_task_iterations.py +++ b/packages/examples/cvat/recording-oracle/alembic/versions/00271dfae3b1_add_task_iterations.py @@ -5,24 +5,25 @@ Create Date: 2024-05-08 18:48:53.897599 """ + from alembic import op import sqlalchemy as sa # revision identifiers, used by Alembic. -revision = '00271dfae3b1' -down_revision = 'a0c5c3a4c13f' +revision = "00271dfae3b1" +down_revision = "a0c5c3a4c13f" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.add_column('tasks', sa.Column('iteration', sa.Integer(), server_default='0', nullable=False)) + op.add_column("tasks", sa.Column("iteration", sa.Integer(), server_default="0", nullable=False)) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('tasks', 'iteration') + op.drop_column("tasks", "iteration") # ### end Alembic commands ### diff --git a/packages/examples/cvat/recording-oracle/alembic/versions/a0c5c3a4c13f_add_gt_stats.py b/packages/examples/cvat/recording-oracle/alembic/versions/a0c5c3a4c13f_add_gt_stats.py index eb6471554d..a12dc90481 100644 --- a/packages/examples/cvat/recording-oracle/alembic/versions/a0c5c3a4c13f_add_gt_stats.py +++ b/packages/examples/cvat/recording-oracle/alembic/versions/a0c5c3a4c13f_add_gt_stats.py @@ -5,32 +5,34 @@ Create Date: 2024-03-08 11:34:02.458845 """ + from alembic import op import sqlalchemy as sa # revision identifiers, used by Alembic. -revision = 'a0c5c3a4c13f' -down_revision = 'ca93dce1a618' +revision = "a0c5c3a4c13f" +down_revision = "ca93dce1a618" branch_labels = None depends_on = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.create_table('gt_stats', - sa.Column('task_id', sa.String(), nullable=False), - sa.Column('gt_key', sa.String(), nullable=False), - sa.Column('failed_attempts', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['task_id'], ['tasks.id'], ondelete='CASCADE'), - sa.PrimaryKeyConstraint('task_id', 'gt_key') + op.create_table( + "gt_stats", + sa.Column("task_id", sa.String(), nullable=False), + sa.Column("gt_key", sa.String(), nullable=False), + sa.Column("failed_attempts", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(["task_id"], ["tasks.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("task_id", "gt_key"), ) - op.create_index(op.f('ix_gt_stats_gt_key'), 'gt_stats', ['gt_key'], unique=False) + op.create_index(op.f("ix_gt_stats_gt_key"), "gt_stats", ["gt_key"], unique=False) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_index(op.f('ix_gt_stats_gt_key'), table_name='gt_stats') - op.drop_table('gt_stats') + op.drop_index(op.f("ix_gt_stats_gt_key"), table_name="gt_stats") + op.drop_table("gt_stats") # ### end Alembic commands ### diff --git a/packages/examples/cvat/recording-oracle/alembic/versions/ca93dce1a618_init.py b/packages/examples/cvat/recording-oracle/alembic/versions/ca93dce1a618_init.py index b737376fe0..76f102d52c 100644 --- a/packages/examples/cvat/recording-oracle/alembic/versions/ca93dce1a618_init.py +++ b/packages/examples/cvat/recording-oracle/alembic/versions/ca93dce1a618_init.py @@ -1,10 +1,11 @@ """init Revision ID: ca93dce1a618 -Revises: +Revises: Create Date: 2023-09-05 15:02:51.779529 """ + import sqlalchemy as sa from alembic import op diff --git a/packages/examples/cvat/recording-oracle/src/core/config.py b/packages/examples/cvat/recording-oracle/src/core/config.py index c948fe474e..85dbd2dd67 100644 --- a/packages/examples/cvat/recording-oracle/src/core/config.py +++ b/packages/examples/cvat/recording-oracle/src/core/config.py @@ -1,5 +1,6 @@ # pylint: disable=too-few-public-methods,missing-class-docstring -""" Project configuration from env vars """ +"""Project configuration from env vars""" + import inspect import os from typing import ClassVar, Iterable, Optional diff --git a/packages/examples/cvat/recording-oracle/src/endpoints/__init__.py b/packages/examples/cvat/recording-oracle/src/endpoints/__init__.py index 4b1802bc67..f8ea5264d1 100644 --- a/packages/examples/cvat/recording-oracle/src/endpoints/__init__.py +++ b/packages/examples/cvat/recording-oracle/src/endpoints/__init__.py @@ -1,4 +1,5 @@ -""" API endpoints """ +"""API endpoints""" + from fastapi import APIRouter, FastAPI from src.core.config import Config diff --git a/packages/examples/cvat/recording-oracle/src/endpoints/error_handlers.py b/packages/examples/cvat/recording-oracle/src/endpoints/error_handlers.py index b8d1c424b1..294867ea07 100644 --- a/packages/examples/cvat/recording-oracle/src/endpoints/error_handlers.py +++ b/packages/examples/cvat/recording-oracle/src/endpoints/error_handlers.py @@ -1,4 +1,5 @@ -""" Custom error handlers for the FastAPI""" +"""Custom error handlers for the FastAPI""" + from fastapi import FastAPI from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse diff --git a/packages/examples/cvat/recording-oracle/src/handlers/error_handlers.py b/packages/examples/cvat/recording-oracle/src/handlers/error_handlers.py index b8d1c424b1..294867ea07 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/error_handlers.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/error_handlers.py @@ -1,4 +1,5 @@ -""" Custom error handlers for the FastAPI""" +"""Custom error handlers for the FastAPI""" + from fastapi import FastAPI from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse diff --git a/packages/examples/cvat/recording-oracle/src/log.py b/packages/examples/cvat/recording-oracle/src/log.py index c533f89d0e..9a2f7e4e7e 100644 --- a/packages/examples/cvat/recording-oracle/src/log.py +++ b/packages/examples/cvat/recording-oracle/src/log.py @@ -1,4 +1,5 @@ -""" Config for the application logger""" +"""Config for the application logger""" + import logging from logging.config import dictConfig diff --git a/packages/examples/cvat/recording-oracle/src/schemas/__init__.py b/packages/examples/cvat/recording-oracle/src/schemas/__init__.py index a0811a145d..84f4ff6429 100644 --- a/packages/examples/cvat/recording-oracle/src/schemas/__init__.py +++ b/packages/examples/cvat/recording-oracle/src/schemas/__init__.py @@ -1,5 +1,5 @@ # pylint: disable=too-few-public-methods -""" Schema for API input&output""" +"""Schema for API input&output""" from typing import List, Optional diff --git a/packages/examples/cvat/recording-oracle/src/services/cloud/client.py b/packages/examples/cvat/recording-oracle/src/services/cloud/client.py index 5bb92d77e3..4f8e7e7007 100644 --- a/packages/examples/cvat/recording-oracle/src/services/cloud/client.py +++ b/packages/examples/cvat/recording-oracle/src/services/cloud/client.py @@ -11,26 +11,21 @@ def __init__( self._bucket = unquote(bucket) if bucket else None @abstractmethod - def create_file(self, key: str, data: bytes = b"", *, bucket: Optional[str] = None): - ... + def create_file(self, key: str, data: bytes = b"", *, bucket: Optional[str] = None): ... @abstractmethod - def remove_file(self, key: str, *, bucket: Optional[str] = None): - ... + def remove_file(self, key: str, *, bucket: Optional[str] = None): ... @abstractmethod - def file_exists(self, key: str, *, bucket: Optional[str] = None) -> bool: - ... + def file_exists(self, key: str, *, bucket: Optional[str] = None) -> bool: ... @abstractmethod - def download_file(self, key: str, *, bucket: Optional[str] = None) -> bytes: - ... + def download_file(self, key: str, *, bucket: Optional[str] = None) -> bytes: ... @abstractmethod def list_files( self, *, bucket: Optional[str] = None, prefix: Optional[str] = None - ) -> List[str]: - ... + ) -> List[str]: ... @staticmethod def normalize_prefix(prefix: Optional[str]) -> Optional[str]: diff --git a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py index cce97e89aa..73e09fab01 100644 --- a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py +++ b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py @@ -25,8 +25,7 @@ class SimilarityFunction(metaclass=ABCMeta): "A function to compute similarity between 2 annotations" - def __call__(self, gt_ann: dm.Annotation, ds_ann: dm.Annotation) -> float: - ... + def __call__(self, gt_ann: dm.Annotation, ds_ann: dm.Annotation) -> float: ... class CachedSimilarityFunction(SimilarityFunction): @@ -119,8 +118,7 @@ def compare(self, gt_dataset: dm.Dataset, ds_dataset: dm.Dataset) -> float: @abstractmethod def compare_sample_annotations( self, gt_sample: dm.DatasetItem, ds_sample: dm.DatasetItem, *, similarity_threshold: float - ) -> Tuple[MatchResult, SimilarityFunction]: - ... + ) -> Tuple[MatchResult, SimilarityFunction]: ... class BboxDatasetComparator(DatasetComparator): diff --git a/packages/examples/cvat/recording-oracle/src/validators/__init__.py b/packages/examples/cvat/recording-oracle/src/validators/__init__.py index c8ce69d5f2..51a596f70e 100644 --- a/packages/examples/cvat/recording-oracle/src/validators/__init__.py +++ b/packages/examples/cvat/recording-oracle/src/validators/__init__.py @@ -1,4 +1,4 @@ -""" Validation utils""" +"""Validation utils""" class ValidationResult: diff --git a/packages/examples/cvat/recording-oracle/src/validators/validation.py b/packages/examples/cvat/recording-oracle/src/validators/validation.py index c8ce69d5f2..51a596f70e 100644 --- a/packages/examples/cvat/recording-oracle/src/validators/validation.py +++ b/packages/examples/cvat/recording-oracle/src/validators/validation.py @@ -1,4 +1,4 @@ -""" Validation utils""" +"""Validation utils""" class ValidationResult: diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py index b0565f5ee9..07389de51f 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py @@ -92,9 +92,10 @@ def test_validate_escrow_invalid_status(self): validate_escrow(self.w3.eth.chain_id, escrow_address) def test_get_escrow_manifest(self): - with patch("src.chain.escrow.get_escrow") as mock_get_escrow, patch( - "src.chain.escrow.StorageUtils.download_file_from_url" - ) as mock_download: + with ( + patch("src.chain.escrow.get_escrow") as mock_get_escrow, + patch("src.chain.escrow.StorageUtils.download_file_from_url") as mock_download, + ): mock_download.return_value = json.dumps({"title": "test"}).encode() mock_get_escrow.return_value = self.escrow() @@ -158,9 +159,10 @@ def test_store_results_invalid_hash(self): def test_get_reputation_oracle_address(self): escrow_address = create_escrow(self.w3) - with patch("src.chain.escrow.get_web3") as mock_get_web3, patch( - "src.chain.escrow.get_escrow" - ) as mock_get_escrow: + with ( + patch("src.chain.escrow.get_web3") as mock_get_web3, + patch("src.chain.escrow.get_escrow") as mock_get_escrow, + ): mock_get_web3.return_value = self.w3 mock_escrow = MagicMock() mock_escrow.reputation_oracle = REPUTATION_ORACLE_ADDRESS diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py index 4e37ea8a45..f3eb4130e7 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py @@ -62,18 +62,24 @@ def test_get_web3_invalid_chain_id(self): ) def test_sign_message_polygon(self): - with patch("src.chain.web3.get_web3") as mock_function, patch( - "src.chain.web3.Config.polygon_mainnet.private_key", - DEFAULT_GAS_PAYER_PRIV, + with ( + patch("src.chain.web3.get_web3") as mock_function, + patch( + "src.chain.web3.Config.polygon_mainnet.private_key", + DEFAULT_GAS_PAYER_PRIV, + ), ): mock_function.return_value = self.w3 signed_message, _ = sign_message(ChainId.POLYGON.value, "message") self.assertEqual(signed_message, SIGNATURE) def test_sign_message_amoy(self): - with patch("src.chain.web3.get_web3") as mock_function, patch( - "src.chain.web3.Config.polygon_amoy.private_key", - DEFAULT_GAS_PAYER_PRIV, + with ( + patch("src.chain.web3.get_web3") as mock_function, + patch( + "src.chain.web3.Config.polygon_amoy.private_key", + DEFAULT_GAS_PAYER_PRIV, + ), ): mock_function.return_value = self.w3 signed_message, _ = sign_message(ChainId.POLYGON_AMOY.value, "message") diff --git a/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py b/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py index a31c512f85..4195fe940c 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py @@ -53,13 +53,15 @@ def get_webhook(self, escrow_address, chain_id, event_data): def test_process_reputation_oracle_webhooks(self): expected_url = "expected_url" - with patch( - "src.crons.process_reputation_oracle_webhooks.httpx.Client.post" - ) as mock_httpx, patch( - "src.crons.process_reputation_oracle_webhooks.get_reputation_oracle_url" - ) as mock_get_repo_url, patch( - "src.crons.process_reputation_oracle_webhooks.prepare_signed_message" - ) as mock_signature: + with ( + patch("src.crons.process_reputation_oracle_webhooks.httpx.Client.post") as mock_httpx, + patch( + "src.crons.process_reputation_oracle_webhooks.get_reputation_oracle_url" + ) as mock_get_repo_url, + patch( + "src.crons.process_reputation_oracle_webhooks.prepare_signed_message" + ) as mock_signature, + ): mock_response = MagicMock() mock_response.raise_for_status.return_value = None mock_httpx.return_value = mock_response From 6cb94d103353485e7cbba7642bdf79280043fca3 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:43 +0200 Subject: [PATCH 03/44] [Recording Oracle] Autofix F401: unused-import ## What it does Checks for unused imports. ## Why is this bad? Unused imports add a performance overhead at runtime, and risk creating import cycles. They also increase the cognitive load of reading the code. If an import statement is used to check for the availability or existence of a module, consider using `importlib.util.find_spec` instead. If an import statement is used to re-export a symbol as part of a module's public interface, consider using a "redundant" import alias, which instructs Ruff (and other tools) to respect the re-export, and avoid marking it as unused, as in: ```python from module import member as member ``` Alternatively, you can use `__all__` to declare a symbol as part of the module's interface, as in: ```python # __init__.py import some_module __all__ = ["some_module"] ``` ## Fix safety Fixes to remove unused imports are safe, except in `__init__.py` files. Applying fixes to `__init__.py` files is currently in preview. The fix offered depends on the type of the unused import. Ruff will suggest a safe fix to export first-party imports with either a redundant alias or, if already present in the file, an `__all__` entry. If multiple `__all__` declarations are present, Ruff will not offer a fix. Ruff will suggest an unsafe fix to remove third-party and standard library imports -- the fix is unsafe because the module's interface changes. ## Example ```python import numpy as np # unused import def area(radius): return 3.14 * radius**2 ``` Use instead: ```python def area(radius): return 3.14 * radius**2 ``` To check the availability of a module, use `importlib.util.find_spec`: ```python from importlib.util import find_spec if find_spec("numpy") is not None: print("numpy is installed") else: print("numpy is not installed") ``` ## Options - `lint.ignore-init-module-imports` ## References - [Python documentation: `import`](https://docs.python.org/3/reference/simple_stmts.html#the-import-statement) - [Python documentation: `importlib.util.find_spec`](https://docs.python.org/3/library/importlib.html#importlib.util.find_spec) - [Typing documentation: interface conventions](https://typing.readthedocs.io/en/latest/source/libraries.html#library-interface-public-and-private-symbols) --- packages/examples/cvat/recording-oracle/alembic/env.py | 1 - .../alembic/versions/00271dfae3b1_add_task_iterations.py | 2 +- .../alembic/versions/a0c5c3a4c13f_add_gt_stats.py | 2 +- packages/examples/cvat/recording-oracle/src/chain/escrow.py | 2 +- .../src/crons/process_exchange_oracle_webhooks.py | 1 - 5 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/alembic/env.py b/packages/examples/cvat/recording-oracle/alembic/env.py index e6c51d07f3..e810187f35 100644 --- a/packages/examples/cvat/recording-oracle/alembic/env.py +++ b/packages/examples/cvat/recording-oracle/alembic/env.py @@ -22,7 +22,6 @@ # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata -from src.models.webhook import Webhook target_metadata = Base.metadata diff --git a/packages/examples/cvat/recording-oracle/alembic/versions/00271dfae3b1_add_task_iterations.py b/packages/examples/cvat/recording-oracle/alembic/versions/00271dfae3b1_add_task_iterations.py index 3a5e4d9ea0..971891752b 100644 --- a/packages/examples/cvat/recording-oracle/alembic/versions/00271dfae3b1_add_task_iterations.py +++ b/packages/examples/cvat/recording-oracle/alembic/versions/00271dfae3b1_add_task_iterations.py @@ -6,9 +6,9 @@ """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = "00271dfae3b1" diff --git a/packages/examples/cvat/recording-oracle/alembic/versions/a0c5c3a4c13f_add_gt_stats.py b/packages/examples/cvat/recording-oracle/alembic/versions/a0c5c3a4c13f_add_gt_stats.py index a12dc90481..be55735af5 100644 --- a/packages/examples/cvat/recording-oracle/alembic/versions/a0c5c3a4c13f_add_gt_stats.py +++ b/packages/examples/cvat/recording-oracle/alembic/versions/a0c5c3a4c13f_add_gt_stats.py @@ -6,9 +6,9 @@ """ -from alembic import op import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = "a0c5c3a4c13f" diff --git a/packages/examples/cvat/recording-oracle/src/chain/escrow.py b/packages/examples/cvat/recording-oracle/src/chain/escrow.py index 00381be5f4..94f569647e 100644 --- a/packages/examples/cvat/recording-oracle/src/chain/escrow.py +++ b/packages/examples/cvat/recording-oracle/src/chain/escrow.py @@ -2,7 +2,7 @@ from typing import List from human_protocol_sdk.constants import ChainId, Status -from human_protocol_sdk.encryption import Encryption, EncryptionUtils +from human_protocol_sdk.encryption import Encryption from human_protocol_sdk.escrow import EscrowClient, EscrowData, EscrowUtils from human_protocol_sdk.storage import StorageUtils diff --git a/packages/examples/cvat/recording-oracle/src/crons/process_exchange_oracle_webhooks.py b/packages/examples/cvat/recording-oracle/src/crons/process_exchange_oracle_webhooks.py index 86d1097ece..236ce24719 100644 --- a/packages/examples/cvat/recording-oracle/src/crons/process_exchange_oracle_webhooks.py +++ b/packages/examples/cvat/recording-oracle/src/crons/process_exchange_oracle_webhooks.py @@ -1,5 +1,4 @@ import logging -from typing import Dict import httpx from sqlalchemy.orm import Session From dd32999b74cb134108382f93685a3452a3d7f1d4 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:45 +0200 Subject: [PATCH 04/44] [Recording Oracle] Autofix B905: zip-without-explicit-strict ## What it does Checks for `zip` calls without an explicit `strict` parameter. ## Why is this bad? By default, if the iterables passed to `zip` are of different lengths, the resulting iterator will be silently truncated to the length of the shortest iterable. This can lead to subtle bugs. Use the `strict` parameter to raise a `ValueError` if the iterables are of non-uniform length. If the iterables are intentionally different lengths, the parameter should be explicitly set to `False`. ## Example ```python zip(a, b) ``` Use instead: ```python zip(a, b, strict=True) ``` ## Fix safety This rule's fix is marked as unsafe for `zip` calls that contain `**kwargs`, as adding a `strict` keyword argument to such a call may lead to a duplicate keyword argument error. ## References - [Python documentation: `zip`](https://docs.python.org/3/library/functions.html#zip) --- .../cvat/recording-oracle/src/validation/annotation_matching.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/examples/cvat/recording-oracle/src/validation/annotation_matching.py b/packages/examples/cvat/recording-oracle/src/validation/annotation_matching.py index fa3a965f7f..cd712451cc 100644 --- a/packages/examples/cvat/recording-oracle/src/validation/annotation_matching.py +++ b/packages/examples/cvat/recording-oracle/src/validation/annotation_matching.py @@ -121,7 +121,7 @@ def match_annotations( a_unmatched = [] b_unmatched = [] - for a_idx, b_idx in zip(a_matches, b_matches): + for a_idx, b_idx in zip(a_matches, b_matches, strict=False): dist = distances[a_idx, b_idx] if dist > 1 - min_similarity or dist == 1: if a_idx < len(a_anns): From 03c59a6e41e6cea37511dd3adb5dd57aaa302eff Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:46 +0200 Subject: [PATCH 05/44] [Recording Oracle] Autofix E713: not-in-test ## What it does Checks for membership tests using `not {element} in {collection}`. ## Why is this bad? Testing membership with `{element} not in {collection}` is more readable. ## Example ```python Z = not X in Y if not X.B in Y: pass ``` Use instead: ```python Z = X not in Y if X.B not in Y: pass ``` --- .../examples/cvat/recording-oracle/src/core/oracle_events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/examples/cvat/recording-oracle/src/core/oracle_events.py b/packages/examples/cvat/recording-oracle/src/core/oracle_events.py index 519152e2de..362de794da 100644 --- a/packages/examples/cvat/recording-oracle/src/core/oracle_events.py +++ b/packages/examples/cvat/recording-oracle/src/core/oracle_events.py @@ -78,7 +78,7 @@ def parse_event( sender_events = sender_events_mapping.get(sender) if sender_events is not None: - if not event_type in sender_events: + if event_type not in sender_events: raise ValueError(f"Unknown event '{sender}.{event_type}'") else: assert False, f"Unknown event sender type '{sender}'" From 5c62d1622b903666b58ffafd4f9b76fb86b21a20 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:46 +0200 Subject: [PATCH 06/44] [Recording Oracle] Autofix F541: f-string-missing-placeholders ## What it does Checks for f-strings that do not contain any placeholder expressions. ## Why is this bad? f-strings are a convenient way to format strings, but they are not necessary if there are no placeholder expressions to format. In this case, a regular string should be used instead, as an f-string without placeholders can be confusing for readers, who may expect such a placeholder to be present. An f-string without any placeholders could also indicate that the author forgot to add a placeholder expression. ## Example ```python f"Hello, world!" ``` Use instead: ```python "Hello, world!" ``` **Note:** to maintain compatibility with PyFlakes, this rule only flags f-strings that are part of an implicit concatenation if _none_ of the f-string segments contain placeholder expressions. For example: ```python # Will not be flagged. ( f"Hello," f" {name}!" ) # Will be flagged. ( f"Hello," f" World!" ) ``` See [#10885](https://github.com/astral-sh/ruff/issues/10885) for more. ## References - [PEP 498](https://www.python.org/dev/peps/pep-0498/) --- .../examples/cvat/recording-oracle/src/services/webhook.py | 2 +- .../recording-oracle/tests/integration/chain/test_escrow.py | 4 ++-- .../recording-oracle/tests/integration/chain/test_kvstore.py | 4 ++-- .../recording-oracle/tests/integration/chain/test_web3.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/services/webhook.py b/packages/examples/cvat/recording-oracle/src/services/webhook.py index e14af05321..ad560e5621 100644 --- a/packages/examples/cvat/recording-oracle/src/services/webhook.py +++ b/packages/examples/cvat/recording-oracle/src/services/webhook.py @@ -45,7 +45,7 @@ def create_webhook( assert not event_data or event_type, "'event_data' requires 'event_type'" assert bool(event) ^ bool( event_type - ), f"'event' and 'event_type' cannot be used together. Please use only one of the fields" + ), "'event' and 'event_type' cannot be used together. Please use only one of the fields" if event_type: if self.direction == OracleWebhookDirectionTags.incoming: diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py index 07389de51f..26e4382d19 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py @@ -147,7 +147,7 @@ def test_store_results_invalid_url(self): mock_function.return_value = self.w3 with self.assertRaises(EscrowClientError) as error: store_results(self.w3.eth.chain_id, escrow_address, "invalid_url", DEFAULT_HASH) - self.assertEqual(f"Invalid URL: invalid_url", str(error.exception)) + self.assertEqual("Invalid URL: invalid_url", str(error.exception)) def test_store_results_invalid_hash(self): escrow_address = create_escrow(self.w3) @@ -155,7 +155,7 @@ def test_store_results_invalid_hash(self): mock_function.return_value = self.w3 with self.assertRaises(EscrowClientError) as error: store_results(self.w3.eth.chain_id, escrow_address, DEFAULT_MANIFEST_URL, "") - self.assertEqual(f"Invalid empty hash", str(error.exception)) + self.assertEqual("Invalid empty hash", str(error.exception)) def test_get_reputation_oracle_address(self): escrow_address = create_escrow(self.w3) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py index 245d50594e..7e06ba49fe 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py @@ -86,7 +86,7 @@ def test_get_role_by_address_invalid_escrow(self): mock_function.return_value = self.w3 with self.assertRaises(KVStoreClientError) as error: get_role_by_address(self.w3.eth.chain_id, "invalid_address") - self.assertEqual(f"Invalid address: invalid_address", str(error.exception)) + self.assertEqual("Invalid address: invalid_address", str(error.exception)) def test_get_role_by_address_invalid_address(self): create_escrow(self.w3) @@ -114,7 +114,7 @@ def get_file_url_and_verify_hash(*args, **kwargs): hash_ = store["public_key_hash"] if hash_ != hash(public_key): - raise KVStoreClientError(f"Invalid hash") + raise KVStoreClientError("Invalid hash") return public_key diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py index f3eb4130e7..2569b2ec7b 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py @@ -112,4 +112,4 @@ def test_validate_address(self): def test_validate_address_invalid_address(self): with self.assertRaises(ValueError) as error: validate_address("invalid_address") - self.assertEqual(f"invalid_address is not a correct Web3 address", str(error.exception)) + self.assertEqual("invalid_address is not a correct Web3 address", str(error.exception)) From 790fb45769401631c234a9e5c3f0d5024def6942 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:48 +0200 Subject: [PATCH 07/44] [Recording Oracle] Autofix PLR0402: manual-from-import ## What it does Checks for submodule imports that are aliased to the submodule name. ## Why is this bad? Using the `from` keyword to import the submodule is more concise and readable. ## Example ```python import concurrent.futures as futures ``` Use instead: ```python from concurrent import futures ``` ## References - [Python documentation: Submodules](https://docs.python.org/3/reference/import.html#submodules) --- .../examples/cvat/recording-oracle/src/handlers/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/examples/cvat/recording-oracle/src/handlers/validation.py b/packages/examples/cvat/recording-oracle/src/handlers/validation.py index e398e91a6f..1901a1a4b2 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/validation.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/validation.py @@ -6,10 +6,10 @@ from sqlalchemy.orm import Session -import src.chain.escrow as escrow import src.core.annotation_meta as annotation import src.core.validation_meta as validation import src.services.webhook as oracle_db_service +from src.chain import escrow from src.core.config import Config from src.core.manifest import TaskManifest, parse_manifest from src.core.oracle_events import ( From 2e8300713bf11c0c83ac9ca6a5388709f53a2b54 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:50 +0200 Subject: [PATCH 08/44] [Recording Oracle] Autofix SIM300: yoda-conditions ## What it does Checks for conditions that position a constant on the left-hand side of the comparison operator, rather than the right-hand side. ## Why is this bad? These conditions (sometimes referred to as "Yoda conditions") are less readable than conditions that place the variable on the left-hand side of the comparison operator. In some languages, Yoda conditions are used to prevent accidental assignment in conditions (i.e., accidental uses of the `=` operator, instead of the `==` operator). However, Python does not allow assignments in conditions unless using the `:=` operator, so Yoda conditions provide no benefit in this regard. ## Example ```python if "Foo" == foo: ... ``` Use instead: ```python if foo == "Foo": ... ``` ## References - [Python documentation: Comparisons](https://docs.python.org/3/reference/expressions.html#comparisons) - [Python documentation: Assignment statements](https://docs.python.org/3/reference/simple_stmts.html#assignment-statements) --- .../src/handlers/process_intermediate_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py index 1ff839f4f7..f2e514804a 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py @@ -1006,7 +1006,7 @@ def process_intermediate_results( should_complete = False - if 0 < Config.validation.max_escrow_iterations: + if Config.validation.max_escrow_iterations > 0: escrow_iteration = task.iteration if escrow_iteration and Config.validation.max_escrow_iterations <= escrow_iteration: logger.info( From f5c2f5c7e38624fc302d39fcf471cefb92e4bb32 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:51 +0200 Subject: [PATCH 09/44] [Recording Oracle] Autofix SIM910: dict-get-with-none-default ## What it does Checks for `dict.get()` calls that pass `None` as the default value. ## Why is this bad? `None` is the default value for `dict.get()`, so it is redundant to pass it explicitly. ## Example ```python ages = {"Tom": 23, "Maria": 23, "Dog": 11} age = ages.get("Cat", None) ``` Use instead: ```python ages = {"Tom": 23, "Maria": 23, "Dog": 11} age = ages.get("Cat") ``` ## References - [Python documentation: `dict.get`](https://docs.python.org/3/library/stdtypes.html#dict.get) --- .../examples/cvat/recording-oracle/src/utils/annotations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/examples/cvat/recording-oracle/src/utils/annotations.py b/packages/examples/cvat/recording-oracle/src/utils/annotations.py index 75ec29ec15..fbfcfdae0a 100644 --- a/packages/examples/cvat/recording-oracle/src/utils/annotations.py +++ b/packages/examples/cvat/recording-oracle/src/utils/annotations.py @@ -181,7 +181,7 @@ def _make_label_id_map(self, src_label_cat, dst_label_cat): src_id: dst_label_cat.find(src_label_cat[src_id].name, src_label_cat[src_id].parent)[0] for src_id in range(len(src_label_cat or ())) } - self._map_id = lambda src_id: id_mapping.get(src_id, None) + self._map_id = lambda src_id: id_mapping.get(src_id) def categories(self): return self._categories From fa32619069b3ba1d19f0afe22e95ce14623bde48 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:52 +0200 Subject: [PATCH 10/44] [Recording Oracle] Autofix UP005: deprecated-unittest-alias ## What it does Checks for uses of deprecated methods from the `unittest` module. ## Why is this bad? The `unittest` module has deprecated aliases for some of its methods. The aliases may be removed in future versions of Python. Instead, use their non-deprecated counterparts. ## Example ```python from unittest import TestCase class SomeTest(TestCase): def test_something(self): self.assertEquals(1, 1) ``` Use instead: ```python from unittest import TestCase class SomeTest(TestCase): def test_something(self): self.assertEqual(1, 1) ``` ## References - [Python documentation: Deprecated aliases](https://docs.python.org/3/library/unittest.html#deprecated-aliases) --- .../tests/integration/chain/test_kvstore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py index 7e06ba49fe..fb70e97e52 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py @@ -150,7 +150,7 @@ def set_file_url_and_hash(url: str, key: str): mock_set_file_url_and_hash.side_effect = set_file_url_and_hash register_in_kvstore() mock_set_file_url_and_hash.assert_called_once() - self.assertEquals( + self.assertEqual( kvstore_client.get_file_url_and_verify_hash(LocalhostConfig.addr), PGP_PUBLIC_KEY_URL_1, ) @@ -171,7 +171,7 @@ def set_file_url_and_hash(url: str, key: str): store["public_key_hash"] = "corrupted_hash" register_in_kvstore() mock_set_file_url_and_hash.assert_called_once() - self.assertNotEquals(store["public_key_hash"], "corrupted_hash") + self.assertNotEqual(store["public_key_hash"], "corrupted_hash") # check that a new public key URL will be written to KVStore when an outdated URL is stored there with ( @@ -186,7 +186,7 @@ def set_file_url_and_hash(url: str, key: str): mock_set_file_url_and_hash.side_effect = set_file_url_and_hash register_in_kvstore() mock_set_file_url_and_hash.assert_called_once() - self.assertEquals( + self.assertEqual( kvstore_client.get_file_url_and_verify_hash(LocalhostConfig.addr), PGP_PUBLIC_KEY_URL_2, ) From 9ae59be0a5f8ccfd0e4ba336e2bace1e233b281e Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:52 +0200 Subject: [PATCH 11/44] [Recording Oracle] Autofix UP006: non-pep585-annotation ## What it does Checks for the use of generics that can be replaced with standard library variants based on [PEP 585]. ## Why is this bad? [PEP 585] enabled collections in the Python standard library (like `list`) to be used as generics directly, instead of importing analogous members from the `typing` module (like `typing.List`). When available, the [PEP 585] syntax should be used instead of importing members from the `typing` module, as it's more concise and readable. Importing those members from `typing` is considered deprecated as of [PEP 585]. This rule is enabled when targeting Python 3.9 or later (see: [`target-version`]). By default, it's _also_ enabled for earlier Python versions if `from __future__ import annotations` is present, as `__future__` annotations are not evaluated at runtime. If your code relies on runtime type annotations (either directly or via a library like Pydantic), you can disable this behavior for Python versions prior to 3.9 by setting [`lint.pyupgrade.keep-runtime-typing`] to `true`. ## Example ```python from typing import List foo: List[int] = [1, 2, 3] ``` Use instead: ```python foo: list[int] = [1, 2, 3] ``` ## Fix safety This rule's fix is marked as unsafe, as it may lead to runtime errors when alongside libraries that rely on runtime type annotations, like Pydantic, on Python versions prior to Python 3.9. ## Options - `target-version` - `lint.pyupgrade.keep-runtime-typing` [PEP 585]: https://peps.python.org/pep-0585/ --- .../cvat/recording-oracle/src/chain/escrow.py | 3 +- .../src/core/annotation_meta.py | 3 +- .../recording-oracle/src/core/manifest.py | 8 ++--- .../src/core/oracle_events.py | 6 ++-- .../src/core/tasks/boxes_from_points.py | 6 ++-- .../src/core/tasks/skeletons_from_boxes.py | 8 ++--- .../src/core/validation_meta.py | 6 ++-- .../src/core/validation_results.py | 3 +- .../handlers/process_intermediate_results.py | 30 +++++++++---------- .../src/handlers/validation.py | 4 +-- .../recording-oracle/src/models/validation.py | 8 ++--- .../recording-oracle/src/schemas/__init__.py | 2 +- .../src/services/cloud/client.py | 4 +-- .../src/services/cloud/gcs.py | 6 ++-- .../recording-oracle/src/services/cloud/s3.py | 4 +-- .../src/services/cloud/types.py | 14 ++++----- .../src/services/validation.py | 8 ++--- .../recording-oracle/src/services/webhook.py | 4 +-- .../recording-oracle/src/utils/annotations.py | 4 +-- .../recording-oracle/src/utils/webhooks.py | 6 ++-- .../src/validation/annotation_matching.py | 10 +++---- .../src/validation/dataset_comparison.py | 24 +++++++-------- 22 files changed, 82 insertions(+), 89 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/chain/escrow.py b/packages/examples/cvat/recording-oracle/src/chain/escrow.py index 94f569647e..26630f9045 100644 --- a/packages/examples/cvat/recording-oracle/src/chain/escrow.py +++ b/packages/examples/cvat/recording-oracle/src/chain/escrow.py @@ -1,5 +1,4 @@ import json -from typing import List from human_protocol_sdk.constants import ChainId, Status from human_protocol_sdk.encryption import Encryption @@ -22,7 +21,7 @@ def validate_escrow( chain_id: int, escrow_address: str, *, - accepted_states: List[Status] = [Status.Pending], + accepted_states: list[Status] = [Status.Pending], allow_no_funds: bool = False, ) -> None: assert accepted_states diff --git a/packages/examples/cvat/recording-oracle/src/core/annotation_meta.py b/packages/examples/cvat/recording-oracle/src/core/annotation_meta.py index e19a77efd5..d539787315 100644 --- a/packages/examples/cvat/recording-oracle/src/core/annotation_meta.py +++ b/packages/examples/cvat/recording-oracle/src/core/annotation_meta.py @@ -1,5 +1,4 @@ from pathlib import Path -from typing import List from pydantic import BaseModel @@ -15,4 +14,4 @@ class JobMeta(BaseModel): class AnnotationMeta(BaseModel): - jobs: List[JobMeta] + jobs: list[JobMeta] diff --git a/packages/examples/cvat/recording-oracle/src/core/manifest.py b/packages/examples/cvat/recording-oracle/src/core/manifest.py index c8a4c2260d..f0456ee7c8 100644 --- a/packages/examples/cvat/recording-oracle/src/core/manifest.py +++ b/packages/examples/cvat/recording-oracle/src/core/manifest.py @@ -1,6 +1,6 @@ from decimal import Decimal from enum import Enum -from typing import Annotated, Any, Dict, List, Literal, Optional, Tuple, Union +from typing import Annotated, Any, Literal, Optional, Union from pydantic import AnyUrl, BaseModel, Field, root_validator @@ -28,7 +28,7 @@ class AwsBucketUrl(BucketUrlBase, BaseModel): class GcsBucketUrl(BucketUrlBase, BaseModel): provider: Literal[BucketProviders.gcs] - service_account_key: Dict[str, Any] = {} # (optional) Contents of GCS key file + service_account_key: dict[str, Any] = {} # (optional) Contents of GCS key file BucketUrl = Annotated[Union[AwsBucketUrl, GcsBucketUrl], Field(discriminator="provider")] @@ -67,7 +67,7 @@ class PlainLabelInfo(LabelInfoBase): class SkeletonLabelInfo(LabelInfoBase): type: Literal[LabelTypes.skeleton] - nodes: List[str] = Field(min_items=1) + nodes: list[str] = Field(min_items=1) """ A list of node label names (only points are supposed to be nodes). Example: @@ -76,7 +76,7 @@ class SkeletonLabelInfo(LabelInfoBase): ] """ - joints: Optional[List[Tuple[int, int]]] = Field(default_factory=list) + joints: Optional[list[tuple[int, int]]] = Field(default_factory=list) "A list of node adjacency, e.g. [[0, 1], [1, 2], [1, 3]]" @root_validator diff --git a/packages/examples/cvat/recording-oracle/src/core/oracle_events.py b/packages/examples/cvat/recording-oracle/src/core/oracle_events.py index 362de794da..dccdb7206d 100644 --- a/packages/examples/cvat/recording-oracle/src/core/oracle_events.py +++ b/packages/examples/cvat/recording-oracle/src/core/oracle_events.py @@ -1,4 +1,4 @@ -from typing import Optional, Type, Union +from typing import Optional, Union from pydantic import BaseModel @@ -50,7 +50,7 @@ class ExchangeOracleEvent_TaskFinished(OracleEvent): } -def get_class_for_event_type(event_type: str) -> Type[OracleEvent]: +def get_class_for_event_type(event_type: str) -> type[OracleEvent]: event_class = next((v for k, v in _event_type_map.items() if k == event_type), None) if not event_class: @@ -59,7 +59,7 @@ def get_class_for_event_type(event_type: str) -> Type[OracleEvent]: return event_class -def get_type_tag_for_event_class(event_class: Type[OracleEvent]) -> EventTypeTag: +def get_type_tag_for_event_class(event_class: type[OracleEvent]) -> EventTypeTag: event_type = next((k for k, v in _event_type_map.items() if v == event_class), None) if not event_type: diff --git a/packages/examples/cvat/recording-oracle/src/core/tasks/boxes_from_points.py b/packages/examples/cvat/recording-oracle/src/core/tasks/boxes_from_points.py index c9320473b6..6ce3b57ff8 100644 --- a/packages/examples/cvat/recording-oracle/src/core/tasks/boxes_from_points.py +++ b/packages/examples/cvat/recording-oracle/src/core/tasks/boxes_from_points.py @@ -1,14 +1,14 @@ import os from pathlib import Path from tempfile import TemporaryDirectory -from typing import Dict, Sequence +from typing import Sequence import attrs import datumaro as dm from attrs import frozen from datumaro.util import dump_json, parse_json -BboxPointMapping = Dict[int, int] +BboxPointMapping = dict[int, int] @frozen @@ -28,7 +28,7 @@ def asdict(self) -> dict: RoiInfos = Sequence[RoiInfo] -RoiFilenames = Dict[int, str] +RoiFilenames = dict[int, str] class TaskMetaLayout: diff --git a/packages/examples/cvat/recording-oracle/src/core/tasks/skeletons_from_boxes.py b/packages/examples/cvat/recording-oracle/src/core/tasks/skeletons_from_boxes.py index b6b680c19a..205337691e 100644 --- a/packages/examples/cvat/recording-oracle/src/core/tasks/skeletons_from_boxes.py +++ b/packages/examples/cvat/recording-oracle/src/core/tasks/skeletons_from_boxes.py @@ -1,14 +1,14 @@ import os from pathlib import Path from tempfile import TemporaryDirectory -from typing import Dict, Sequence, Tuple +from typing import Sequence import attrs import datumaro as dm from attrs import frozen from datumaro.util import dump_json, parse_json -SkeletonBboxMapping = Dict[int, int] +SkeletonBboxMapping = dict[int, int] # TODO: migrate to pydantic @@ -34,9 +34,9 @@ def asdict(self) -> dict: RoiInfos = Sequence[RoiInfo] -RoiFilenames = Dict[int, str] +RoiFilenames = dict[int, str] -PointLabelsMapping = Dict[Tuple[str, str], str] +PointLabelsMapping = dict[tuple[str, str], str] "(skeleton, point) -> job point name" diff --git a/packages/examples/cvat/recording-oracle/src/core/validation_meta.py b/packages/examples/cvat/recording-oracle/src/core/validation_meta.py index 027e3ed419..66b9aca8e0 100644 --- a/packages/examples/cvat/recording-oracle/src/core/validation_meta.py +++ b/packages/examples/cvat/recording-oracle/src/core/validation_meta.py @@ -1,5 +1,3 @@ -from typing import List - from pydantic import BaseModel VALIDATION_METAFILE_NAME = "validation_meta.json" @@ -19,5 +17,5 @@ class ResultMeta(BaseModel): class ValidationMeta(BaseModel): - jobs: List[JobMeta] - results: List[ResultMeta] + jobs: list[JobMeta] + results: list[ResultMeta] diff --git a/packages/examples/cvat/recording-oracle/src/core/validation_results.py b/packages/examples/cvat/recording-oracle/src/core/validation_results.py index 8d78bfc7c8..79d8ba001e 100644 --- a/packages/examples/cvat/recording-oracle/src/core/validation_results.py +++ b/packages/examples/cvat/recording-oracle/src/core/validation_results.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import Dict from src.core.validation_errors import DatasetValidationError from src.core.validation_meta import ValidationMeta @@ -14,4 +13,4 @@ class ValidationSuccess: @dataclass class ValidationFailure: - rejected_jobs: Dict[int, DatasetValidationError] + rejected_jobs: dict[int, DatasetValidationError] diff --git a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py index f2e514804a..314d54a347 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, field from pathlib import Path from tempfile import TemporaryDirectory -from typing import Dict, NamedTuple, Optional, Set, Type, TypeVar, Union +from typing import NamedTuple, Optional, TypeVar, Union import datumaro as dm import numpy as np @@ -54,7 +54,7 @@ } -DATASET_COMPARATOR_TYPE_MAP: Dict[TaskTypes, Type[DatasetComparator]] = { +DATASET_COMPARATOR_TYPE_MAP: dict[TaskTypes, type[DatasetComparator]] = { # TaskType.image_label_binary: TagDatasetComparator, # TODO: implement if support is needed TaskTypes.image_boxes: BboxDatasetComparator, TaskTypes.image_points: PointsDatasetComparator, @@ -62,21 +62,21 @@ TaskTypes.image_skeletons_from_boxes: SkeletonDatasetComparator, } -_JobResults = Dict[int, float] +_JobResults = dict[int, float] -_RejectedJobs = Dict[int, DatasetValidationError] +_RejectedJobs = dict[int, DatasetValidationError] -_FailedGtAttempts = Dict[str, int] +_FailedGtAttempts = dict[str, int] "gt key -> attempts" @dataclass class _UpdatedFailedGtInfo: - failed_jobs: Set[int] = field(default_factory=set) + failed_jobs: set[int] = field(default_factory=set) occurrences: int = 0 -_UpdatedFailedGtStats = Dict[str, _UpdatedFailedGtInfo] +_UpdatedFailedGtStats = dict[str, _UpdatedFailedGtInfo] @dataclass @@ -100,7 +100,7 @@ def __init__( chain_id: int, manifest: TaskManifest, *, - job_annotations: Dict[int, io.IOBase], + job_annotations: dict[int, io.IOBase], merged_annotations: io.IOBase, gt_stats: Optional[_FailedGtAttempts] = None, ): @@ -109,7 +109,7 @@ def __init__( self.manifest = manifest self._initial_gt_attempts: _FailedGtAttempts = gt_stats or {} - self._job_annotations: Dict[int, io.IOBase] = job_annotations + self._job_annotations: dict[int, io.IOBase] = job_annotations self._merged_annotations: io.IOBase = merged_annotations self._updated_merged_dataset_archive: Optional[io.IOBase] = None @@ -133,7 +133,7 @@ def _get_gt_weight(self, failed_attempts: int) -> float: return weight - def _get_gt_weights(self) -> Dict[str, float]: + def _get_gt_weights(self) -> dict[str, float]: weights = {} ban_threshold = Config.validation.gt_ban_threshold @@ -331,7 +331,7 @@ class _TaskValidatorWithPerJobGt(_TaskValidator): def _make_gt_dataset_for_job(self, job_id: int, job_dataset: dm.Dataset) -> dm.Dataset: raise NotImplementedError - def _get_gt_weights(self, *, job_cvat_id: int, job_gt_dataset: dm.Dataset) -> Dict[str, float]: + def _get_gt_weights(self, *, job_cvat_id: int, job_gt_dataset: dm.Dataset) -> dict[str, float]: weights = {} ban_threshold = Config.validation.gt_ban_threshold @@ -448,7 +448,7 @@ def __init__(self, *args, **kwargs): self._point_key_to_bbox_key = {v: k for k, v in boxes_to_points_mapping.items()} self._roi_info_by_id = {roi_info.point_id: roi_info for roi_info in roi_infos} - self._roi_name_to_roi_info: Dict[str, boxes_from_points_task.RoiInfo] = { + self._roi_name_to_roi_info: dict[str, boxes_from_points_task.RoiInfo] = { os.path.splitext(roi_filename)[0]: self._roi_info_by_id[roi_id] for roi_id, roi_filename in roi_filenames.items() } @@ -589,7 +589,7 @@ def __init__(self, *args, **kwargs): self._bbox_key_to_skeleton_key = {v: k for k, v in skeletons_to_boxes_mapping.items()} self._roi_info_by_id = {roi_info.bbox_id: roi_info for roi_info in roi_infos} - self._roi_name_to_roi_info: Dict[str, skeletons_from_boxes_task.RoiInfo] = { + self._roi_name_to_roi_info: dict[str, skeletons_from_boxes_task.RoiInfo] = { os.path.splitext(roi_filename)[0]: self._roi_info_by_id[roi_id] for roi_id, roi_filename in roi_filenames.items() } @@ -910,7 +910,7 @@ def process_intermediate_results( escrow_address: str, chain_id: int, meta: AnnotationMeta, - job_annotations: Dict[int, io.RawIOBase], + job_annotations: dict[int, io.RawIOBase], merged_annotations: io.RawIOBase, manifest: TaskManifest, logger: logging.Logger, @@ -979,7 +979,7 @@ def process_intermediate_results( db_service.update_gt_stats(session, task.id, updated_gt_stats) - job_final_result_ids: Dict[int, str] = {} + job_final_result_ids: dict[int, str] = {} for job_meta in meta.jobs: job = db_service.get_job_by_cvat_id(session, job_meta.job_id) if not job: diff --git a/packages/examples/cvat/recording-oracle/src/handlers/validation.py b/packages/examples/cvat/recording-oracle/src/handlers/validation.py index 1901a1a4b2..5788a9822c 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/validation.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/validation.py @@ -2,7 +2,7 @@ import os from collections import Counter from logging import Logger -from typing import Dict, Optional, Union +from typing import Optional, Union from sqlalchemy.orm import Session @@ -49,7 +49,7 @@ def __init__( self.data_bucket = BucketAccessInfo.parse_obj(Config.exchange_oracle_storage_config) self.annotation_meta: Optional[annotation.AnnotationMeta] = None - self.job_annotations: Optional[Dict[int, bytes]] = None + self.job_annotations: Optional[dict[int, bytes]] = None self.merged_annotations: Optional[bytes] = None def set_logger(self, logger: Logger): diff --git a/packages/examples/cvat/recording-oracle/src/models/validation.py b/packages/examples/cvat/recording-oracle/src/models/validation.py index 80a250891f..fad68c5e42 100644 --- a/packages/examples/cvat/recording-oracle/src/models/validation.py +++ b/packages/examples/cvat/recording-oracle/src/models/validation.py @@ -1,8 +1,6 @@ # pylint: disable=too-few-public-methods from __future__ import annotations -from typing import List - from sqlalchemy import Column, DateTime, Enum, Float, ForeignKey, Integer, String from sqlalchemy.orm import Mapped, relationship from sqlalchemy.sql import func @@ -20,10 +18,10 @@ class Task(Base): updated_at = Column(DateTime(timezone=True), onupdate=func.now()) iteration = Column(Integer, server_default="0", nullable=False) - jobs: Mapped[List["Job"]] = relationship( + jobs: Mapped[list["Job"]] = relationship( back_populates="task", cascade="all, delete", passive_deletes=True ) - gt_stats: Mapped[List["GtStats"]] = relationship( + gt_stats: Mapped[list["GtStats"]] = relationship( back_populates="task", cascade="all, delete", passive_deletes=True ) @@ -35,7 +33,7 @@ class Job(Base): task_id = Column(String, ForeignKey("tasks.id", ondelete="CASCADE"), nullable=False) task: Mapped["Task"] = relationship(back_populates="jobs") - validation_results: Mapped[List["ValidationResult"]] = relationship( + validation_results: Mapped[list["ValidationResult"]] = relationship( back_populates="job", cascade="all, delete", passive_deletes=True ) diff --git a/packages/examples/cvat/recording-oracle/src/schemas/__init__.py b/packages/examples/cvat/recording-oracle/src/schemas/__init__.py index 84f4ff6429..be6ae34639 100644 --- a/packages/examples/cvat/recording-oracle/src/schemas/__init__.py +++ b/packages/examples/cvat/recording-oracle/src/schemas/__init__.py @@ -35,4 +35,4 @@ class MetaResponse(BaseModel): message: str version: str - supported_networks: List[SupportedNetwork] + supported_networks: list[SupportedNetwork] diff --git a/packages/examples/cvat/recording-oracle/src/services/cloud/client.py b/packages/examples/cvat/recording-oracle/src/services/cloud/client.py index 4f8e7e7007..7bfaa2d3ef 100644 --- a/packages/examples/cvat/recording-oracle/src/services/cloud/client.py +++ b/packages/examples/cvat/recording-oracle/src/services/cloud/client.py @@ -1,5 +1,5 @@ from abc import ABCMeta, abstractmethod -from typing import List, Optional +from typing import Optional from urllib.parse import unquote @@ -25,7 +25,7 @@ def download_file(self, key: str, *, bucket: Optional[str] = None) -> bytes: ... @abstractmethod def list_files( self, *, bucket: Optional[str] = None, prefix: Optional[str] = None - ) -> List[str]: ... + ) -> list[str]: ... @staticmethod def normalize_prefix(prefix: Optional[str]) -> Optional[str]: diff --git a/packages/examples/cvat/recording-oracle/src/services/cloud/gcs.py b/packages/examples/cvat/recording-oracle/src/services/cloud/gcs.py index 36611b363f..10a5fef517 100644 --- a/packages/examples/cvat/recording-oracle/src/services/cloud/gcs.py +++ b/packages/examples/cvat/recording-oracle/src/services/cloud/gcs.py @@ -1,5 +1,5 @@ from io import BytesIO -from typing import Dict, List, Optional +from typing import Optional from urllib.parse import unquote from google.cloud import storage @@ -14,7 +14,7 @@ def __init__( self, *, bucket: Optional[str] = None, - service_account_key: Optional[Dict] = None, + service_account_key: Optional[dict] = None, ) -> None: super().__init__(bucket) @@ -49,7 +49,7 @@ def download_file(self, key: str, *, bucket: Optional[str] = None) -> bytes: def list_files( self, *, bucket: Optional[str] = None, prefix: Optional[str] = None - ) -> List[str]: + ) -> list[str]: bucket = unquote(bucket) if bucket else self._bucket prefix = self.normalize_prefix(prefix) diff --git a/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py b/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py index e8e608ce99..03b8603753 100644 --- a/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py +++ b/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py @@ -1,5 +1,5 @@ from io import BytesIO -from typing import List, Optional +from typing import Optional from urllib.parse import unquote import boto3 @@ -61,7 +61,7 @@ def download_file(self, key: str, *, bucket: Optional[str] = None) -> bytes: def list_files( self, *, bucket: Optional[str] = None, prefix: Optional[str] = None - ) -> List[str]: + ) -> list[str]: bucket = unquote(bucket) if bucket else self._bucket objects = self.resource.Bucket(bucket).objects if prefix: diff --git a/packages/examples/cvat/recording-oracle/src/services/cloud/types.py b/packages/examples/cvat/recording-oracle/src/services/cloud/types.py index ec8cd9df9e..c41bbcfaf8 100644 --- a/packages/examples/cvat/recording-oracle/src/services/cloud/types.py +++ b/packages/examples/cvat/recording-oracle/src/services/cloud/types.py @@ -4,7 +4,7 @@ from dataclasses import asdict, dataclass, is_dataclass from enum import Enum, auto from inspect import isclass -from typing import Dict, Optional, Type, Union +from typing import Optional, Union from urllib.parse import urlparse from src.core import manifest @@ -31,14 +31,14 @@ def from_str(cls, provider: str) -> CloudProviders: class BucketCredentials: - def to_dict(self) -> Dict: + def to_dict(self) -> dict: if not is_dataclass(self): raise NotImplementedError return asdict(self) @classmethod - def from_storage_config(cls, config: Type[IStorageConfig]) -> Optional[BucketCredentials]: + def from_storage_config(cls, config: type[IStorageConfig]) -> Optional[BucketCredentials]: credentials = None if (config.access_key or config.secret_key) and config.provider.lower() != "aws": @@ -71,7 +71,7 @@ def from_storage_config(cls, config: Type[IStorageConfig]) -> Optional[BucketCre @dataclass class GcsBucketCredentials(BucketCredentials): - service_account_key: Dict + service_account_key: dict @dataclass @@ -129,7 +129,7 @@ def from_url(cls, url: str) -> BucketAccessInfo: raise ValueError(f"{parsed_url.netloc} cloud provider is not supported.") @classmethod - def _from_dict(cls, data: Dict) -> BucketAccessInfo: + def _from_dict(cls, data: dict) -> BucketAccessInfo: for required_field in ( "provider", "bucket_name", @@ -159,7 +159,7 @@ def _from_dict(cls, data: Dict) -> BucketAccessInfo: return BucketAccessInfo(**data) @classmethod - def from_storage_config(cls, config: Type[IStorageConfig]) -> BucketAccessInfo: + def from_storage_config(cls, config: type[IStorageConfig]) -> BucketAccessInfo: credentials = BucketCredentials.from_storage_config(config) return BucketAccessInfo( @@ -175,7 +175,7 @@ def from_bucket_url(cls, bucket_url: manifest.BucketUrl) -> BucketAccessInfo: @classmethod def parse_obj( - cls, data: Union[str, Type[IStorageConfig], manifest.BucketUrl] + cls, data: Union[str, type[IStorageConfig], manifest.BucketUrl] ) -> BucketAccessInfo: if isinstance(data, manifest.BucketUrlBase): return cls.from_bucket_url(data) diff --git a/packages/examples/cvat/recording-oracle/src/services/validation.py b/packages/examples/cvat/recording-oracle/src/services/validation.py index ba139d3fec..97e4d04574 100644 --- a/packages/examples/cvat/recording-oracle/src/services/validation.py +++ b/packages/examples/cvat/recording-oracle/src/services/validation.py @@ -1,5 +1,5 @@ import uuid -from typing import Dict, List, Optional, Union +from typing import Optional, Union from sqlalchemy import update from sqlalchemy.orm import Session @@ -39,7 +39,7 @@ def get_task_by_id( def get_task_validation_results( session: Session, task_id: str, *, for_update: Union[bool, ForUpdateParams] = False -) -> List[ValidationResult]: +) -> list[ValidationResult]: return ( _maybe_for_update(session.query(ValidationResult), enable=for_update) .where(ValidationResult.job.has(Job.task_id == task_id)) @@ -115,7 +115,7 @@ def get_validation_result_by_assignment_id( def get_task_gt_stats( session: Session, task_id: str, *, for_update: Union[bool, ForUpdateParams] = False -) -> List[GtStats]: +) -> list[GtStats]: return ( _maybe_for_update(session.query(GtStats), enable=for_update) .where(GtStats.task_id == task_id) @@ -123,7 +123,7 @@ def get_task_gt_stats( ) -def update_gt_stats(session: Session, task_id: str, values: Dict[str, int]): +def update_gt_stats(session: Session, task_id: str, values: dict[str, int]): # Read more about upsert: # https://docs.sqlalchemy.org/en/20/orm/queryguide/dml.html#orm-upsert-statements diff --git a/packages/examples/cvat/recording-oracle/src/services/webhook.py b/packages/examples/cvat/recording-oracle/src/services/webhook.py index ad560e5621..6eccb2f266 100644 --- a/packages/examples/cvat/recording-oracle/src/services/webhook.py +++ b/packages/examples/cvat/recording-oracle/src/services/webhook.py @@ -1,7 +1,7 @@ import datetime import uuid from enum import Enum -from typing import List, Optional, Union +from typing import Optional, Union from attrs import define from sqlalchemy import case, update @@ -94,7 +94,7 @@ def get_pending_webhooks( *, limit: int = 10, for_update: Union[bool, ForUpdateParams] = False, - ) -> List[Webhook]: + ) -> list[Webhook]: webhooks = ( _maybe_for_update(session.query(Webhook), enable=for_update) .where( diff --git a/packages/examples/cvat/recording-oracle/src/utils/annotations.py b/packages/examples/cvat/recording-oracle/src/utils/annotations.py index fbfcfdae0a..27b7885b75 100644 --- a/packages/examples/cvat/recording-oracle/src/utils/annotations.py +++ b/packages/examples/cvat/recording-oracle/src/utils/annotations.py @@ -1,5 +1,5 @@ from copy import deepcopy -from typing import Iterable, Optional, Tuple, Union +from typing import Iterable, Optional, Union import datumaro as dm import numpy as np @@ -79,7 +79,7 @@ def build_cmdline_parser(cls, **kwargs): def __init__( self, extractor: dm.IExtractor, - dst_labels: Union[Iterable[Union[str, Tuple[str, str]]], dm.LabelCategories], + dst_labels: Union[Iterable[Union[str, tuple[str, str]]], dm.LabelCategories], ): super().__init__(extractor) diff --git a/packages/examples/cvat/recording-oracle/src/utils/webhooks.py b/packages/examples/cvat/recording-oracle/src/utils/webhooks.py index e6d39202a3..7e6662cc15 100644 --- a/packages/examples/cvat/recording-oracle/src/utils/webhooks.py +++ b/packages/examples/cvat/recording-oracle/src/utils/webhooks.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Dict, Optional, Tuple +from typing import Optional from src.chain.web3 import sign_message from src.core.oracle_events import parse_event @@ -12,7 +12,7 @@ def prepare_outgoing_webhook_body( event_type: str, event_data: dict, timestamp: Optional[datetime], -) -> Dict: +) -> dict: body = {"escrow_address": escrow_address, "chain_id": chain_id} if timestamp: @@ -33,7 +33,7 @@ def prepare_signed_message( chain_id: Networks, message: Optional[str] = None, body: Optional[dict] = None, -) -> Tuple[str, str]: +) -> tuple[str, str]: """ Sign the message with the service identity. Optionally, can serialize the input structure. diff --git a/packages/examples/cvat/recording-oracle/src/validation/annotation_matching.py b/packages/examples/cvat/recording-oracle/src/validation/annotation_matching.py index cd712451cc..f63e80e880 100644 --- a/packages/examples/cvat/recording-oracle/src/validation/annotation_matching.py +++ b/packages/examples/cvat/recording-oracle/src/validation/annotation_matching.py @@ -1,5 +1,5 @@ import itertools -from typing import Callable, List, NamedTuple, Sequence, Tuple, TypeVar +from typing import Callable, NamedTuple, Sequence, TypeVar import numpy as np from scipy.optimize import linear_sum_assignment @@ -77,10 +77,10 @@ def point_to_bbox_cmp( class MatchResult(NamedTuple): - matches: List[Tuple[Annotation, Annotation]] - mispred: List[Tuple[Annotation, Annotation]] - a_extra: List[Annotation] - b_extra: List[Annotation] + matches: list[tuple[Annotation, Annotation]] + mispred: list[tuple[Annotation, Annotation]] + a_extra: list[Annotation] + b_extra: list[Annotation] def match_annotations( diff --git a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py index 73e09fab01..718fcb4104 100644 --- a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py +++ b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py @@ -2,7 +2,7 @@ import itertools from abc import ABCMeta, abstractmethod -from typing import Callable, Dict, Optional, Sequence, Set, Tuple, Union +from typing import Callable, Optional, Sequence, Union import datumaro as dm import numpy as np @@ -30,9 +30,9 @@ def __call__(self, gt_ann: dm.Annotation, ds_ann: dm.Annotation) -> float: ... class CachedSimilarityFunction(SimilarityFunction): def __init__( - self, sim_fn: Callable, *, cache: Optional[Dict[Tuple[int, int], float]] = None + self, sim_fn: Callable, *, cache: Optional[dict[tuple[int, int], float]] = None ) -> None: - self.cache: Dict[Tuple[int, int], float] = cache or {} + self.cache: dict[tuple[int, int], float] = cache or {} self.sim_fn = sim_fn def __call__(self, gt_ann: dm.Annotation, ds_ann: dm.Annotation) -> float: @@ -55,9 +55,9 @@ def clear_cache(self): @define class DatasetComparator(metaclass=ABCMeta): _min_similarity_threshold: float - _gt_weights: Dict[str, float] = field(factory=dict) + _gt_weights: dict[str, float] = field(factory=dict) - failed_gts: Set[str] = field(factory=set, init=False) + failed_gts: set[str] = field(factory=set, init=False) "Recorded list of failed GT samples, available after compare() call" def compare(self, gt_dataset: dm.Dataset, ds_dataset: dm.Dataset) -> float: @@ -118,13 +118,13 @@ def compare(self, gt_dataset: dm.Dataset, ds_dataset: dm.Dataset) -> float: @abstractmethod def compare_sample_annotations( self, gt_sample: dm.DatasetItem, ds_sample: dm.DatasetItem, *, similarity_threshold: float - ) -> Tuple[MatchResult, SimilarityFunction]: ... + ) -> tuple[MatchResult, SimilarityFunction]: ... class BboxDatasetComparator(DatasetComparator): def compare_sample_annotations( self, gt_sample: dm.DatasetItem, ds_sample: dm.DatasetItem, *, similarity_threshold: float - ) -> Tuple[MatchResult, SimilarityFunction]: + ) -> tuple[MatchResult, SimilarityFunction]: similarity_fn = CachedSimilarityFunction(bbox_iou) ds_boxes = [ @@ -151,7 +151,7 @@ def compare_sample_annotations( class PointsDatasetComparator(DatasetComparator): def compare_sample_annotations( self, gt_sample: dm.DatasetItem, ds_sample: dm.DatasetItem, *, similarity_threshold: float - ) -> Tuple[MatchResult, SimilarityFunction]: + ) -> tuple[MatchResult, SimilarityFunction]: similarity_fn = CachedSimilarityFunction(point_to_bbox_cmp) ds_points = [ @@ -184,7 +184,7 @@ def compare_sample_annotations( @define class SkeletonDatasetComparator(DatasetComparator): - _skeleton_info: Dict[int, _SkeletonInfo] = field(factory=dict, init=False) + _skeleton_info: dict[int, _SkeletonInfo] = field(factory=dict, init=False) _categories: Optional[dm.CategoriesInfo] = field(default=None, init=False) # TODO: find better strategy for sigma estimation @@ -196,7 +196,7 @@ def compare(self, gt_dataset: dm.Dataset, ds_dataset: dm.Dataset) -> float: def compare_sample_annotations( self, gt_sample: dm.DatasetItem, ds_sample: dm.DatasetItem, *, similarity_threshold: float - ) -> Tuple[MatchResult, SimilarityFunction]: + ) -> tuple[MatchResult, SimilarityFunction]: return self._match_skeletons( gt_sample, ds_sample, similarity_threshold=similarity_threshold ) @@ -218,7 +218,7 @@ def _get_skeleton_info(self, skeleton_label_id: int) -> _SkeletonInfo: def _match_skeletons( self, item_a: dm.DatasetItem, item_b: dm.DatasetItem, *, similarity_threshold: float - ) -> Tuple[MatchResult, SimilarityFunction]: + ) -> tuple[MatchResult, SimilarityFunction]: a_skeletons = [a for a in item_a.annotations if isinstance(a, dm.Skeleton)] b_skeletons = [a for a in item_b.annotations if isinstance(a, dm.Skeleton)] @@ -301,7 +301,7 @@ def _match_skeletons( def _instance_bbox( self, instance_anns: Sequence[dm.Annotation] - ) -> Tuple[float, float, float, float]: + ) -> tuple[float, float, float, float]: return dm.ops.max_bbox( a.get_bbox() if isinstance(a, dm.Skeleton) else a for a in instance_anns From fe01340a43c4505816b502bdf7dc22d2629d302c Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:53 +0200 Subject: [PATCH 12/44] [Recording Oracle] Autofix UP007: non-pep604-annotation ## What it does Check for type annotations that can be rewritten based on [PEP 604] syntax. ## Why is this bad? [PEP 604] introduced a new syntax for union type annotations based on the `|` operator. This syntax is more concise and readable than the previous `typing.Union` and `typing.Optional` syntaxes. This rule is enabled when targeting Python 3.10 or later (see: [`target-version`]). By default, it's _also_ enabled for earlier Python versions if `from __future__ import annotations` is present, as `__future__` annotations are not evaluated at runtime. If your code relies on runtime type annotations (either directly or via a library like Pydantic), you can disable this behavior for Python versions prior to 3.10 by setting [`lint.pyupgrade.keep-runtime-typing`] to `true`. ## Example ```python from typing import Union foo: Union[int, str] = 1 ``` Use instead: ```python foo: int | str = 1 ``` ## Fix safety This rule's fix is marked as unsafe, as it may lead to runtime errors when alongside libraries that rely on runtime type annotations, like Pydantic, on Python versions prior to Python 3.10. It may also lead to runtime errors in unusual and likely incorrect type annotations where the type does not support the `|` operator. ## Options - `target-version` - `lint.pyupgrade.keep-runtime-typing` [PEP 604]: https://peps.python.org/pep-0604/ --- .../cvat/recording-oracle/src/core/config.py | 16 ++++++------ .../recording-oracle/src/core/manifest.py | 18 ++++++------- .../src/core/oracle_events.py | 4 +-- .../cvat/recording-oracle/src/db/utils.py | 4 +-- .../recording-oracle/src/endpoints/webhook.py | 4 +-- .../handlers/process_intermediate_results.py | 24 +++++++++--------- .../src/handlers/validation.py | 8 +++--- .../recording-oracle/src/schemas/__init__.py | 2 +- .../recording-oracle/src/schemas/webhook.py | 5 ++-- .../src/services/cloud/client.py | 17 ++++++------- .../src/services/cloud/gcs.py | 17 ++++++------- .../recording-oracle/src/services/cloud/s3.py | 21 +++++++--------- .../src/services/cloud/types.py | 11 +++----- .../src/services/cloud/utils.py | 4 +-- .../src/services/validation.py | 25 +++++++++---------- .../recording-oracle/src/services/webhook.py | 13 +++++----- .../recording-oracle/src/utils/annotations.py | 8 +++--- .../recording-oracle/src/utils/logging.py | 4 +-- .../recording-oracle/src/utils/requests.py | 6 ++--- .../recording-oracle/src/utils/webhooks.py | 7 +++--- .../src/validation/dataset_comparison.py | 16 ++++++------ 21 files changed, 107 insertions(+), 127 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/core/config.py b/packages/examples/cvat/recording-oracle/src/core/config.py index 85dbd2dd67..948cf330f9 100644 --- a/packages/examples/cvat/recording-oracle/src/core/config.py +++ b/packages/examples/cvat/recording-oracle/src/core/config.py @@ -3,7 +3,7 @@ import inspect import os -from typing import ClassVar, Iterable, Optional +from typing import ClassVar, Iterable from attrs.converters import to_bool from dotenv import load_dotenv @@ -42,9 +42,9 @@ def connection_url(cls): class _NetworkConfig: chain_id: ClassVar[int] - rpc_api: ClassVar[Optional[str]] - private_key: ClassVar[Optional[str]] - addr: ClassVar[Optional[str]] + rpc_api: ClassVar[str | None] + private_key: ClassVar[str | None] + addr: ClassVar[str | None] @classmethod def is_configured(cls) -> bool: @@ -102,12 +102,12 @@ class IStorageConfig: data_bucket_name: ClassVar[str] secure: ClassVar[bool] endpoint_url: ClassVar[str] # TODO: probably should be optional - region: ClassVar[Optional[str]] + region: ClassVar[str | None] # AWS S3 specific attributes - access_key: ClassVar[Optional[str]] - secret_key: ClassVar[Optional[str]] + access_key: ClassVar[str | None] + secret_key: ClassVar[str | None] # GCS specific attributes - key_file_path: ClassVar[Optional[str]] + key_file_path: ClassVar[str | None] @classmethod def get_scheme(cls) -> str: diff --git a/packages/examples/cvat/recording-oracle/src/core/manifest.py b/packages/examples/cvat/recording-oracle/src/core/manifest.py index f0456ee7c8..c70c1ed91a 100644 --- a/packages/examples/cvat/recording-oracle/src/core/manifest.py +++ b/packages/examples/cvat/recording-oracle/src/core/manifest.py @@ -1,6 +1,6 @@ from decimal import Decimal from enum import Enum -from typing import Annotated, Any, Literal, Optional, Union +from typing import Annotated, Any, Literal from pydantic import AnyUrl, BaseModel, Field, root_validator @@ -31,19 +31,19 @@ class GcsBucketUrl(BucketUrlBase, BaseModel): service_account_key: dict[str, Any] = {} # (optional) Contents of GCS key file -BucketUrl = Annotated[Union[AwsBucketUrl, GcsBucketUrl], Field(discriminator="provider")] +BucketUrl = Annotated[AwsBucketUrl | GcsBucketUrl, Field(discriminator="provider")] class DataInfo(BaseModel): - data_url: Union[AnyUrl, BucketUrl] + data_url: AnyUrl | BucketUrl "Bucket URL, AWS S3 | GCS, virtual-hosted-style access" # https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-bucket-intro.html - points_url: Optional[Union[AnyUrl, BucketUrl]] = None + points_url: AnyUrl | BucketUrl | None = None "A path to an archive with a set of points in COCO Keypoints format, " "which provides information about all objects on images" - boxes_url: Optional[Union[AnyUrl, BucketUrl]] = None + boxes_url: AnyUrl | BucketUrl | None = None "A path to an archive with a set of boxes in COCO Instances format, " "which provides information about all objects on images" @@ -76,7 +76,7 @@ class SkeletonLabelInfo(LabelInfoBase): ] """ - joints: Optional[list[tuple[int, int]]] = Field(default_factory=list) + joints: list[tuple[int, int]] | None = Field(default_factory=list) "A list of node adjacency, e.g. [[0, 1], [1, 2], [1, 3]]" @root_validator @@ -114,7 +114,7 @@ def validate_type(cls, values: dict) -> dict: return values -LabelInfo = Annotated[Union[PlainLabelInfo, SkeletonLabelInfo], Field(discriminator="type")] +LabelInfo = Annotated[PlainLabelInfo | SkeletonLabelInfo, Field(discriminator="type")] class AnnotationInfo(BaseModel): @@ -132,7 +132,7 @@ class AnnotationInfo(BaseModel): job_size: int = 10 "Frames per job, validation frames are not included" - max_time: Optional[int] = None # deprecated, TODO: mark deprecated with pydantic 2.7+ + max_time: int | None = None # deprecated, TODO: mark deprecated with pydantic 2.7+ "Maximum time per job (assignment) for an annotator, in seconds" @root_validator(pre=True) @@ -161,7 +161,7 @@ class ValidationInfo(BaseModel): val_size: int = Field(default=2, gt=0) "Validation frames per job" - gt_url: Union[AnyUrl, BucketUrl] + gt_url: AnyUrl | BucketUrl "URL to the archive with Ground Truth annotations, the format is COCO keypoints" diff --git a/packages/examples/cvat/recording-oracle/src/core/oracle_events.py b/packages/examples/cvat/recording-oracle/src/core/oracle_events.py index dccdb7206d..6d12c94a82 100644 --- a/packages/examples/cvat/recording-oracle/src/core/oracle_events.py +++ b/packages/examples/cvat/recording-oracle/src/core/oracle_events.py @@ -1,4 +1,4 @@ -from typing import Optional, Union +from typing import Union from pydantic import BaseModel @@ -69,7 +69,7 @@ def get_type_tag_for_event_class(event_class: type[OracleEvent]) -> EventTypeTag def parse_event( - sender: OracleWebhookTypes, event_type: str, event_data: Optional[dict] = None + sender: OracleWebhookTypes, event_type: str, event_data: dict | None = None ) -> OracleEvent: sender_events_mapping = { OracleWebhookTypes.recording_oracle: RecordingOracleEventTypes, diff --git a/packages/examples/cvat/recording-oracle/src/db/utils.py b/packages/examples/cvat/recording-oracle/src/db/utils.py index 24dfb41561..dedb7d08cb 100644 --- a/packages/examples/cvat/recording-oracle/src/db/utils.py +++ b/packages/examples/cvat/recording-oracle/src/db/utils.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import TypeVar, Union +from typing import TypeVar from sqlalchemy import Select from sqlalchemy.orm import Query @@ -14,7 +14,7 @@ class ForUpdateParams: T = TypeVar("T", Query, Select) -def maybe_for_update(query: T, enable: Union[bool, ForUpdateParams]) -> T: +def maybe_for_update(query: T, enable: bool | ForUpdateParams) -> T: if not enable: return query diff --git a/packages/examples/cvat/recording-oracle/src/endpoints/webhook.py b/packages/examples/cvat/recording-oracle/src/endpoints/webhook.py index f5fee7b483..df2e44dc80 100644 --- a/packages/examples/cvat/recording-oracle/src/endpoints/webhook.py +++ b/packages/examples/cvat/recording-oracle/src/endpoints/webhook.py @@ -1,5 +1,3 @@ -from typing import Union - from fastapi import APIRouter, Header, HTTPException, Request import src.services.webhook as oracle_db_service @@ -15,7 +13,7 @@ async def receive_oracle_webhook( webhook: OracleWebhook, request: Request, - human_signature: Union[str, None] = Header(default=None), + human_signature: str | None = Header(default=None), ) -> OracleWebhookResponse: try: sender = await validate_oracle_webhook_signature(request, human_signature, webhook) diff --git a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py index 314d54a347..4e5cd15f88 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, field from pathlib import Path from tempfile import TemporaryDirectory -from typing import NamedTuple, Optional, TypeVar, Union +from typing import NamedTuple, TypeVar import datumaro as dm import numpy as np @@ -102,7 +102,7 @@ def __init__( *, job_annotations: dict[int, io.IOBase], merged_annotations: io.IOBase, - gt_stats: Optional[_FailedGtAttempts] = None, + gt_stats: _FailedGtAttempts | None = None, ): self.escrow_address = escrow_address self.chain_id = chain_id @@ -112,15 +112,15 @@ def __init__( self._job_annotations: dict[int, io.IOBase] = job_annotations self._merged_annotations: io.IOBase = merged_annotations - self._updated_merged_dataset_archive: Optional[io.IOBase] = None - self._updated_gt_stats: Optional[_UpdatedFailedGtStats] = None - self._job_results: Optional[_JobResults] = None - self._rejected_jobs: Optional[_RejectedJobs] = None + self._updated_merged_dataset_archive: io.IOBase | None = None + self._updated_gt_stats: _UpdatedFailedGtStats | None = None + self._job_results: _JobResults | None = None + self._rejected_jobs: _RejectedJobs | None = None - self._temp_dir: Optional[Path] = None - self._gt_dataset: Optional[dm.Dataset] = None + self._temp_dir: Path | None = None + self._gt_dataset: dm.Dataset | None = None - def _require_field(self, field: Optional[T]) -> T: + def _require_field(self, field: T | None) -> T: assert field is not None return field @@ -351,7 +351,7 @@ def _get_gt_weights(self, *, job_cvat_id: int, job_gt_dataset: dm.Dataset) -> di def _gt_key_to_sample_id( self, gt_key: str, *, job_cvat_id: int, job_gt_dataset: dm.Dataset - ) -> Optional[str]: + ) -> str | None: return gt_key def _update_gt_stats( @@ -847,7 +847,7 @@ def _get_gt_dataset_label_id(self, job_gt_dataset: dm.Dataset) -> _LabelId: def _gt_key_to_sample_id( self, gt_key: str, *, job_cvat_id: int, job_gt_dataset: dm.Dataset - ) -> Optional[str]: + ) -> str | None: parsed_gt_key = self._parse_gt_key(gt_key) job_label_id = self._get_gt_dataset_label_id(job_gt_dataset) if (parsed_gt_key.skeleton_id, parsed_gt_key.point_id) != job_label_id: @@ -914,7 +914,7 @@ def process_intermediate_results( merged_annotations: io.RawIOBase, manifest: TaskManifest, logger: logging.Logger, -) -> Union[ValidationSuccess, ValidationFailure]: +) -> ValidationSuccess | ValidationFailure: # actually validate jobs task_type = manifest.annotation.type diff --git a/packages/examples/cvat/recording-oracle/src/handlers/validation.py b/packages/examples/cvat/recording-oracle/src/handlers/validation.py index 5788a9822c..bb2470b64d 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/validation.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/validation.py @@ -2,7 +2,7 @@ import os from collections import Counter from logging import Logger -from typing import Optional, Union +from typing import Union from sqlalchemy.orm import Session @@ -48,9 +48,9 @@ def __init__( self.data_bucket = BucketAccessInfo.parse_obj(Config.exchange_oracle_storage_config) - self.annotation_meta: Optional[annotation.AnnotationMeta] = None - self.job_annotations: Optional[dict[int, bytes]] = None - self.merged_annotations: Optional[bytes] = None + self.annotation_meta: annotation.AnnotationMeta | None = None + self.job_annotations: dict[int, bytes] | None = None + self.merged_annotations: bytes | None = None def set_logger(self, logger: Logger): self.logger = logger diff --git a/packages/examples/cvat/recording-oracle/src/schemas/__init__.py b/packages/examples/cvat/recording-oracle/src/schemas/__init__.py index be6ae34639..787e0573bf 100644 --- a/packages/examples/cvat/recording-oracle/src/schemas/__init__.py +++ b/packages/examples/cvat/recording-oracle/src/schemas/__init__.py @@ -27,7 +27,7 @@ class ResponseError(BaseModel): class SupportedNetwork(BaseModel): chain_id: int - addr: Optional[str] + addr: str | None class MetaResponse(BaseModel): diff --git a/packages/examples/cvat/recording-oracle/src/schemas/webhook.py b/packages/examples/cvat/recording-oracle/src/schemas/webhook.py index 62ae808f0a..c725d2cfa0 100644 --- a/packages/examples/cvat/recording-oracle/src/schemas/webhook.py +++ b/packages/examples/cvat/recording-oracle/src/schemas/webhook.py @@ -1,5 +1,4 @@ from datetime import datetime -from typing import Optional from pydantic import BaseModel, validator @@ -11,8 +10,8 @@ class OracleWebhook(BaseModel): escrow_address: str chain_id: Networks event_type: str - event_data: Optional[dict] = None - timestamp: Optional[datetime] = None # TODO: remove optional + event_data: dict | None = None + timestamp: datetime | None = None # TODO: remove optional @validator("escrow_address", allow_reuse=True) def validate_escrow_(cls, value): diff --git a/packages/examples/cvat/recording-oracle/src/services/cloud/client.py b/packages/examples/cvat/recording-oracle/src/services/cloud/client.py index 7bfaa2d3ef..53cf13dcc0 100644 --- a/packages/examples/cvat/recording-oracle/src/services/cloud/client.py +++ b/packages/examples/cvat/recording-oracle/src/services/cloud/client.py @@ -1,32 +1,29 @@ from abc import ABCMeta, abstractmethod -from typing import Optional from urllib.parse import unquote class StorageClient(metaclass=ABCMeta): def __init__( self, - bucket: Optional[str] = None, + bucket: str | None = None, ) -> None: self._bucket = unquote(bucket) if bucket else None @abstractmethod - def create_file(self, key: str, data: bytes = b"", *, bucket: Optional[str] = None): ... + def create_file(self, key: str, data: bytes = b"", *, bucket: str | None = None): ... @abstractmethod - def remove_file(self, key: str, *, bucket: Optional[str] = None): ... + def remove_file(self, key: str, *, bucket: str | None = None): ... @abstractmethod - def file_exists(self, key: str, *, bucket: Optional[str] = None) -> bool: ... + def file_exists(self, key: str, *, bucket: str | None = None) -> bool: ... @abstractmethod - def download_file(self, key: str, *, bucket: Optional[str] = None) -> bytes: ... + def download_file(self, key: str, *, bucket: str | None = None) -> bytes: ... @abstractmethod - def list_files( - self, *, bucket: Optional[str] = None, prefix: Optional[str] = None - ) -> list[str]: ... + def list_files(self, *, bucket: str | None = None, prefix: str | None = None) -> list[str]: ... @staticmethod - def normalize_prefix(prefix: Optional[str]) -> Optional[str]: + def normalize_prefix(prefix: str | None) -> str | None: return unquote(prefix).strip("/\\") + "/" if prefix else prefix diff --git a/packages/examples/cvat/recording-oracle/src/services/cloud/gcs.py b/packages/examples/cvat/recording-oracle/src/services/cloud/gcs.py index 10a5fef517..014be4ce1a 100644 --- a/packages/examples/cvat/recording-oracle/src/services/cloud/gcs.py +++ b/packages/examples/cvat/recording-oracle/src/services/cloud/gcs.py @@ -1,5 +1,4 @@ from io import BytesIO -from typing import Optional from urllib.parse import unquote from google.cloud import storage @@ -13,8 +12,8 @@ class GcsClient(StorageClient): def __init__( self, *, - bucket: Optional[str] = None, - service_account_key: Optional[dict] = None, + bucket: str | None = None, + service_account_key: dict | None = None, ) -> None: super().__init__(bucket) @@ -23,22 +22,22 @@ def __init__( else: self.client = storage.Client.create_anonymous_client() - def create_file(self, key: str, data: bytes = b"", *, bucket: Optional[str] = None) -> None: + def create_file(self, key: str, data: bytes = b"", *, bucket: str | None = None) -> None: bucket = unquote(bucket) if bucket else self._bucket bucket_client = self.client.get_bucket(bucket) bucket_client.blob(unquote(key)).upload_from_string(data) - def remove_file(self, key: str, *, bucket: Optional[str] = None) -> None: + def remove_file(self, key: str, *, bucket: str | None = None) -> None: bucket = unquote(bucket) if bucket else self._bucket bucket_client = self.client.get_bucket(bucket) bucket_client.delete_blob(unquote(key)) - def file_exists(self, key: str, *, bucket: Optional[str] = None) -> bool: + def file_exists(self, key: str, *, bucket: str | None = None) -> bool: bucket = unquote(bucket) if bucket else self._bucket bucket_client = self.client.get_bucket(bucket) return bucket_client.blob(unquote(key)).exists() - def download_file(self, key: str, *, bucket: Optional[str] = None) -> bytes: + def download_file(self, key: str, *, bucket: str | None = None) -> bytes: bucket = unquote(bucket) if bucket else self._bucket bucket_client = self.client.get_bucket(bucket) blob = bucket_client.blob(unquote(key)) @@ -47,9 +46,7 @@ def download_file(self, key: str, *, bucket: Optional[str] = None) -> bytes: self.client.download_blob_to_file(blob, data) return data.getvalue() - def list_files( - self, *, bucket: Optional[str] = None, prefix: Optional[str] = None - ) -> list[str]: + def list_files(self, *, bucket: str | None = None, prefix: str | None = None) -> list[str]: bucket = unquote(bucket) if bucket else self._bucket prefix = self.normalize_prefix(prefix) diff --git a/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py b/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py index 03b8603753..74b93f6ef8 100644 --- a/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py +++ b/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py @@ -1,5 +1,4 @@ from io import BytesIO -from typing import Optional from urllib.parse import unquote import boto3 @@ -15,10 +14,10 @@ class S3Client(StorageClient): def __init__( self, *, - bucket: Optional[str] = None, - access_key: Optional[str] = None, - secret_key: Optional[str] = None, - endpoint_url: Optional[str] = None, + bucket: str | None = None, + access_key: str | None = None, + secret_key: str | None = None, + endpoint_url: str | None = None, ) -> None: super().__init__(bucket) session = boto3.Session( @@ -34,15 +33,15 @@ def __init__( if not access_key and not secret_key: self.client.meta.events.register("choose-signer.s3.*", disable_signing) - def create_file(self, key: str, data: bytes = b"", *, bucket: Optional[str] = None): + def create_file(self, key: str, data: bytes = b"", *, bucket: str | None = None): bucket = unquote(bucket) if bucket else self._bucket self.client.put_object(Body=data, Bucket=bucket, Key=unquote(key)) - def remove_file(self, key: str, *, bucket: Optional[str] = None): + def remove_file(self, key: str, *, bucket: str | None = None): bucket = unquote(bucket) if bucket else self._bucket self.client.delete_object(Bucket=bucket, Key=unquote(key)) - def file_exists(self, key: str, *, bucket: Optional[str] = None) -> bool: + def file_exists(self, key: str, *, bucket: str | None = None) -> bool: bucket = unquote(bucket) if bucket else self._bucket try: self.client.head_object(Bucket=bucket, Key=unquote(key)) @@ -53,15 +52,13 @@ def file_exists(self, key: str, *, bucket: Optional[str] = None) -> bool: else: raise - def download_file(self, key: str, *, bucket: Optional[str] = None) -> bytes: + def download_file(self, key: str, *, bucket: str | None = None) -> bytes: bucket = unquote(bucket) if bucket else self._bucket with BytesIO() as data: self.client.download_fileobj(Bucket=bucket, Key=unquote(key), Fileobj=data) return data.getvalue() - def list_files( - self, *, bucket: Optional[str] = None, prefix: Optional[str] = None - ) -> list[str]: + def list_files(self, *, bucket: str | None = None, prefix: str | None = None) -> list[str]: bucket = unquote(bucket) if bucket else self._bucket objects = self.resource.Bucket(bucket).objects if prefix: diff --git a/packages/examples/cvat/recording-oracle/src/services/cloud/types.py b/packages/examples/cvat/recording-oracle/src/services/cloud/types.py index c41bbcfaf8..c41e7c0de8 100644 --- a/packages/examples/cvat/recording-oracle/src/services/cloud/types.py +++ b/packages/examples/cvat/recording-oracle/src/services/cloud/types.py @@ -4,7 +4,6 @@ from dataclasses import asdict, dataclass, is_dataclass from enum import Enum, auto from inspect import isclass -from typing import Optional, Union from urllib.parse import urlparse from src.core import manifest @@ -38,7 +37,7 @@ def to_dict(self) -> dict: return asdict(self) @classmethod - def from_storage_config(cls, config: type[IStorageConfig]) -> Optional[BucketCredentials]: + def from_storage_config(cls, config: type[IStorageConfig]) -> BucketCredentials | None: credentials = None if (config.access_key or config.secret_key) and config.provider.lower() != "aws": @@ -85,8 +84,8 @@ class BucketAccessInfo: provider: CloudProviders host_url: str bucket_name: str - path: Optional[str] = None - credentials: Optional[BucketCredentials] = None + path: str | None = None + credentials: BucketCredentials | None = None @classmethod def from_url(cls, url: str) -> BucketAccessInfo: @@ -174,9 +173,7 @@ def from_bucket_url(cls, bucket_url: manifest.BucketUrl) -> BucketAccessInfo: return cls._from_dict(bucket_url.dict()) @classmethod - def parse_obj( - cls, data: Union[str, type[IStorageConfig], manifest.BucketUrl] - ) -> BucketAccessInfo: + def parse_obj(cls, data: str | type[IStorageConfig] | manifest.BucketUrl) -> BucketAccessInfo: if isinstance(data, manifest.BucketUrlBase): return cls.from_bucket_url(data) elif isinstance(data, str): diff --git a/packages/examples/cvat/recording-oracle/src/services/cloud/utils.py b/packages/examples/cvat/recording-oracle/src/services/cloud/utils.py index a9f821d174..bfc23305c7 100644 --- a/packages/examples/cvat/recording-oracle/src/services/cloud/utils.py +++ b/packages/examples/cvat/recording-oracle/src/services/cloud/utils.py @@ -1,5 +1,3 @@ -from typing import Optional - from src.services.cloud.client import StorageClient from src.services.cloud.gcs import DEFAULT_GCS_HOST, GcsClient from src.services.cloud.s3 import DEFAULT_S3_HOST, S3Client @@ -7,7 +5,7 @@ def compose_bucket_url( - bucket_name: str, provider: CloudProviders, *, bucket_host: Optional[str] = None + bucket_name: str, provider: CloudProviders, *, bucket_host: str | None = None ) -> str: match provider: case CloudProviders.aws: diff --git a/packages/examples/cvat/recording-oracle/src/services/validation.py b/packages/examples/cvat/recording-oracle/src/services/validation.py index 97e4d04574..0ad3dcc185 100644 --- a/packages/examples/cvat/recording-oracle/src/services/validation.py +++ b/packages/examples/cvat/recording-oracle/src/services/validation.py @@ -1,5 +1,4 @@ import uuid -from typing import Optional, Union from sqlalchemy import update from sqlalchemy.orm import Session @@ -20,8 +19,8 @@ def create_task(session: Session, escrow_address: str, chain_id: int) -> str: def get_task_by_escrow_address( - session: Session, escrow_address: str, *, for_update: Union[bool, ForUpdateParams] = False -) -> Optional[Task]: + session: Session, escrow_address: str, *, for_update: bool | ForUpdateParams = False +) -> Task | None: return ( _maybe_for_update(session.query(Task), enable=for_update) .where(Task.escrow_address == escrow_address) @@ -30,15 +29,15 @@ def get_task_by_escrow_address( def get_task_by_id( - session: Session, task_id: str, *, for_update: Union[bool, ForUpdateParams] = False -) -> Optional[Task]: + session: Session, task_id: str, *, for_update: bool | ForUpdateParams = False +) -> Task | None: return ( _maybe_for_update(session.query(Task), enable=for_update).where(Task.id == task_id).first() ) def get_task_validation_results( - session: Session, task_id: str, *, for_update: Union[bool, ForUpdateParams] = False + session: Session, task_id: str, *, for_update: bool | ForUpdateParams = False ) -> list[ValidationResult]: return ( _maybe_for_update(session.query(ValidationResult), enable=for_update) @@ -67,8 +66,8 @@ def create_job(session: Session, job_cvat_id: int, task_id: str) -> str: def get_job_by_cvat_id( - session: Session, job_cvat_id: int, *, for_update: Union[bool, ForUpdateParams] = False -) -> Optional[Job]: + session: Session, job_cvat_id: int, *, for_update: bool | ForUpdateParams = False +) -> Job | None: return ( _maybe_for_update(session.query(Job), enable=for_update) .where(Job.cvat_id == job_cvat_id) @@ -77,8 +76,8 @@ def get_job_by_cvat_id( def get_job_by_id( - session: Session, job_id: str, *, for_update: Union[bool, ForUpdateParams] = False -) -> Optional[Job]: + session: Session, job_id: str, *, for_update: bool | ForUpdateParams = False +) -> Job | None: return _maybe_for_update(session.query(Job), enable=for_update).where(Job.id == job_id).first() @@ -104,8 +103,8 @@ def create_validation_result( def get_validation_result_by_assignment_id( - session: Session, assignment_id: str, *, for_update: Union[bool, ForUpdateParams] = False -) -> Optional[ValidationResult]: + session: Session, assignment_id: str, *, for_update: bool | ForUpdateParams = False +) -> ValidationResult | None: return ( _maybe_for_update(session.query(ValidationResult), enable=for_update) .where(ValidationResult.assignment_id == assignment_id) @@ -114,7 +113,7 @@ def get_validation_result_by_assignment_id( def get_task_gt_stats( - session: Session, task_id: str, *, for_update: Union[bool, ForUpdateParams] = False + session: Session, task_id: str, *, for_update: bool | ForUpdateParams = False ) -> list[GtStats]: return ( _maybe_for_update(session.query(GtStats), enable=for_update) diff --git a/packages/examples/cvat/recording-oracle/src/services/webhook.py b/packages/examples/cvat/recording-oracle/src/services/webhook.py index 6eccb2f266..6627d11285 100644 --- a/packages/examples/cvat/recording-oracle/src/services/webhook.py +++ b/packages/examples/cvat/recording-oracle/src/services/webhook.py @@ -1,7 +1,6 @@ import datetime import uuid from enum import Enum -from typing import Optional, Union from attrs import define from sqlalchemy import case, update @@ -26,7 +25,7 @@ class OracleWebhookDirectionTags(str, Enum, metaclass=BetterEnumMeta): @define class OracleWebhookQueue: direction: OracleWebhookDirectionTags - default_sender: Optional[OracleWebhookTypes] = None + default_sender: OracleWebhookTypes | None = None def create_webhook( self, @@ -34,10 +33,10 @@ def create_webhook( escrow_address: str, chain_id: int, type: OracleWebhookTypes, - signature: Optional[str] = None, - event_type: Optional[str] = None, - event_data: Optional[dict] = None, - event: Optional[OracleEvent] = None, + signature: str | None = None, + event_type: str | None = None, + event_data: dict | None = None, + event: OracleEvent | None = None, ) -> str: """ Creates a webhook in a database @@ -93,7 +92,7 @@ def get_pending_webhooks( type: OracleWebhookTypes, *, limit: int = 10, - for_update: Union[bool, ForUpdateParams] = False, + for_update: bool | ForUpdateParams = False, ) -> list[Webhook]: webhooks = ( _maybe_for_update(session.query(Webhook), enable=for_update) diff --git a/packages/examples/cvat/recording-oracle/src/utils/annotations.py b/packages/examples/cvat/recording-oracle/src/utils/annotations.py index 27b7885b75..0a0993fb83 100644 --- a/packages/examples/cvat/recording-oracle/src/utils/annotations.py +++ b/packages/examples/cvat/recording-oracle/src/utils/annotations.py @@ -1,5 +1,5 @@ from copy import deepcopy -from typing import Iterable, Optional, Union +from typing import Iterable import datumaro as dm import numpy as np @@ -79,7 +79,7 @@ def build_cmdline_parser(cls, **kwargs): def __init__( self, extractor: dm.IExtractor, - dst_labels: Union[Iterable[Union[str, tuple[str, str]]], dm.LabelCategories], + dst_labels: Iterable[str | tuple[str, str]] | dm.LabelCategories, ): super().__init__(extractor) @@ -87,8 +87,8 @@ def __init__( src_categories = self._extractor.categories() - src_label_cat: Optional[dm.LabelCategories] = src_categories.get(dm.AnnotationType.label) - src_point_cat: Optional[dm.PointsCategories] = src_categories.get(dm.AnnotationType.points) + src_label_cat: dm.LabelCategories | None = src_categories.get(dm.AnnotationType.label) + src_point_cat: dm.PointsCategories | None = src_categories.get(dm.AnnotationType.points) if isinstance(dst_labels, dm.LabelCategories): dst_label_cat = deepcopy(dst_labels) diff --git a/packages/examples/cvat/recording-oracle/src/utils/logging.py b/packages/examples/cvat/recording-oracle/src/utils/logging.py index be2c5feba3..e7660eb0d7 100644 --- a/packages/examples/cvat/recording-oracle/src/utils/logging.py +++ b/packages/examples/cvat/recording-oracle/src/utils/logging.py @@ -1,5 +1,5 @@ import logging -from typing import NewType, Optional, Union +from typing import NewType from src.utils.stack import current_function_name @@ -15,7 +15,7 @@ def parse_log_level(level: str) -> LogLevel: def get_function_logger( - parent_logger: Optional[Union[str, logging.Logger]] = None, + parent_logger: str | logging.Logger | None = None, ) -> logging.Logger: if isinstance(parent_logger, str): parent_logger = logging.getLogger(parent_logger) diff --git a/packages/examples/cvat/recording-oracle/src/utils/requests.py b/packages/examples/cvat/recording-oracle/src/utils/requests.py index 785c2cfc87..ef2174f9b9 100644 --- a/packages/examples/cvat/recording-oracle/src/utils/requests.py +++ b/packages/examples/cvat/recording-oracle/src/utils/requests.py @@ -1,4 +1,4 @@ -from typing import Optional, TypeVar +from typing import TypeVar from fastapi import HTTPException @@ -7,11 +7,11 @@ def get_or_404( - obj: Optional[T], + obj: T | None, object_id: V, object_type_name: str, *, - reason: Optional[str] = None, + reason: str | None = None, ) -> T: if obj is None: raise HTTPException( diff --git a/packages/examples/cvat/recording-oracle/src/utils/webhooks.py b/packages/examples/cvat/recording-oracle/src/utils/webhooks.py index 7e6662cc15..97a74f7348 100644 --- a/packages/examples/cvat/recording-oracle/src/utils/webhooks.py +++ b/packages/examples/cvat/recording-oracle/src/utils/webhooks.py @@ -1,5 +1,4 @@ from datetime import datetime -from typing import Optional from src.chain.web3 import sign_message from src.core.oracle_events import parse_event @@ -11,7 +10,7 @@ def prepare_outgoing_webhook_body( chain_id: Networks, event_type: str, event_data: dict, - timestamp: Optional[datetime], + timestamp: datetime | None, ) -> dict: body = {"escrow_address": escrow_address, "chain_id": chain_id} @@ -31,8 +30,8 @@ def prepare_outgoing_webhook_body( def prepare_signed_message( escrow_address: str, chain_id: Networks, - message: Optional[str] = None, - body: Optional[dict] = None, + message: str | None = None, + body: dict | None = None, ) -> tuple[str, str]: """ Sign the message with the service identity. diff --git a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py index 718fcb4104..a610910eee 100644 --- a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py +++ b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py @@ -2,7 +2,7 @@ import itertools from abc import ABCMeta, abstractmethod -from typing import Callable, Optional, Sequence, Union +from typing import Callable, Sequence import datumaro as dm import numpy as np @@ -30,7 +30,7 @@ def __call__(self, gt_ann: dm.Annotation, ds_ann: dm.Annotation) -> float: ... class CachedSimilarityFunction(SimilarityFunction): def __init__( - self, sim_fn: Callable, *, cache: Optional[dict[tuple[int, int], float]] = None + self, sim_fn: Callable, *, cache: dict[tuple[int, int], float] | None = None ) -> None: self.cache: dict[tuple[int, int], float] = cache or {} self.sim_fn = sim_fn @@ -185,7 +185,7 @@ def compare_sample_annotations( @define class SkeletonDatasetComparator(DatasetComparator): _skeleton_info: dict[int, _SkeletonInfo] = field(factory=dict, init=False) - _categories: Optional[dm.CategoriesInfo] = field(default=None, init=False) + _categories: dm.CategoriesInfo | None = field(default=None, init=False) # TODO: find better strategy for sigma estimation _oks_sigma: float = Config.validation.default_oks_sigma @@ -334,11 +334,11 @@ def _compute_oks( a: dm.Points, b: dm.Points, *, - sigma: Union[float, np.ndarray] = 0.1, - bbox: Optional[BboxCoords] = None, - scale: Union[None, float, np.ndarray] = None, - visibility_a: Union[None, bool, Sequence[bool]] = None, - visibility_b: Union[None, bool, Sequence[bool]] = None, + sigma: float | np.ndarray = 0.1, + bbox: BboxCoords | None = None, + scale: None | float | np.ndarray = None, + visibility_a: None | bool | Sequence[bool] = None, + visibility_b: None | bool | Sequence[bool] = None, ) -> float: """ Computes Object Keypoint Similarity metric for a pair of point sets. From 837dd5763c2008efedf5fa5171fd957e468b9bae Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:53 +0200 Subject: [PATCH 13/44] [Recording Oracle] Autofix UP012: unnecessary-encode-utf8 ## What it does Checks for unnecessary calls to `encode` as UTF-8. ## Why is this bad? UTF-8 is the default encoding in Python, so there is no need to call `encode` when UTF-8 is the desired encoding. Instead, use a bytes literal. ## Example ```python "foo".encode("utf-8") ``` Use instead: ```python b"foo" ``` ## References - [Python documentation: `str.encode`](https://docs.python.org/3/library/stdtypes.html#str.encode) --- .../tests/integration/services/cloud/test_client_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/services/cloud/test_client_service.py b/packages/examples/cvat/recording-oracle/tests/integration/services/cloud/test_client_service.py index 75cf94eec5..19fabef7be 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/services/cloud/test_client_service.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/services/cloud/test_client_service.py @@ -35,7 +35,7 @@ def test_file_operations(self): assert len(client.list_files()) == 0 file_name = "test_file" - data = "this is a test".encode("utf-8") + data = b"this is a test" assert not client.file_exists(file_name) client.create_file(file_name, data) From beccdda6250545cbf66f98b5452758cad3bd8b00 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:54 +0200 Subject: [PATCH 14/44] [Recording Oracle] Autofix UP032: f-string ## What it does Checks for `str.format` calls that can be replaced with f-strings. ## Why is this bad? f-strings are more readable and generally preferred over `str.format` calls. ## Example ```python "{}".format(foo) ``` Use instead: ```python f"{foo}" ``` ## References - [Python documentation: f-strings](https://docs.python.org/3/reference/lexical_analysis.html#f-strings) --- .../cvat/recording-oracle/src/db/__init__.py | 2 +- .../handlers/process_intermediate_results.py | 22 +++++-------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/db/__init__.py b/packages/examples/cvat/recording-oracle/src/db/__init__.py index 6e9c85cded..68068770e3 100644 --- a/packages/examples/cvat/recording-oracle/src/db/__init__.py +++ b/packages/examples/cvat/recording-oracle/src/db/__init__.py @@ -8,7 +8,7 @@ engine = sqlalchemy.create_engine( DATABASE_URL, echo="debug" if Config.loglevel <= src.utils.logging.TRACE else False, - connect_args={"options": "-c lock_timeout={:d}".format(Config.postgres_config.lock_timeout)}, + connect_args={"options": f"-c lock_timeout={Config.postgres_config.lock_timeout:d}"}, ) SessionLocal = sessionmaker(autocommit=False, bind=engine) diff --git a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py index 4e5cd15f88..9cb0776a1c 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py @@ -1010,9 +1010,7 @@ def process_intermediate_results( escrow_iteration = task.iteration if escrow_iteration and Config.validation.max_escrow_iterations <= escrow_iteration: logger.info( - "Validation for escrow_address={}: too many iterations, stopping annotation".format( - escrow_address - ) + f"Validation for escrow_address={escrow_address}: too many iterations, stopping annotation" ) should_complete = True @@ -1028,24 +1026,16 @@ def process_intermediate_results( < unverifiable_jobs_count ): logger.info( - "Validation for escrow_address={}: " - "too many assignments have insufficient GT for validation ({} of {} ({:.2f}%)), " - "stopping annotation".format( - escrow_address, - unverifiable_jobs_count, - total_jobs, - unverifiable_jobs_count / total_jobs * 100, - ) + f"Validation for escrow_address={escrow_address}: " + f"too many assignments have insufficient GT for validation ({unverifiable_jobs_count} of {total_jobs} ({unverifiable_jobs_count / total_jobs * 100:.2f}%)), " + "stopping annotation" ) should_complete = True elif len(rejected_jobs) == unverifiable_jobs_count: if unverifiable_jobs_count: logger.info( - "Validation for escrow_address={}: " - "only unverifiable assignments left ({}), stopping annotation".format( - escrow_address, - unverifiable_jobs_count, - ) + f"Validation for escrow_address={escrow_address}: " + f"only unverifiable assignments left ({unverifiable_jobs_count}), stopping annotation" ) should_complete = True From 8bd1dec2c81f42b3b60a6c0e11fd4a39d6c252c9 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:55 +0200 Subject: [PATCH 15/44] [Recording Oracle] Autofix UP035: deprecated-import ## What it does Checks for uses of deprecated imports based on the minimum supported Python version. ## Why is this bad? Deprecated imports may be removed in future versions of Python, and should be replaced with their new equivalents. Note that, in some cases, it may be preferable to continue importing members from `typing_extensions` even after they're added to the Python standard library, as `typing_extensions` can backport bugfixes and optimizations from later Python versions. This rule thus avoids flagging imports from `typing_extensions` in such cases. ## Example ```python from collections import Sequence ``` Use instead: ```python from collections.abc import Sequence ``` --- packages/examples/cvat/recording-oracle/src/core/config.py | 3 ++- .../cvat/recording-oracle/src/core/tasks/boxes_from_points.py | 2 +- .../recording-oracle/src/core/tasks/skeletons_from_boxes.py | 2 +- .../examples/cvat/recording-oracle/src/utils/annotations.py | 2 +- .../recording-oracle/src/validation/annotation_matching.py | 3 ++- .../cvat/recording-oracle/src/validation/dataset_comparison.py | 2 +- packages/examples/cvat/recording-oracle/tests/conftest.py | 2 +- 7 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/core/config.py b/packages/examples/cvat/recording-oracle/src/core/config.py index 948cf330f9..fb7ccd220e 100644 --- a/packages/examples/cvat/recording-oracle/src/core/config.py +++ b/packages/examples/cvat/recording-oracle/src/core/config.py @@ -3,7 +3,8 @@ import inspect import os -from typing import ClassVar, Iterable +from collections.abc import Iterable +from typing import ClassVar from attrs.converters import to_bool from dotenv import load_dotenv diff --git a/packages/examples/cvat/recording-oracle/src/core/tasks/boxes_from_points.py b/packages/examples/cvat/recording-oracle/src/core/tasks/boxes_from_points.py index 6ce3b57ff8..bd26aacc14 100644 --- a/packages/examples/cvat/recording-oracle/src/core/tasks/boxes_from_points.py +++ b/packages/examples/cvat/recording-oracle/src/core/tasks/boxes_from_points.py @@ -1,7 +1,7 @@ import os +from collections.abc import Sequence from pathlib import Path from tempfile import TemporaryDirectory -from typing import Sequence import attrs import datumaro as dm diff --git a/packages/examples/cvat/recording-oracle/src/core/tasks/skeletons_from_boxes.py b/packages/examples/cvat/recording-oracle/src/core/tasks/skeletons_from_boxes.py index 205337691e..fb5f09b418 100644 --- a/packages/examples/cvat/recording-oracle/src/core/tasks/skeletons_from_boxes.py +++ b/packages/examples/cvat/recording-oracle/src/core/tasks/skeletons_from_boxes.py @@ -1,7 +1,7 @@ import os +from collections.abc import Sequence from pathlib import Path from tempfile import TemporaryDirectory -from typing import Sequence import attrs import datumaro as dm diff --git a/packages/examples/cvat/recording-oracle/src/utils/annotations.py b/packages/examples/cvat/recording-oracle/src/utils/annotations.py index 0a0993fb83..eb095b43b5 100644 --- a/packages/examples/cvat/recording-oracle/src/utils/annotations.py +++ b/packages/examples/cvat/recording-oracle/src/utils/annotations.py @@ -1,5 +1,5 @@ +from collections.abc import Iterable from copy import deepcopy -from typing import Iterable import datumaro as dm import numpy as np diff --git a/packages/examples/cvat/recording-oracle/src/validation/annotation_matching.py b/packages/examples/cvat/recording-oracle/src/validation/annotation_matching.py index f63e80e880..b5d7855a85 100644 --- a/packages/examples/cvat/recording-oracle/src/validation/annotation_matching.py +++ b/packages/examples/cvat/recording-oracle/src/validation/annotation_matching.py @@ -1,5 +1,6 @@ import itertools -from typing import Callable, NamedTuple, Sequence, TypeVar +from collections.abc import Callable, Sequence +from typing import NamedTuple, TypeVar import numpy as np from scipy.optimize import linear_sum_assignment diff --git a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py index a610910eee..b28350d994 100644 --- a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py +++ b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py @@ -2,7 +2,7 @@ import itertools from abc import ABCMeta, abstractmethod -from typing import Callable, Sequence +from collections.abc import Callable, Sequence import datumaro as dm import numpy as np diff --git a/packages/examples/cvat/recording-oracle/tests/conftest.py b/packages/examples/cvat/recording-oracle/tests/conftest.py index e027999628..f39f40a8ca 100644 --- a/packages/examples/cvat/recording-oracle/tests/conftest.py +++ b/packages/examples/cvat/recording-oracle/tests/conftest.py @@ -1,4 +1,4 @@ -from typing import Generator +from collections.abc import Generator import pytest from fastapi.testclient import TestClient From b5d4f3ce2c14463f85169c163c373aead8977e50 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:55 +0200 Subject: [PATCH 16/44] [Recording Oracle] Autofix UP037: quoted-annotation ## What it does Checks for the presence of unnecessary quotes in type annotations. ## Why is this bad? In Python, type annotations can be quoted to avoid forward references. However, if `from __future__ import annotations` is present, Python will always evaluate type annotations in a deferred manner, making the quotes unnecessary. Similarly, if the annotation is located in a typing-only context and won't be evaluated by Python at runtime, the quotes will also be considered unnecessary. For example, Python does not evaluate type annotations on assignments in function bodies. ## Example Given: ```python from __future__ import annotations def foo(bar: "Bar") -> "Bar": ... ``` Use instead: ```python from __future__ import annotations def foo(bar: Bar) -> Bar: ... ``` Given: ```python def foo() -> None: bar: "Bar" ``` Use instead: ```python def foo() -> None: bar: Bar ``` ## References - [PEP 563](https://peps.python.org/pep-0563/) - [Python documentation: `__future__`](https://docs.python.org/3/library/__future__.html#module-__future__) --- .../cvat/recording-oracle/src/models/validation.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/models/validation.py b/packages/examples/cvat/recording-oracle/src/models/validation.py index fad68c5e42..51f8f681f5 100644 --- a/packages/examples/cvat/recording-oracle/src/models/validation.py +++ b/packages/examples/cvat/recording-oracle/src/models/validation.py @@ -18,10 +18,10 @@ class Task(Base): updated_at = Column(DateTime(timezone=True), onupdate=func.now()) iteration = Column(Integer, server_default="0", nullable=False) - jobs: Mapped[list["Job"]] = relationship( + jobs: Mapped[list[Job]] = relationship( back_populates="task", cascade="all, delete", passive_deletes=True ) - gt_stats: Mapped[list["GtStats"]] = relationship( + gt_stats: Mapped[list[GtStats]] = relationship( back_populates="task", cascade="all, delete", passive_deletes=True ) @@ -32,8 +32,8 @@ class Job(Base): cvat_id = Column(Integer, unique=True, index=True, nullable=False) task_id = Column(String, ForeignKey("tasks.id", ondelete="CASCADE"), nullable=False) - task: Mapped["Task"] = relationship(back_populates="jobs") - validation_results: Mapped[list["ValidationResult"]] = relationship( + task: Mapped[Task] = relationship(back_populates="jobs") + validation_results: Mapped[list[ValidationResult]] = relationship( back_populates="job", cascade="all, delete", passive_deletes=True ) @@ -46,7 +46,7 @@ class ValidationResult(Base): annotator_wallet_address = Column(String, nullable=False) annotation_quality = Column(Float, nullable=False) - job: Mapped["Job"] = relationship(back_populates="validation_results") + job: Mapped[Job] = relationship(back_populates="validation_results") class GtStats(Base): @@ -62,4 +62,4 @@ class GtStats(Base): failed_attempts = Column(Integer, default=0, nullable=False) - task: Mapped["Task"] = relationship(back_populates="gt_stats") + task: Mapped[Task] = relationship(back_populates="gt_stats") From de7349b0880e4496f774e39733c7304bf6ffeaac Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:59 +0200 Subject: [PATCH 17/44] [Recording Oracle] Unsafe autofix ANN204: missing-return-type-special-method ## What it does Checks that "special" methods, like `__init__`, `__new__`, and `__call__`, have return type annotations. ## Why is this bad? Type annotations are a good way to document the return types of functions. They also help catch bugs, when used alongside a type checker, by ensuring that the types of any returned values, and the types expected by callers, match expectation. Note that type checkers often allow you to omit the return type annotation for `__init__` methods, as long as at least one argument has a type annotation. To opt in to this behavior, use the `mypy-init-return` setting in your `pyproject.toml` or `ruff.toml` file: ```toml [tool.ruff.lint.flake8-annotations] mypy-init-return = true ``` ## Example ```python class Foo: def __init__(self, x: int): self.x = x ``` Use instead: ```python class Foo: def __init__(self, x: int) -> None: self.x = x ``` --- .../src/handlers/process_intermediate_results.py | 8 ++++---- .../examples/cvat/recording-oracle/src/models/webhook.py | 2 +- .../cvat/recording-oracle/src/utils/annotations.py | 2 +- .../examples/cvat/recording-oracle/src/utils/enums.py | 2 +- .../cvat/recording-oracle/src/validators/__init__.py | 2 +- .../cvat/recording-oracle/src/validators/validation.py | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py index 9cb0776a1c..a0c476dede 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py @@ -103,7 +103,7 @@ def __init__( job_annotations: dict[int, io.IOBase], merged_annotations: io.IOBase, gt_stats: _FailedGtAttempts | None = None, - ): + ) -> None: self.escrow_address = escrow_address self.chain_id = chain_id self.manifest = manifest @@ -217,7 +217,7 @@ def _validate_jobs(self): def _restore_original_image_paths(self, merged_dataset: dm.Dataset) -> dm.Dataset: class RemoveCommonPrefix(dm.ItemTransform): - def __init__(self, extractor: dm.IExtractor, *, prefix: str): + def __init__(self, extractor: dm.IExtractor, *, prefix: str) -> None: super().__init__(extractor) self._prefix = prefix @@ -418,7 +418,7 @@ def _validate_jobs(self): class _BoxesFromPointsValidator(_TaskValidatorWithPerJobGt): - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) ( @@ -554,7 +554,7 @@ def _prepare_merged_dataset(self): class _SkeletonsFromBoxesValidator(_TaskValidatorWithPerJobGt): - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) ( diff --git a/packages/examples/cvat/recording-oracle/src/models/webhook.py b/packages/examples/cvat/recording-oracle/src/models/webhook.py index 10c4bec69c..64ca2fb0e7 100644 --- a/packages/examples/cvat/recording-oracle/src/models/webhook.py +++ b/packages/examples/cvat/recording-oracle/src/models/webhook.py @@ -26,5 +26,5 @@ class Webhook(Base): event_data = Column(JSON, nullable=True, server_default=None) direction = Column(String, nullable=False) - def __repr__(self): + def __repr__(self) -> str: return f"Webhook. id={self.id} type={self.type}.{self.event_type}" diff --git a/packages/examples/cvat/recording-oracle/src/utils/annotations.py b/packages/examples/cvat/recording-oracle/src/utils/annotations.py index eb095b43b5..036778a6ff 100644 --- a/packages/examples/cvat/recording-oracle/src/utils/annotations.py +++ b/packages/examples/cvat/recording-oracle/src/utils/annotations.py @@ -80,7 +80,7 @@ def __init__( self, extractor: dm.IExtractor, dst_labels: Iterable[str | tuple[str, str]] | dm.LabelCategories, - ): + ) -> None: super().__init__(extractor) self._categories = {} diff --git a/packages/examples/cvat/recording-oracle/src/utils/enums.py b/packages/examples/cvat/recording-oracle/src/utils/enums.py index 4f3d688251..d4c133b0e5 100644 --- a/packages/examples/cvat/recording-oracle/src/utils/enums.py +++ b/packages/examples/cvat/recording-oracle/src/utils/enums.py @@ -6,5 +6,5 @@ class BetterEnumMeta(EnumMeta): Extends the default enum metaclass with extra methods for better usability """ - def __contains__(cls, item): + def __contains__(cls, item) -> bool: return isinstance(item, cls) or item in [v.value for v in cls.__members__.values()] diff --git a/packages/examples/cvat/recording-oracle/src/validators/__init__.py b/packages/examples/cvat/recording-oracle/src/validators/__init__.py index 51a596f70e..4d9d0279c8 100644 --- a/packages/examples/cvat/recording-oracle/src/validators/__init__.py +++ b/packages/examples/cvat/recording-oracle/src/validators/__init__.py @@ -7,7 +7,7 @@ class ValidationResult: It encapsulates validation logic and helping during generating response body """ - def __init__(self): + def __init__(self) -> None: self.is_valid = True self.errors = [] diff --git a/packages/examples/cvat/recording-oracle/src/validators/validation.py b/packages/examples/cvat/recording-oracle/src/validators/validation.py index 51a596f70e..4d9d0279c8 100644 --- a/packages/examples/cvat/recording-oracle/src/validators/validation.py +++ b/packages/examples/cvat/recording-oracle/src/validators/validation.py @@ -7,7 +7,7 @@ class ValidationResult: It encapsulates validation logic and helping during generating response body """ - def __init__(self): + def __init__(self) -> None: self.is_valid = True self.errors = [] From d8173eaae5e48ad1414aa65f31a117a90cbcf167 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:13:59 +0200 Subject: [PATCH 18/44] [Recording Oracle] Unsafe autofix ANN206: missing-return-type-class-method ## What it does Checks that class methods have return type annotations. ## Why is this bad? Type annotations are a good way to document the return types of functions. They also help catch bugs, when used alongside a type checker, by ensuring that the types of any returned values, and the types expected by callers, match expectation. ## Example ```python class Foo: @classmethod def bar(cls): return 1 ``` Use instead: ```python class Foo: @classmethod def bar(cls) -> int: return 1 ``` --- packages/examples/cvat/recording-oracle/src/core/config.py | 2 +- .../src/handlers/process_intermediate_results.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/core/config.py b/packages/examples/cvat/recording-oracle/src/core/config.py index fb7ccd220e..5199a35f9f 100644 --- a/packages/examples/cvat/recording-oracle/src/core/config.py +++ b/packages/examples/cvat/recording-oracle/src/core/config.py @@ -37,7 +37,7 @@ class Postgres: lock_timeout = int(os.environ.get("PG_LOCK_TIMEOUT", "3000")) # milliseconds @classmethod - def connection_url(cls): + def connection_url(cls) -> str: return f"postgresql://{cls.user}:{cls.password}@{cls.host}:{cls.port}/{cls.database}" diff --git a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py index a0c476dede..4af3ddcd57 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py @@ -262,7 +262,7 @@ def _prepare_merged_dataset(self): @classmethod def _put_gt_into_merged_dataset( cls, gt_dataset: dm.Dataset, merged_dataset: dm.Dataset, *, manifest: TaskManifest - ): + ) -> None: """ Updates the merged dataset inplace, writing GT annotations corresponding to the task type. """ From 733af17a514199d098123343a9361f2e743b8c80 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:00 +0200 Subject: [PATCH 19/44] [Recording Oracle] Unsafe autofix B006: mutable-argument-default ## What it does Checks for uses of mutable objects as function argument defaults. ## Why is this bad? Function defaults are evaluated once, when the function is defined. The same mutable object is then shared across all calls to the function. If the object is modified, those modifications will persist across calls, which can lead to unexpected behavior. Instead, prefer to use immutable data structures, or take `None` as a default, and initialize a new mutable object inside the function body for each call. Arguments with immutable type annotations will be ignored by this rule. Types outside of the standard library can be marked as immutable with the [`lint.flake8-bugbear.extend-immutable-calls`] configuration option. ## Known problems Mutable argument defaults can be used intentionally to cache computation results. Replacing the default with `None` or an immutable data structure does not work for such usages. Instead, prefer the `@functools.lru_cache` decorator from the standard library. ## Example ```python def add_to_list(item, some_list=[]): some_list.append(item) return some_list l1 = add_to_list(0) # [0] l2 = add_to_list(1) # [0, 1] ``` Use instead: ```python def add_to_list(item, some_list=None): if some_list is None: some_list = [] some_list.append(item) return some_list l1 = add_to_list(0) # [0] l2 = add_to_list(1) # [1] ``` ## Options - `lint.flake8-bugbear.extend-immutable-calls` ## References - [Python documentation: Default Argument Values](https://docs.python.org/3/tutorial/controlflow.html#default-argument-values) --- packages/examples/cvat/recording-oracle/src/chain/escrow.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/examples/cvat/recording-oracle/src/chain/escrow.py b/packages/examples/cvat/recording-oracle/src/chain/escrow.py index 26630f9045..8d4de44468 100644 --- a/packages/examples/cvat/recording-oracle/src/chain/escrow.py +++ b/packages/examples/cvat/recording-oracle/src/chain/escrow.py @@ -21,9 +21,11 @@ def validate_escrow( chain_id: int, escrow_address: str, *, - accepted_states: list[Status] = [Status.Pending], + accepted_states: list[Status] = None, allow_no_funds: bool = False, ) -> None: + if accepted_states is None: + accepted_states = [Status.Pending] assert accepted_states escrow = get_escrow(chain_id, escrow_address) From 373aed4f6d8c4132eae78db73c101de3a83a0fe9 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:01 +0200 Subject: [PATCH 20/44] [Recording Oracle] Unsafe autofix B007: unused-loop-control-variable ## What it does Checks for unused variables in loops (e.g., `for` and `while` statements). ## Why is this bad? Defining a variable in a loop statement that is never used can confuse readers. If the variable is intended to be unused (e.g., to facilitate destructuring of a tuple or other object), prefix it with an underscore to indicate the intent. Otherwise, remove the variable entirely. ## Example ```python for i, j in foo: bar(i) ``` Use instead: ```python for i, _j in foo: bar(i) ``` ## References - [PEP 8: Naming Conventions](https://peps.python.org/pep-0008/#naming-conventions) --- .../tests/integration/services/test_webhook_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py b/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py index a7cf4673d4..401cd7fcf1 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py @@ -140,7 +140,7 @@ def test_handle_webhook_fail(self): self.assertEqual(webhook.status, OracleWebhookStatuses.pending.value) # assumes Config.webhook_max_retries == 5 - for i in range(4): + for _i in range(4): inbox.handle_webhook_fail(self.session, webhook_id) webhook = self.session.query(Webhook).filter_by(id=webhook_id).first() From 364244a29820fc06234f65484f6adfd97786eff8 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:02 +0200 Subject: [PATCH 21/44] [Recording Oracle] Unsafe autofix B011: assert-false ## What it does Checks for uses of `assert False`. ## Why is this bad? Python removes `assert` statements when running in optimized mode (`python -O`), making `assert False` an unreliable means of raising an `AssertionError`. Instead, raise an `AssertionError` directly. ## Example ```python assert False ``` Use instead: ```python raise AssertionError ``` ## Fix safety This rule's fix is marked as unsafe, as changing an `assert` to a `raise` will change the behavior of your program when running in optimized mode (`python -O`). ## References - [Python documentation: `assert`](https://docs.python.org/3/reference/simple_stmts.html#the-assert-statement) --- .../examples/cvat/recording-oracle/src/core/oracle_events.py | 2 +- .../src/crons/process_exchange_oracle_webhooks.py | 2 +- .../src/handlers/process_intermediate_results.py | 2 +- .../examples/cvat/recording-oracle/src/utils/annotations.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/core/oracle_events.py b/packages/examples/cvat/recording-oracle/src/core/oracle_events.py index 6d12c94a82..7ef349cd55 100644 --- a/packages/examples/cvat/recording-oracle/src/core/oracle_events.py +++ b/packages/examples/cvat/recording-oracle/src/core/oracle_events.py @@ -81,7 +81,7 @@ def parse_event( if event_type not in sender_events: raise ValueError(f"Unknown event '{sender}.{event_type}'") else: - assert False, f"Unknown event sender type '{sender}'" + raise AssertionError(f"Unknown event sender type '{sender}'") event_class = get_class_for_event_type(event_type) return event_class.parse_obj(event_data or {}) diff --git a/packages/examples/cvat/recording-oracle/src/crons/process_exchange_oracle_webhooks.py b/packages/examples/cvat/recording-oracle/src/crons/process_exchange_oracle_webhooks.py index 236ce24719..95f7cd0bd9 100644 --- a/packages/examples/cvat/recording-oracle/src/crons/process_exchange_oracle_webhooks.py +++ b/packages/examples/cvat/recording-oracle/src/crons/process_exchange_oracle_webhooks.py @@ -74,7 +74,7 @@ def handle_exchange_oracle_event(webhook: Webhook, *, db_session: Session, logge ) case _: - assert False, f"Unknown exchange oracle event {webhook.event_type}" + raise AssertionError(f"Unknown exchange oracle event {webhook.event_type}") def process_outgoing_exchange_oracle_webhooks(): diff --git a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py index 4af3ddcd57..a4d6da12f3 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py @@ -309,7 +309,7 @@ def _put_gt_into_merged_dataset( ) merged_dataset.update(gt_dataset) case _: - assert False, f"Unknown task type {manifest.annotation.type}" + raise AssertionError(f"Unknown task type {manifest.annotation.type}") def validate(self) -> _ValidationResult: with TemporaryDirectory() as tempdir: diff --git a/packages/examples/cvat/recording-oracle/src/utils/annotations.py b/packages/examples/cvat/recording-oracle/src/utils/annotations.py index 036778a6ff..e49a0caee7 100644 --- a/packages/examples/cvat/recording-oracle/src/utils/annotations.py +++ b/packages/examples/cvat/recording-oracle/src/utils/annotations.py @@ -38,7 +38,7 @@ def shift_ann( ] ) else: - assert False, f"Unsupported annotation type '{ann.type}'" + raise AssertionError(f"Unsupported annotation type '{ann.type}'") return shifted_ann From 7f3d1613ad69e47f632ca6d27ec4a82e425bb257 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:03 +0200 Subject: [PATCH 22/44] [Recording Oracle] Unsafe autofix C400: unnecessary-generator-list ## What it does Checks for unnecessary generators that can be rewritten as `list` comprehensions (or with `list` directly). ## Why is this bad? It is unnecessary to use `list` around a generator expression, since there are equivalent comprehensions for these types. Using a comprehension is clearer and more idiomatic. Further, if the comprehension can be removed entirely, as in the case of `list(x for x in foo)`, it's better to use `list(foo)` directly, since it's even more direct. ## Examples ```python list(f(x) for x in foo) list(x for x in foo) list((x for x in foo)) ``` Use instead: ```python [f(x) for x in foo] list(foo) list(foo) ``` ## Fix safety This rule's fix is marked as unsafe, as it may occasionally drop comments when rewriting the call. In most cases, though, comments will be preserved. --- .../src/handlers/process_intermediate_results.py | 2 +- .../examples/cvat/recording-oracle/src/handlers/validation.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py index a4d6da12f3..e71adc8db6 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py @@ -1072,7 +1072,7 @@ def process_intermediate_results( validation_meta=validation_meta, resulting_annotations=updated_merged_dataset_archive.getvalue(), average_quality=np.mean( - list(v for v in job_results.values() if v != _TaskValidator.UNKNOWN_QUALITY and v >= 0) + [v for v in job_results.values() if v != _TaskValidator.UNKNOWN_QUALITY and v >= 0] or [0] ), ) diff --git a/packages/examples/cvat/recording-oracle/src/handlers/validation.py b/packages/examples/cvat/recording-oracle/src/handlers/validation.py index bb2470b64d..10a5b03e42 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/validation.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/validation.py @@ -194,13 +194,13 @@ def _handle_validation_result(self, validation_result: ValidationResult): OracleWebhookTypes.exchange_oracle, event=RecordingOracleEvent_TaskRejected( # TODO: update wrt. M2 API changes, send reason - rejected_job_ids=list( + rejected_job_ids=[ jid for jid, reason in validation_result.rejected_jobs.items() if not isinstance( reason, TooFewGtError ) # prevent such jobs from reannotation, can also be handled in ExcOr - ) + ] ), ) From 6a1f7c9b87629fda383bebf6ef5d470796692f33 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:04 +0200 Subject: [PATCH 23/44] [Recording Oracle] Unsafe autofix C408: unnecessary-collection-call ## What it does Checks for unnecessary `dict`, `list` or `tuple` calls that can be rewritten as empty literals. ## Why is this bad? It's unnecessary to call, e.g., `dict()` as opposed to using an empty literal (`{}`). The former is slower because the name `dict` must be looked up in the global scope in case it has been rebound. ## Examples ```python dict() dict(a=1, b=2) list() tuple() ``` Use instead: ```python {} {"a": 1, "b": 2} [] () ``` ## Fix safety This rule's fix is marked as unsafe, as it may occasionally drop comments when rewriting the call. In most cases, though, comments will be preserved. ## Options - `lint.flake8-comprehensions.allow-dict-calls-with-keyword-arguments` --- .../recording-oracle/src/endpoints/__init__.py | 10 +++++----- .../recording-oracle/src/services/cloud/s3.py | 4 ++-- .../recording-oracle/src/services/validation.py | 10 +++++----- .../test_process_reputation_oracle_webhooks.py | 2 +- .../integration/services/test_webhook_service.py | 16 ++++++++-------- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/endpoints/__init__.py b/packages/examples/cvat/recording-oracle/src/endpoints/__init__.py index f8ea5264d1..f3b167a2d7 100644 --- a/packages/examples/cvat/recording-oracle/src/endpoints/__init__.py +++ b/packages/examples/cvat/recording-oracle/src/endpoints/__init__.py @@ -22,11 +22,11 @@ def meta_route() -> MetaResponse: ] return MetaResponse.parse_obj( - dict( - message="Recording Oracle API", - version="0.1.0", - supported_networks=networks_info, - ) + { + "message": "Recording Oracle API", + "version": "0.1.0", + "supported_networks": networks_info, + } ) diff --git a/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py b/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py index 74b93f6ef8..ac229c5e9a 100644 --- a/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py +++ b/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py @@ -21,8 +21,8 @@ def __init__( ) -> None: super().__init__(bucket) session = boto3.Session( - **(dict(aws_access_key_id=access_key) if access_key else {}), - **(dict(aws_secret_access_key=secret_key) if secret_key else {}), + **({"aws_access_key_id": access_key} if access_key else {}), + **({"aws_secret_access_key": secret_key} if secret_key else {}), ) s3 = session.resource( "s3", **({"endpoint_url": unquote(endpoint_url)} if endpoint_url else {}) diff --git a/packages/examples/cvat/recording-oracle/src/services/validation.py b/packages/examples/cvat/recording-oracle/src/services/validation.py index 0ad3dcc185..4204bd90df 100644 --- a/packages/examples/cvat/recording-oracle/src/services/validation.py +++ b/packages/examples/cvat/recording-oracle/src/services/validation.py @@ -137,11 +137,11 @@ def update_gt_stats(session: Session, task_id: str, values: dict[str, int]): statement = psql_insert(GtStats).values( [ - dict( - task_id=task_id, - gt_key=gt_key, - failed_attempts=failed_attempts, - ) + { + "task_id": task_id, + "gt_key": gt_key, + "failed_attempts": failed_attempts, + } for gt_key, failed_attempts in values.items() ], ) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py b/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py index 4195fe940c..505a451dc9 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py @@ -70,7 +70,7 @@ def test_process_reputation_oracle_webhooks(self): chain_id = Networks.localhost.value escrow_address = create_escrow(self.w3) store_kvstore_value("webhook_url", expected_url) - event_data = dict() + event_data = {} mock_signature.return_value = (None, SIGNATURE) webhook = self.get_webhook(escrow_address, chain_id, event_data) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py b/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py index 401cd7fcf1..848ebc9f46 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py @@ -13,14 +13,14 @@ class ServiceIntegrationTest(unittest.TestCase): def setUp(self): self.session = SessionLocal() - self.webhook_kwargs = dict( - session=self.session, - escrow_address="0x1234567890123456789012345678901234567890", - chain_id=Networks.polygon_mainnet.value, - type=OracleWebhookTypes.exchange_oracle, - signature="signature", - event_type="task_finished", - ) + self.webhook_kwargs = { + "session": self.session, + "escrow_address": "0x1234567890123456789012345678901234567890", + "chain_id": Networks.polygon_mainnet.value, + "type": OracleWebhookTypes.exchange_oracle, + "signature": "signature", + "event_type": "task_finished", + } random.seed(42) def tearDown(self): From ffe11dc425bf9b9a632e3bfafe56cb97e2ac7b2f Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:05 +0200 Subject: [PATCH 24/44] [Recording Oracle] Unsafe autofix F841: unused-variable ## What it does Checks for the presence of unused variables in function scopes. ## Why is this bad? A variable that is defined but not used is likely a mistake, and should be removed to avoid confusion. If a variable is intentionally defined-but-not-used, it should be prefixed with an underscore, or some other value that adheres to the [`lint.dummy-variable-rgx`] pattern. Under [preview mode](https://docs.astral.sh/ruff/preview), this rule also triggers on unused unpacked assignments (for example, `x, y = foo()`). ## Example ```python def foo(): x = 1 y = 2 return x ``` Use instead: ```python def foo(): x = 1 return x ``` ## Options - `lint.dummy-variable-rgx` --- .../cvat/recording-oracle/tests/integration/chain/test_web3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py index 2569b2ec7b..8f73fc9fea 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py @@ -55,7 +55,7 @@ def test_get_web3_localhost(self): def test_get_web3_invalid_chain_id(self): with self.assertRaises(ValueError) as error: - w3 = get_web3(1234) + get_web3(1234) self.assertEqual( "1234 is not in available list of networks.", str(error.exception), From a6048c652bdc16b6fa6f48b706d87a9b77351576 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:06 +0200 Subject: [PATCH 25/44] [Recording Oracle] Unsafe autofix FLY002: static-join-to-f-string ## What it does Checks for `str.join` calls that can be replaced with f-strings. ## Why is this bad? f-strings are more readable and generally preferred over `str.join` calls. ## Example ```python " ".join((foo, bar)) ``` Use instead: ```python f"{foo} {bar}" ``` ## References - [Python documentation: f-strings](https://docs.python.org/3/reference/lexical_analysis.html#f-strings) --- packages/examples/cvat/recording-oracle/src/core/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/examples/cvat/recording-oracle/src/core/config.py b/packages/examples/cvat/recording-oracle/src/core/config.py index 5199a35f9f..c3dae16ebf 100644 --- a/packages/examples/cvat/recording-oracle/src/core/config.py +++ b/packages/examples/cvat/recording-oracle/src/core/config.py @@ -212,7 +212,7 @@ def validate(cls) -> None: ex_prefix = "Wrong server configuration." if (cls.pgp_public_key_url or cls.pgp_passphrase) and not cls.pgp_private_key: - raise Exception(" ".join([ex_prefix, "The PGP_PRIVATE_KEY environment is not set."])) + raise Exception(f"{ex_prefix} The PGP_PRIVATE_KEY environment is not set.") if cls.pgp_private_key: try: From d6ccab755e718865d93cd92c0e0786eb20aeff0c Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:07 +0200 Subject: [PATCH 26/44] [Recording Oracle] Unsafe autofix PT009: pytest-unittest-assertion ## What it does Checks for uses of assertion methods from the `unittest` module. ## Why is this bad? To make use of `pytest`'s assertion rewriting, a regular `assert` statement is preferred over `unittest`'s assertion methods. ## Example ```python import unittest class TestFoo(unittest.TestCase): def test_foo(self): self.assertEqual(a, b) ``` Use instead: ```python import unittest class TestFoo(unittest.TestCase): def test_foo(self): assert a == b ``` ## References - [`pytest` documentation: Assertion introspection details](https://docs.pytest.org/en/7.1.x/how-to/assert.html#assertion-introspection-details) --- .../tests/integration/chain/test_escrow.py | 18 ++++----- .../tests/integration/chain/test_kvstore.py | 26 ++++++------ .../tests/integration/chain/test_web3.py | 40 ++++++++----------- .../test_process_exchange_oracle_webhooks.py | 16 ++++---- ...test_process_reputation_oracle_webhooks.py | 12 +++--- .../services/test_webhook_service.py | 40 +++++++++---------- 6 files changed, 73 insertions(+), 79 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py index 26e4382d19..c5c7cc8cf2 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py @@ -100,8 +100,8 @@ def test_get_escrow_manifest(self): mock_get_escrow.return_value = self.escrow() manifest = get_escrow_manifest(self.network_config.chain_id, self.escrow_address) - self.assertIsInstance(manifest, dict) - self.assertIsNotNone(manifest) + assert isinstance(manifest, dict) + assert manifest is not None def test_get_encrypted_escrow_manifest(self): with ( @@ -122,7 +122,7 @@ def test_get_encrypted_escrow_manifest(self): encrypted_manifest = EncryptionUtils.encrypt( original_manifest, public_keys=[PGP_PUBLIC_KEY1, PGP_PUBLIC_KEY2] ) - self.assertNotEqual(encrypted_manifest, original_manifest) + assert encrypted_manifest != original_manifest mock_download.return_value = encrypted_manifest.encode() downloaded_manifest_content = get_escrow_manifest( @@ -137,9 +137,9 @@ def test_store_results(self): results = store_results( self.w3.eth.chain_id, escrow_address, DEFAULT_MANIFEST_URL, DEFAULT_HASH ) - self.assertIsNone(results) + assert results is None intermediate_results_url = get_intermediate_results_url(self.w3, escrow_address) - self.assertEqual(intermediate_results_url, DEFAULT_MANIFEST_URL) + assert intermediate_results_url == DEFAULT_MANIFEST_URL def test_store_results_invalid_url(self): escrow_address = create_escrow(self.w3) @@ -147,7 +147,7 @@ def test_store_results_invalid_url(self): mock_function.return_value = self.w3 with self.assertRaises(EscrowClientError) as error: store_results(self.w3.eth.chain_id, escrow_address, "invalid_url", DEFAULT_HASH) - self.assertEqual("Invalid URL: invalid_url", str(error.exception)) + assert "Invalid URL: invalid_url" == str(error.exception) def test_store_results_invalid_hash(self): escrow_address = create_escrow(self.w3) @@ -155,7 +155,7 @@ def test_store_results_invalid_hash(self): mock_function.return_value = self.w3 with self.assertRaises(EscrowClientError) as error: store_results(self.w3.eth.chain_id, escrow_address, DEFAULT_MANIFEST_URL, "") - self.assertEqual("Invalid empty hash", str(error.exception)) + assert "Invalid empty hash" == str(error.exception) def test_get_reputation_oracle_address(self): escrow_address = create_escrow(self.w3) @@ -168,8 +168,8 @@ def test_get_reputation_oracle_address(self): mock_escrow.reputation_oracle = REPUTATION_ORACLE_ADDRESS mock_get_escrow.return_value = mock_escrow address = get_reputation_oracle_address(self.w3.eth.chain_id, escrow_address) - self.assertIsInstance(address, str) - self.assertIsNotNone(address) + assert isinstance(address, str) + assert address is not None def test_get_reputation_oracle_address_invalid_address(self): with patch("src.chain.escrow.get_web3") as mock_function: diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py index fb70e97e52..7908b8df2a 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py @@ -47,7 +47,7 @@ def test_get_reputation_oracle_url(self): mock_leader.return_value = MagicMock(webhook_url=DEFAULT_MANIFEST_URL) reputation_url = get_reputation_oracle_url(self.w3.eth.chain_id, escrow_address) - self.assertEqual(reputation_url, DEFAULT_MANIFEST_URL) + assert reputation_url == DEFAULT_MANIFEST_URL def test_get_reputation_oracle_url_invalid_escrow(self): with patch("src.chain.kvstore.get_web3") as mock_function: @@ -72,21 +72,21 @@ def test_get_reputation_oracle_url_invalid_address(self): reputation_url = get_reputation_oracle_url( self.w3.eth.chain_id, REPUTATION_ORACLE_ADDRESS ) - self.assertEqual(reputation_url, "") + assert reputation_url == "" def test_get_role_by_address(self): store_kvstore_value("role", "Reputation Oracle") with patch("src.chain.kvstore.get_web3") as mock_function: mock_function.return_value = self.w3 reputation_url = get_role_by_address(self.w3.eth.chain_id, REPUTATION_ORACLE_ADDRESS) - self.assertEqual(reputation_url, "Reputation Oracle") + assert reputation_url == "Reputation Oracle" def test_get_role_by_address_invalid_escrow(self): with patch("src.chain.kvstore.get_web3") as mock_function: mock_function.return_value = self.w3 with self.assertRaises(KVStoreClientError) as error: get_role_by_address(self.w3.eth.chain_id, "invalid_address") - self.assertEqual("Invalid address: invalid_address", str(error.exception)) + assert "Invalid address: invalid_address" == str(error.exception) def test_get_role_by_address_invalid_address(self): create_escrow(self.w3) @@ -94,7 +94,7 @@ def test_get_role_by_address_invalid_address(self): with patch("src.chain.kvstore.get_web3") as mock_function: mock_function.return_value = self.w3 reputation_url = get_role_by_address(self.w3.eth.chain_id, REPUTATION_ORACLE_ADDRESS) - self.assertEqual(reputation_url, "") + assert reputation_url == "" def test_store_public_key(self): PGP_PUBLIC_KEY_URL_1 = "http://pgp-public-key-url-1" @@ -141,7 +141,7 @@ def set_file_url_and_hash(url: str, key: str): mock_web3.return_value = self.w3 kvstore_client = KVStoreClient(self.w3) - self.assertIsNone(kvstore_client.get_file_url_and_verify_hash(LocalhostConfig.addr)) + assert kvstore_client.get_file_url_and_verify_hash(LocalhostConfig.addr) is None # check that public key will be set to KVStore at first time with patch( @@ -150,9 +150,9 @@ def set_file_url_and_hash(url: str, key: str): mock_set_file_url_and_hash.side_effect = set_file_url_and_hash register_in_kvstore() mock_set_file_url_and_hash.assert_called_once() - self.assertEqual( - kvstore_client.get_file_url_and_verify_hash(LocalhostConfig.addr), - PGP_PUBLIC_KEY_URL_1, + assert ( + kvstore_client.get_file_url_and_verify_hash(LocalhostConfig.addr) + == PGP_PUBLIC_KEY_URL_1 ) # check that the same public key URL is not written to KVStore a second time @@ -171,7 +171,7 @@ def set_file_url_and_hash(url: str, key: str): store["public_key_hash"] = "corrupted_hash" register_in_kvstore() mock_set_file_url_and_hash.assert_called_once() - self.assertNotEqual(store["public_key_hash"], "corrupted_hash") + assert store["public_key_hash"] != "corrupted_hash" # check that a new public key URL will be written to KVStore when an outdated URL is stored there with ( @@ -186,7 +186,7 @@ def set_file_url_and_hash(url: str, key: str): mock_set_file_url_and_hash.side_effect = set_file_url_and_hash register_in_kvstore() mock_set_file_url_and_hash.assert_called_once() - self.assertEqual( - kvstore_client.get_file_url_and_verify_hash(LocalhostConfig.addr), - PGP_PUBLIC_KEY_URL_2, + assert ( + kvstore_client.get_file_url_and_verify_hash(LocalhostConfig.addr) + == PGP_PUBLIC_KEY_URL_2 ) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py index 8f73fc9fea..525b50ab1b 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py @@ -31,9 +31,9 @@ class PolygonMainnetConfig: with patch("src.chain.web3.Config.polygon_mainnet", PolygonMainnetConfig): w3 = get_web3(ChainId.POLYGON.value) - self.assertIsInstance(w3, Web3) - self.assertEqual(w3.eth.default_account, DEFAULT_GAS_PAYER) - self.assertEqual(w3.manager._provider.endpoint_uri, PolygonMainnetConfig.rpc_api) + assert isinstance(w3, Web3) + assert w3.eth.default_account == DEFAULT_GAS_PAYER + assert w3.manager._provider.endpoint_uri == PolygonMainnetConfig.rpc_api def test_get_web3_amoy(self): class PolygonAmoyConfig: @@ -43,23 +43,20 @@ class PolygonAmoyConfig: with patch("src.chain.web3.Config.polygon_amoy", PolygonAmoyConfig): w3 = get_web3(ChainId.POLYGON_AMOY.value) - self.assertIsInstance(w3, Web3) - self.assertEqual(w3.eth.default_account, DEFAULT_GAS_PAYER) - self.assertEqual(w3.manager._provider.endpoint_uri, PolygonAmoyConfig.rpc_api) + assert isinstance(w3, Web3) + assert w3.eth.default_account == DEFAULT_GAS_PAYER + assert w3.manager._provider.endpoint_uri == PolygonAmoyConfig.rpc_api def test_get_web3_localhost(self): w3 = get_web3(ChainId.LOCALHOST.value) - self.assertIsInstance(w3, Web3) - self.assertEqual(w3.eth.default_account, DEFAULT_GAS_PAYER) - self.assertEqual(w3.manager._provider.endpoint_uri, LocalhostConfig.rpc_api) + assert isinstance(w3, Web3) + assert w3.eth.default_account == DEFAULT_GAS_PAYER + assert w3.manager._provider.endpoint_uri == LocalhostConfig.rpc_api def test_get_web3_invalid_chain_id(self): with self.assertRaises(ValueError) as error: get_web3(1234) - self.assertEqual( - "1234 is not in available list of networks.", - str(error.exception), - ) + assert "1234 is not in available list of networks." == str(error.exception) def test_sign_message_polygon(self): with ( @@ -71,7 +68,7 @@ def test_sign_message_polygon(self): ): mock_function.return_value = self.w3 signed_message, _ = sign_message(ChainId.POLYGON.value, "message") - self.assertEqual(signed_message, SIGNATURE) + assert signed_message == SIGNATURE def test_sign_message_amoy(self): with ( @@ -83,33 +80,30 @@ def test_sign_message_amoy(self): ): mock_function.return_value = self.w3 signed_message, _ = sign_message(ChainId.POLYGON_AMOY.value, "message") - self.assertEqual(signed_message, SIGNATURE) + assert signed_message == SIGNATURE def test_sign_message_invalid_chain_id(self): with self.assertRaises(ValueError) as error: sign_message(1234, "message") - self.assertEqual( - "1234 is not in available list of networks.", - str(error.exception), - ) + assert "1234 is not in available list of networks." == str(error.exception) def test_recover_signer(self): with patch("src.chain.web3.get_web3") as mock_function: mock_function.return_value = self.w3 signer = recover_signer(ChainId.POLYGON.value, "message", SIGNATURE) - self.assertEqual(signer, DEFAULT_GAS_PAYER) + assert signer == DEFAULT_GAS_PAYER def test_recover_signer_invalid_signature(self): with patch("src.chain.web3.get_web3") as mock_function: mock_function.return_value = self.w3 signer = recover_signer(ChainId.POLYGON.value, "test", SIGNATURE) - self.assertNotEqual(signer, DEFAULT_GAS_PAYER) + assert signer != DEFAULT_GAS_PAYER def test_validate_address(self): address = validate_address(DEFAULT_GAS_PAYER) - self.assertEqual(address, DEFAULT_GAS_PAYER) + assert address == DEFAULT_GAS_PAYER def test_validate_address_invalid_address(self): with self.assertRaises(ValueError) as error: validate_address("invalid_address") - self.assertEqual("invalid_address is not a correct Web3 address", str(error.exception)) + assert "invalid_address is not a correct Web3 address" == str(error.exception) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_exchange_oracle_webhooks.py b/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_exchange_oracle_webhooks.py index 7204d319da..ab90d5cad2 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_exchange_oracle_webhooks.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_exchange_oracle_webhooks.py @@ -63,8 +63,8 @@ def test_process_exchange_oracle_webhook(self): updated_webhook = ( self.session.execute(select(Webhook).where(Webhook.id == webhook.id)).scalars().first() ) - self.assertEqual(updated_webhook.status, OracleWebhookStatuses.completed) - self.assertEqual(updated_webhook.attempts, 1) + assert updated_webhook.status == OracleWebhookStatuses.completed + assert updated_webhook.attempts == 1 def test_process_recording_oracle_webhooks_invalid_escrow_address(self): escrow_address = "invalid_address" @@ -83,8 +83,8 @@ def test_process_recording_oracle_webhooks_invalid_escrow_address(self): self.session.execute(select(Webhook).where(Webhook.id == webhook.id)).scalars().first() ) - self.assertEqual(updated_webhook.status, OracleWebhookStatuses.pending.value) - self.assertEqual(updated_webhook.attempts, 1) + assert updated_webhook.status == OracleWebhookStatuses.pending.value + assert updated_webhook.attempts == 1 def test_process_recording_oracle_webhooks_invalid_escrow_balance(self): escrow_address = create_escrow(self.w3) @@ -103,8 +103,8 @@ def test_process_recording_oracle_webhooks_invalid_escrow_balance(self): self.session.execute(select(Webhook).where(Webhook.id == webhook.id)).scalars().first() ) - self.assertEqual(updated_webhook.status, OracleWebhookStatuses.pending.value) - self.assertEqual(updated_webhook.attempts, 1) + assert updated_webhook.status == OracleWebhookStatuses.pending.value + assert updated_webhook.attempts == 1 @patch("src.chain.escrow.EscrowClient.get_manifest_url") def test_process_job_launcher_webhooks_invalid_manifest_url(self, mock_manifest_url): @@ -123,5 +123,5 @@ def test_process_job_launcher_webhooks_invalid_manifest_url(self, mock_manifest_ self.session.execute(select(Webhook).where(Webhook.id == webhook.id)).scalars().first() ) - self.assertEqual(updated_webhook.status, OracleWebhookStatuses.pending.value) - self.assertEqual(updated_webhook.attempts, 1) + assert updated_webhook.status == OracleWebhookStatuses.pending.value + assert updated_webhook.attempts == 1 diff --git a/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py b/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py index 505a451dc9..743c18af0a 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py @@ -96,8 +96,8 @@ def test_process_reputation_oracle_webhooks(self): "event_type": RecordingOracleEventTypes.task_completed.value, }, ) - self.assertEqual(updated_webhook.status, OracleWebhookStatuses.completed.value) - self.assertEqual(updated_webhook.attempts, 1) + assert updated_webhook.status == OracleWebhookStatuses.completed.value + assert updated_webhook.attempts == 1 def test_process_reputation_oracle_webhooks_invalid_escrow_address(self): chain_id = Networks.localhost.value @@ -114,8 +114,8 @@ def test_process_reputation_oracle_webhooks_invalid_escrow_address(self): self.session.execute(select(Webhook).where(Webhook.id == webhook.id)).scalars().first() ) - self.assertEqual(updated_webhook.status, OracleWebhookStatuses.pending.value) - self.assertEqual(updated_webhook.attempts, 1) + assert updated_webhook.status == OracleWebhookStatuses.pending.value + assert updated_webhook.attempts == 1 def test_process_reputation_oracle_webhooks_invalid_reputation_oracle_url(self): with patch( @@ -133,5 +133,5 @@ def test_process_reputation_oracle_webhooks_invalid_reputation_oracle_url(self): .scalars() .first() ) - self.assertEqual(updated_webhook.status, OracleWebhookStatuses.pending.value) - self.assertEqual(updated_webhook.attempts, 1) + assert updated_webhook.status == OracleWebhookStatuses.pending.value + assert updated_webhook.attempts == 1 diff --git a/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py b/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py index 848ebc9f46..063787370b 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py @@ -44,12 +44,12 @@ def test_create_webhook(self): webhook = self.session.query(Webhook).filter_by(id=webhook_id).first() - self.assertEqual(webhook.escrow_address, self.webhook_kwargs["escrow_address"]) - self.assertEqual(webhook.chain_id, self.webhook_kwargs["chain_id"]) - self.assertEqual(webhook.attempts, 0) - self.assertEqual(webhook.signature, self.webhook_kwargs["signature"]) - self.assertEqual(webhook.type, OracleWebhookTypes.exchange_oracle) - self.assertEqual(webhook.status, OracleWebhookStatuses.pending) + assert webhook.escrow_address == self.webhook_kwargs["escrow_address"] + assert webhook.chain_id == self.webhook_kwargs["chain_id"] + assert webhook.attempts == 0 + assert webhook.signature == self.webhook_kwargs["signature"] + assert webhook.type == OracleWebhookTypes.exchange_oracle + assert webhook.status == OracleWebhookStatuses.pending # TODO: check intended fields and verify those def _test_none_webhook_argument(self, argument_name, error_type): @@ -95,25 +95,25 @@ def test_get_pending_webhooks(self): pending_webhooks = inbox.get_pending_webhooks( self.session, OracleWebhookTypes.exchange_oracle ) - self.assertEqual(len(pending_webhooks), 2) - self.assertEqual(pending_webhooks[0].id, webhook1.id) - self.assertEqual(pending_webhooks[1].id, webhook2.id) + assert len(pending_webhooks) == 2 + assert pending_webhooks[0].id == webhook1.id + assert pending_webhooks[1].id == webhook2.id pending_webhooks = inbox.get_pending_webhooks( self.session, OracleWebhookTypes.reputation_oracle ) - self.assertEqual(len(pending_webhooks), 1) - self.assertEqual(pending_webhooks[0].id, webhook4.id) + assert len(pending_webhooks) == 1 + assert pending_webhooks[0].id == webhook4.id def test_update_webhook_status(self): webhook_id = inbox.create_webhook(**self.webhook_kwargs) webhook = self.session.query(Webhook).filter_by(id=webhook_id).first() - self.assertEqual(webhook.status, OracleWebhookStatuses.pending) + assert webhook.status == OracleWebhookStatuses.pending inbox.update_webhook_status(self.session, webhook_id, OracleWebhookStatuses.completed) webhook = self.session.query(Webhook).filter_by(id=webhook_id).first() - self.assertEqual(webhook.status, OracleWebhookStatuses.completed) + assert webhook.status == OracleWebhookStatuses.completed def test_update_webhook_invalid_status(self): webhook_id = inbox.create_webhook(**self.webhook_kwargs) @@ -127,17 +127,17 @@ def test_handle_webhook_success(self): webhook = self.session.query(Webhook).filter_by(id=webhook_id).first() - self.assertEqual(webhook.attempts, 1) - self.assertEqual(webhook.status, OracleWebhookStatuses.completed.value) + assert webhook.attempts == 1 + assert webhook.status == OracleWebhookStatuses.completed.value def test_handle_webhook_fail(self): webhook_id = inbox.create_webhook(**self.webhook_kwargs) inbox.handle_webhook_fail(self.session, webhook_id) webhook = self.session.query(Webhook).filter_by(id=webhook_id).first() - self.assertEqual(webhook.attempts, 1) - self.assertEqual(webhook.type, OracleWebhookTypes.exchange_oracle.value) - self.assertEqual(webhook.status, OracleWebhookStatuses.pending.value) + assert webhook.attempts == 1 + assert webhook.type == OracleWebhookTypes.exchange_oracle.value + assert webhook.status == OracleWebhookStatuses.pending.value # assumes Config.webhook_max_retries == 5 for _i in range(4): @@ -145,5 +145,5 @@ def test_handle_webhook_fail(self): webhook = self.session.query(Webhook).filter_by(id=webhook_id).first() - self.assertEqual(webhook.attempts, 5) - self.assertEqual(webhook.status, OracleWebhookStatuses.failed.value) + assert webhook.attempts == 5 + assert webhook.status == OracleWebhookStatuses.failed.value From 73ad3ecfbd2c73006f8a677ce5963b6a96fdb63e Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:08 +0200 Subject: [PATCH 27/44] [Recording Oracle] Unsafe autofix PT027: pytest-unittest-raises-assertion ## What it does Checks for uses of exception-related assertion methods from the `unittest` module. ## Why is this bad? To enforce the assertion style recommended by `pytest`, `pytest.raises` is preferred over the exception-related assertion methods in `unittest`, like `assertRaises`. ## Example ```python import unittest class TestFoo(unittest.TestCase): def test_foo(self): with self.assertRaises(ValueError): raise ValueError("foo") ``` Use instead: ```python import unittest import pytest class TestFoo(unittest.TestCase): def test_foo(self): with pytest.raises(ValueError): raise ValueError("foo") ``` ## References - [`pytest` documentation: Assertions about expected exceptions](https://docs.pytest.org/en/latest/how-to/assert.html#assertions-about-expected-exceptions) --- .../tests/integration/chain/test_escrow.py | 4 ++-- .../tests/integration/chain/test_kvstore.py | 2 +- .../recording-oracle/tests/integration/chain/test_web3.py | 7 ++++--- .../tests/integration/services/test_webhook_service.py | 5 +++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py index c5c7cc8cf2..55441c6c06 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py @@ -145,7 +145,7 @@ def test_store_results_invalid_url(self): escrow_address = create_escrow(self.w3) with patch("src.chain.escrow.get_web3") as mock_function: mock_function.return_value = self.w3 - with self.assertRaises(EscrowClientError) as error: + with pytest.raises(EscrowClientError) as error: store_results(self.w3.eth.chain_id, escrow_address, "invalid_url", DEFAULT_HASH) assert "Invalid URL: invalid_url" == str(error.exception) @@ -153,7 +153,7 @@ def test_store_results_invalid_hash(self): escrow_address = create_escrow(self.w3) with patch("src.chain.escrow.get_web3") as mock_function: mock_function.return_value = self.w3 - with self.assertRaises(EscrowClientError) as error: + with pytest.raises(EscrowClientError) as error: store_results(self.w3.eth.chain_id, escrow_address, DEFAULT_MANIFEST_URL, "") assert "Invalid empty hash" == str(error.exception) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py index 7908b8df2a..d00ceedfc5 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py @@ -84,7 +84,7 @@ def test_get_role_by_address(self): def test_get_role_by_address_invalid_escrow(self): with patch("src.chain.kvstore.get_web3") as mock_function: mock_function.return_value = self.w3 - with self.assertRaises(KVStoreClientError) as error: + with pytest.raises(KVStoreClientError) as error: get_role_by_address(self.w3.eth.chain_id, "invalid_address") assert "Invalid address: invalid_address" == str(error.exception) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py index 525b50ab1b..da88473ae2 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import patch +import pytest from human_protocol_sdk.constants import ChainId from web3 import HTTPProvider, Web3 from web3.middleware import construct_sign_and_send_raw_middleware @@ -54,7 +55,7 @@ def test_get_web3_localhost(self): assert w3.manager._provider.endpoint_uri == LocalhostConfig.rpc_api def test_get_web3_invalid_chain_id(self): - with self.assertRaises(ValueError) as error: + with pytest.raises(ValueError) as error: get_web3(1234) assert "1234 is not in available list of networks." == str(error.exception) @@ -83,7 +84,7 @@ def test_sign_message_amoy(self): assert signed_message == SIGNATURE def test_sign_message_invalid_chain_id(self): - with self.assertRaises(ValueError) as error: + with pytest.raises(ValueError) as error: sign_message(1234, "message") assert "1234 is not in available list of networks." == str(error.exception) @@ -104,6 +105,6 @@ def test_validate_address(self): assert address == DEFAULT_GAS_PAYER def test_validate_address_invalid_address(self): - with self.assertRaises(ValueError) as error: + with pytest.raises(ValueError) as error: validate_address("invalid_address") assert "invalid_address is not a correct Web3 address" == str(error.exception) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py b/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py index 063787370b..08012f66f1 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py @@ -2,6 +2,7 @@ import unittest import uuid +import pytest from sqlalchemy.exc import IntegrityError from src.core.types import Networks, OracleWebhookStatuses, OracleWebhookTypes @@ -55,7 +56,7 @@ def test_create_webhook(self): def _test_none_webhook_argument(self, argument_name, error_type): kwargs = dict(**self.webhook_kwargs) kwargs[argument_name] = None - with self.assertRaises(error_type): + with pytest.raises(error_type): inbox.create_webhook(**kwargs) self.session.commit() @@ -117,7 +118,7 @@ def test_update_webhook_status(self): def test_update_webhook_invalid_status(self): webhook_id = inbox.create_webhook(**self.webhook_kwargs) - with self.assertRaises(AttributeError): + with pytest.raises(AttributeError): inbox.update_webhook_status(self.session, webhook_id, "Invalid status") def test_handle_webhook_success(self): From 9a3f9ab6c7c6010037006d1c6e816a85adcb286b Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:08 +0200 Subject: [PATCH 28/44] [Recording Oracle] Unsafe autofix RET504: unnecessary-assign ## What it does Checks for variable assignments that immediately precede a `return` of the assigned variable. ## Why is this bad? The variable assignment is not necessary, as the value can be returned directly. ## Example ```python def foo(): bar = 1 return bar ``` Use instead: ```python def foo(): return 1 ``` --- .../examples/cvat/recording-oracle/src/chain/kvstore.py | 4 +--- packages/examples/cvat/recording-oracle/src/chain/web3.py | 4 +--- .../examples/cvat/recording-oracle/src/services/webhook.py | 3 +-- .../cvat/recording-oracle/tests/utils/setup_escrow.py | 6 ++---- 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/chain/kvstore.py b/packages/examples/cvat/recording-oracle/src/chain/kvstore.py index b91f2ec3a1..0d7c763914 100644 --- a/packages/examples/cvat/recording-oracle/src/chain/kvstore.py +++ b/packages/examples/cvat/recording-oracle/src/chain/kvstore.py @@ -11,9 +11,7 @@ def get_role_by_address(chain_id: int, address: str) -> str: web3 = get_web3(chain_id) kvstore_client = KVStoreClient(web3) - role = kvstore_client.get(address, "role") - - return role + return kvstore_client.get(address, "role") def get_exchange_oracle_url(chain_id: int, escrow_address: str) -> str: diff --git a/packages/examples/cvat/recording-oracle/src/chain/web3.py b/packages/examples/cvat/recording-oracle/src/chain/web3.py index cc427c96b9..e76f77bf95 100644 --- a/packages/examples/cvat/recording-oracle/src/chain/web3.py +++ b/packages/examples/cvat/recording-oracle/src/chain/web3.py @@ -71,9 +71,7 @@ def sign_message(chain_id: Networks, message) -> str: def recover_signer(chain_id: Networks, message, signature: str) -> str: w3 = get_web3(chain_id) message_hash = encode_defunct(text=serialize_message(message)) - signer = w3.eth.account.recover_message(message_hash, signature=signature) - - return signer + return w3.eth.account.recover_message(message_hash, signature=signature) def validate_address(escrow_address: str) -> str: diff --git a/packages/examples/cvat/recording-oracle/src/services/webhook.py b/packages/examples/cvat/recording-oracle/src/services/webhook.py index 6627d11285..6a2623b184 100644 --- a/packages/examples/cvat/recording-oracle/src/services/webhook.py +++ b/packages/examples/cvat/recording-oracle/src/services/webhook.py @@ -94,7 +94,7 @@ def get_pending_webhooks( limit: int = 10, for_update: bool | ForUpdateParams = False, ) -> list[Webhook]: - webhooks = ( + return ( _maybe_for_update(session.query(Webhook), enable=for_update) .where( Webhook.direction == self.direction.value, @@ -105,7 +105,6 @@ def get_pending_webhooks( .limit(limit) .all() ) - return webhooks def update_webhook_status( self, session: Session, webhook_id: str, status: OracleWebhookStatuses diff --git a/packages/examples/cvat/recording-oracle/tests/utils/setup_escrow.py b/packages/examples/cvat/recording-oracle/tests/utils/setup_escrow.py index afc161a1ed..b840fbc44b 100644 --- a/packages/examples/cvat/recording-oracle/tests/utils/setup_escrow.py +++ b/packages/examples/cvat/recording-oracle/tests/utils/setup_escrow.py @@ -25,7 +25,7 @@ def create_escrow(web3: Web3): escrow_client = EscrowClient(web3) staking_client.approve_stake(amount) staking_client.stake(amount) - escrow_address = escrow_client.create_and_setup_escrow( + return escrow_client.create_and_setup_escrow( token_address=NETWORKS[ChainId.LOCALHOST]["hmt_address"], trusted_handlers=[web3.eth.default_account], job_requester_id=JOB_REQUESTER_ID, @@ -40,7 +40,6 @@ def create_escrow(web3: Web3): hash=DEFAULT_HASH, ), ) - return escrow_address def fund_escrow(web3: Web3, escrow_address: str): @@ -57,7 +56,6 @@ def bulk_payout(web3: Web3, escrow_address: str, recipient: str, amount: Decimal def get_intermediate_results_url(web3: Web3, escrow_address: str): escrow_client = EscrowClient(web3) - intermediate_results_url = ( + return ( escrow_client._get_escrow_contract(escrow_address).functions.intermediateResultsUrl().call() ) - return intermediate_results_url From 497bddd5b6391b6e656d5c69aa6900935e401d1b Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:09 +0200 Subject: [PATCH 29/44] [Recording Oracle] Unsafe autofix RSE102: unnecessary-paren-on-raise-exception ## What it does Checks for unnecessary parentheses on raised exceptions. ## Why is this bad? If an exception is raised without any arguments, parentheses are not required, as the `raise` statement accepts either an exception instance or an exception class (which is then implicitly instantiated). Removing the parentheses makes the code more concise. ## Known problems Parentheses can only be omitted if the exception is a class, as opposed to a function call. This rule isn't always capable of distinguishing between the two. For example, if you import a function `module.get_exception` from another module, and `module.get_exception` returns an exception object, this rule will incorrectly mark the parentheses in `raise module.get_exception()` as unnecessary. ## Example ```python raise TypeError() ``` Use instead: ```python raise TypeError ``` ## References - [Python documentation: The `raise` statement](https://docs.python.org/3/reference/simple_stmts.html#the-raise-statement) --- .../cvat/recording-oracle/src/validation/dataset_comparison.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py index b28350d994..810c080f04 100644 --- a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py +++ b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py @@ -105,7 +105,7 @@ def compare(self, gt_dataset: dm.Dataset, ds_dataset: dm.Dataset) -> float: dataset_failed_gts.add(gt_sample.id) if dataset_excluded_gts_count == len(gt_dataset): - raise TooFewGtError() + raise TooFewGtError dataset_accuracy = 0 if dataset_total_anns_to_compare: From bad2f6d7d5d8dbf1e88886f31e53c5653e6a39c6 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:10 +0200 Subject: [PATCH 30/44] [Recording Oracle] Unsafe autofix SIM101: duplicate-isinstance-call ## What it does Checks for multiple `isinstance` calls on the same target. ## Why is this bad? To check if an object is an instance of any one of multiple types or classes, it is unnecessary to use multiple `isinstance` calls, as the second argument of the `isinstance` built-in function accepts a tuple of types and classes. Using a single `isinstance` call implements the same behavior with more concise code and clearer intent. ## Example ```python if isinstance(obj, int) or isinstance(obj, float): pass ``` Use instead: ```python if isinstance(obj, (int, float)): pass ``` ## References - [Python documentation: `isinstance`](https://docs.python.org/3/library/functions.html#isinstance) --- .../examples/cvat/recording-oracle/src/utils/annotations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/examples/cvat/recording-oracle/src/utils/annotations.py b/packages/examples/cvat/recording-oracle/src/utils/annotations.py index e49a0caee7..2e897b3b93 100644 --- a/packages/examples/cvat/recording-oracle/src/utils/annotations.py +++ b/packages/examples/cvat/recording-oracle/src/utils/annotations.py @@ -99,7 +99,7 @@ def __init__( dst_label_cat = dm.LabelCategories(attributes=deepcopy(src_label_cat.attributes)) for dst_label in dst_labels: - assert isinstance(dst_label, str) or isinstance(dst_label, tuple) + assert isinstance(dst_label, (str, tuple)) dst_parent = "" if isinstance(dst_label, tuple): From 85314ec7d38156610dd465c78d1a3dc79ac44c2c Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:11 +0200 Subject: [PATCH 31/44] [Recording Oracle] Unsafe autofix SIM102: collapsible-if ## What it does Checks for nested `if` statements that can be collapsed into a single `if` statement. ## Why is this bad? Nesting `if` statements leads to deeper indentation and makes code harder to read. Instead, combine the conditions into a single `if` statement with an `and` operator. ## Example ```python if foo: if bar: ... ``` Use instead: ```python if foo and bar: ... ``` ## References - [Python documentation: The `if` statement](https://docs.python.org/3/reference/compound_stmts.html#the-if-statement) - [Python documentation: Boolean operations](https://docs.python.org/3/reference/expressions.html#boolean-operations) --- packages/examples/cvat/recording-oracle/src/chain/escrow.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/chain/escrow.py b/packages/examples/cvat/recording-oracle/src/chain/escrow.py index 8d4de44468..50e136bf93 100644 --- a/packages/examples/cvat/recording-oracle/src/chain/escrow.py +++ b/packages/examples/cvat/recording-oracle/src/chain/escrow.py @@ -38,9 +38,8 @@ def validate_escrow( ) ) - if status == Status.Pending and not allow_no_funds: - if int(escrow.balance) == 0: - raise ValueError("Escrow doesn't have funds") + if status == Status.Pending and not allow_no_funds and int(escrow.balance) == 0: + raise ValueError("Escrow doesn't have funds") def get_escrow_manifest(chain_id: int, escrow_address: str) -> dict: From 88e5d06f6961a359b55b99ebc28a5c991f7f3922 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:11 +0200 Subject: [PATCH 32/44] [Recording Oracle] Unsafe autofix SIM108: if-else-block-instead-of-if-exp ## What it does Check for `if`-`else`-blocks that can be replaced with a ternary operator. ## Why is this bad? `if`-`else`-blocks that assign a value to a variable in both branches can be expressed more concisely by using a ternary operator. ## Example ```python if foo: bar = x else: bar = y ``` Use instead: ```python bar = x if foo else y ``` ## References - [Python documentation: Conditional expressions](https://docs.python.org/3/reference/expressions.html#conditional-expressions) --- packages/examples/cvat/recording-oracle/src/db/utils.py | 5 +---- .../examples/cvat/recording-oracle/src/services/cloud/s3.py | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/db/utils.py b/packages/examples/cvat/recording-oracle/src/db/utils.py index dedb7d08cb..77651fa42a 100644 --- a/packages/examples/cvat/recording-oracle/src/db/utils.py +++ b/packages/examples/cvat/recording-oracle/src/db/utils.py @@ -18,10 +18,7 @@ def maybe_for_update(query: T, enable: bool | ForUpdateParams) -> T: if not enable: return query - if isinstance(enable, ForUpdateParams): - params = enable - else: - params = ForUpdateParams() + params = enable if isinstance(enable, ForUpdateParams) else ForUpdateParams() return query.with_for_update( skip_locked=params.skip_locked, diff --git a/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py b/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py index ac229c5e9a..ccb5557e7d 100644 --- a/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py +++ b/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py @@ -61,8 +61,5 @@ def download_file(self, key: str, *, bucket: str | None = None) -> bytes: def list_files(self, *, bucket: str | None = None, prefix: str | None = None) -> list[str]: bucket = unquote(bucket) if bucket else self._bucket objects = self.resource.Bucket(bucket).objects - if prefix: - objects = objects.filter(Prefix=self.normalize_prefix(prefix)) - else: - objects = objects.all() + objects = objects.filter(Prefix=self.normalize_prefix(prefix)) if prefix else objects.all() return [file_info.key for file_info in objects] From 14c92afe4511cde4393713eb08066aa92cb7f5d9 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:12 +0200 Subject: [PATCH 33/44] [Recording Oracle] Unsafe autofix TCH001: typing-only-first-party-import ## What it does Checks for first-party imports that are only used for type annotations, but aren't defined in a type-checking block. ## Why is this bad? Unused imports add a performance overhead at runtime, and risk creating import cycles. If an import is _only_ used in typing-only contexts, it can instead be imported conditionally under an `if TYPE_CHECKING:` block to minimize runtime overhead. If [`lint.flake8-type-checking.quote-annotations`] is set to `true`, annotations will be wrapped in quotes if doing so would enable the corresponding import to be moved into an `if TYPE_CHECKING:` block. If a class _requires_ that type annotations be available at runtime (as is the case for Pydantic, SQLAlchemy, and other libraries), consider using the [`lint.flake8-type-checking.runtime-evaluated-base-classes`] and [`lint.flake8-type-checking.runtime-evaluated-decorators`] settings to mark them as such. ## Example ```python from __future__ import annotations import local_module def func(sized: local_module.Container) -> int: return len(sized) ``` Use instead: ```python from __future__ import annotations from typing import TYPE_CHECKING if TYPE_CHECKING: import local_module def func(sized: local_module.Container) -> int: return len(sized) ``` ## Options - `lint.flake8-type-checking.quote-annotations` - `lint.flake8-type-checking.runtime-evaluated-base-classes` - `lint.flake8-type-checking.runtime-evaluated-decorators` - `lint.typing-modules` ## References - [PEP 536](https://peps.python.org/pep-0563/#runtime-annotation-resolution-and-type-checking) --- .../src/handlers/process_intermediate_results.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py index e71adc8db6..9e0a9399c2 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, field from pathlib import Path from tempfile import TemporaryDirectory -from typing import NamedTuple, TypeVar +from typing import TYPE_CHECKING, NamedTuple, TypeVar import datumaro as dm import numpy as np @@ -18,7 +18,6 @@ import src.services.validation as db_service from src.core.annotation_meta import AnnotationMeta from src.core.config import Config -from src.core.manifest import TaskManifest from src.core.storage import compose_data_bucket_filename from src.core.types import TaskTypes from src.core.validation_errors import DatasetValidationError, LowAccuracyError @@ -37,6 +36,9 @@ TooFewGtError, ) +if TYPE_CHECKING: + from src.core.manifest import TaskManifest + DM_DATASET_FORMAT_MAPPING = { TaskTypes.image_label_binary: "cvat_images", TaskTypes.image_points: "coco_person_keypoints", From 2932f1b4de695196b48f4a30b60d7021c626c3e3 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:12 +0200 Subject: [PATCH 34/44] [Recording Oracle] Unsafe autofix TCH002: typing-only-third-party-import ## What it does Checks for third-party imports that are only used for type annotations, but aren't defined in a type-checking block. ## Why is this bad? Unused imports add a performance overhead at runtime, and risk creating import cycles. If an import is _only_ used in typing-only contexts, it can instead be imported conditionally under an `if TYPE_CHECKING:` block to minimize runtime overhead. If [`lint.flake8-type-checking.quote-annotations`] is set to `true`, annotations will be wrapped in quotes if doing so would enable the corresponding import to be moved into an `if TYPE_CHECKING:` block. If a class _requires_ that type annotations be available at runtime (as is the case for Pydantic, SQLAlchemy, and other libraries), consider using the [`lint.flake8-type-checking.runtime-evaluated-base-classes`] and [`lint.flake8-type-checking.runtime-evaluated-decorators`] settings to mark them as such. ## Example ```python from __future__ import annotations import pandas as pd def func(df: pd.DataFrame) -> int: return len(df) ``` Use instead: ```python from __future__ import annotations from typing import TYPE_CHECKING if TYPE_CHECKING: import pandas as pd def func(df: pd.DataFrame) -> int: return len(df) ``` ## Options - `lint.flake8-type-checking.quote-annotations` - `lint.flake8-type-checking.runtime-evaluated-base-classes` - `lint.flake8-type-checking.runtime-evaluated-decorators` - `lint.typing-modules` ## References - [PEP 536](https://peps.python.org/pep-0563/#runtime-annotation-resolution-and-type-checking) --- .../src/handlers/process_intermediate_results.py | 3 ++- .../recording-oracle/src/validation/dataset_comparison.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py index 9e0a9399c2..84747ad42e 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py @@ -10,7 +10,6 @@ import datumaro as dm import numpy as np -from sqlalchemy.orm import Session import src.core.tasks.boxes_from_points as boxes_from_points_task import src.core.tasks.simple as simple_task @@ -37,6 +36,8 @@ ) if TYPE_CHECKING: + from sqlalchemy.orm import Session + from src.core.manifest import TaskManifest DM_DATASET_FORMAT_MAPPING = { diff --git a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py index 810c080f04..c918ebc031 100644 --- a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py +++ b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py @@ -3,11 +3,11 @@ import itertools from abc import ABCMeta, abstractmethod from collections.abc import Callable, Sequence +from typing import TYPE_CHECKING import datumaro as dm import numpy as np from attrs import define, field -from datumaro.util.annotation_util import BboxCoords from src.core.config import Config from src.core.validation_errors import TooFewGtError @@ -21,6 +21,9 @@ point_to_bbox_cmp, ) +if TYPE_CHECKING: + from datumaro.util.annotation_util import BboxCoords + class SimilarityFunction(metaclass=ABCMeta): "A function to compute similarity between 2 annotations" From fb8df29835782b7f95f9b05f0fa116a838bc25e8 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:13 +0200 Subject: [PATCH 35/44] [Recording Oracle] Unsafe autofix TCH003: typing-only-standard-library-import ## What it does Checks for standard library imports that are only used for type annotations, but aren't defined in a type-checking block. ## Why is this bad? Unused imports add a performance overhead at runtime, and risk creating import cycles. If an import is _only_ used in typing-only contexts, it can instead be imported conditionally under an `if TYPE_CHECKING:` block to minimize runtime overhead. If [`lint.flake8-type-checking.quote-annotations`] is set to `true`, annotations will be wrapped in quotes if doing so would enable the corresponding import to be moved into an `if TYPE_CHECKING:` block. If a class _requires_ that type annotations be available at runtime (as is the case for Pydantic, SQLAlchemy, and other libraries), consider using the [`lint.flake8-type-checking.runtime-evaluated-base-classes`] and [`lint.flake8-type-checking.runtime-evaluated-decorators`] settings to mark them as such. ## Example ```python from __future__ import annotations from pathlib import Path def func(path: Path) -> str: return str(path) ``` Use instead: ```python from __future__ import annotations from typing import TYPE_CHECKING if TYPE_CHECKING: from pathlib import Path def func(path: Path) -> str: return str(path) ``` ## Options - `lint.flake8-type-checking.quote-annotations` - `lint.flake8-type-checking.runtime-evaluated-base-classes` - `lint.flake8-type-checking.runtime-evaluated-decorators` - `lint.typing-modules` ## References - [PEP 536](https://peps.python.org/pep-0563/#runtime-annotation-resolution-and-type-checking) --- .../cvat/recording-oracle/src/validation/dataset_comparison.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py index c918ebc031..add6157a8c 100644 --- a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py +++ b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py @@ -2,7 +2,6 @@ import itertools from abc import ABCMeta, abstractmethod -from collections.abc import Callable, Sequence from typing import TYPE_CHECKING import datumaro as dm @@ -22,6 +21,8 @@ ) if TYPE_CHECKING: + from collections.abc import Callable, Sequence + from datumaro.util.annotation_util import BboxCoords From 0c55b13826f9957532161fb29de31d59786d1cd2 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:20 +0200 Subject: [PATCH 36/44] [Recording Oracle] Autofix SIM300: yoda-conditions ## What it does Checks for conditions that position a constant on the left-hand side of the comparison operator, rather than the right-hand side. ## Why is this bad? These conditions (sometimes referred to as "Yoda conditions") are less readable than conditions that place the variable on the left-hand side of the comparison operator. In some languages, Yoda conditions are used to prevent accidental assignment in conditions (i.e., accidental uses of the `=` operator, instead of the `==` operator). However, Python does not allow assignments in conditions unless using the `:=` operator, so Yoda conditions provide no benefit in this regard. ## Example ```python if "Foo" == foo: ... ``` Use instead: ```python if foo == "Foo": ... ``` ## References - [Python documentation: Comparisons](https://docs.python.org/3/reference/expressions.html#comparisons) - [Python documentation: Assignment statements](https://docs.python.org/3/reference/simple_stmts.html#assignment-statements) --- .../recording-oracle/tests/integration/chain/test_escrow.py | 4 ++-- .../tests/integration/chain/test_kvstore.py | 2 +- .../recording-oracle/tests/integration/chain/test_web3.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py index 55441c6c06..de1997e019 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py @@ -147,7 +147,7 @@ def test_store_results_invalid_url(self): mock_function.return_value = self.w3 with pytest.raises(EscrowClientError) as error: store_results(self.w3.eth.chain_id, escrow_address, "invalid_url", DEFAULT_HASH) - assert "Invalid URL: invalid_url" == str(error.exception) + assert str(error.exception) == "Invalid URL: invalid_url" def test_store_results_invalid_hash(self): escrow_address = create_escrow(self.w3) @@ -155,7 +155,7 @@ def test_store_results_invalid_hash(self): mock_function.return_value = self.w3 with pytest.raises(EscrowClientError) as error: store_results(self.w3.eth.chain_id, escrow_address, DEFAULT_MANIFEST_URL, "") - assert "Invalid empty hash" == str(error.exception) + assert str(error.exception) == "Invalid empty hash" def test_get_reputation_oracle_address(self): escrow_address = create_escrow(self.w3) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py index d00ceedfc5..e7435e53f9 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py @@ -86,7 +86,7 @@ def test_get_role_by_address_invalid_escrow(self): mock_function.return_value = self.w3 with pytest.raises(KVStoreClientError) as error: get_role_by_address(self.w3.eth.chain_id, "invalid_address") - assert "Invalid address: invalid_address" == str(error.exception) + assert str(error.exception) == "Invalid address: invalid_address" def test_get_role_by_address_invalid_address(self): create_escrow(self.w3) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py index da88473ae2..1933e6d3a5 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py @@ -57,7 +57,7 @@ def test_get_web3_localhost(self): def test_get_web3_invalid_chain_id(self): with pytest.raises(ValueError) as error: get_web3(1234) - assert "1234 is not in available list of networks." == str(error.exception) + assert str(error.exception) == "1234 is not in available list of networks." def test_sign_message_polygon(self): with ( @@ -86,7 +86,7 @@ def test_sign_message_amoy(self): def test_sign_message_invalid_chain_id(self): with pytest.raises(ValueError) as error: sign_message(1234, "message") - assert "1234 is not in available list of networks." == str(error.exception) + assert str(error.exception) == "1234 is not in available list of networks." def test_recover_signer(self): with patch("src.chain.web3.get_web3") as mock_function: @@ -107,4 +107,4 @@ def test_validate_address(self): def test_validate_address_invalid_address(self): with pytest.raises(ValueError) as error: validate_address("invalid_address") - assert "invalid_address is not a correct Web3 address" == str(error.exception) + assert str(error.exception) == "invalid_address is not a correct Web3 address" From f87d955ced0646fc1041fa25bdc6c3291f7bbf27 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:31 +0200 Subject: [PATCH 37/44] [Recording Oracle] Unsafe autofix RUF013: implicit-optional ## What it does Checks for the use of implicit `Optional` in type annotations when the default parameter value is `None`. ## Why is this bad? Implicit `Optional` is prohibited by [PEP 484]. It is confusing and inconsistent with the rest of the type system. It's recommended to use `Optional[T]` instead. For Python 3.10 and later, you can also use `T | None`. ## Example ```python def foo(arg: int = None): pass ``` Use instead: ```python from typing import Optional def foo(arg: Optional[int] = None): pass ``` Or, for Python 3.10 and later: ```python def foo(arg: int | None = None): pass ``` If you want to use the `|` operator in Python 3.9 and earlier, you can use future imports: ```python from __future__ import annotations def foo(arg: int | None = None): pass ``` ## Limitations Type aliases are not supported and could result in false negatives. For example, the following code will not be flagged: ```python Text = str | bytes def foo(arg: Text = None): pass ``` ## Options - `target-version` [PEP 484]: https://peps.python.org/pep-0484/#union-types --- packages/examples/cvat/recording-oracle/src/chain/escrow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/examples/cvat/recording-oracle/src/chain/escrow.py b/packages/examples/cvat/recording-oracle/src/chain/escrow.py index 50e136bf93..5651f21e3c 100644 --- a/packages/examples/cvat/recording-oracle/src/chain/escrow.py +++ b/packages/examples/cvat/recording-oracle/src/chain/escrow.py @@ -21,7 +21,7 @@ def validate_escrow( chain_id: int, escrow_address: str, *, - accepted_states: list[Status] = None, + accepted_states: list[Status] | None = None, allow_no_funds: bool = False, ) -> None: if accepted_states is None: From 3bca1494ba0b1210e61c2715e98b0731e5f72c2d Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sat, 17 Aug 2024 19:14:32 +0200 Subject: [PATCH 38/44] [Recording Oracle] Unsafe autofix UP038: non-pep604-isinstance ## What it does Checks for uses of `isinstance` and `issubclass` that take a tuple of types for comparison. ## Why is this bad? Since Python 3.10, `isinstance` and `issubclass` can be passed a `|`-separated union of types, which is consistent with the union operator introduced in [PEP 604]. Note that this results in slower code. Ignore this rule if the performance of an `isinstance` or `issubclass` check is a concern, e.g., in a hot loop. ## Example ```python isinstance(x, (int, float)) ``` Use instead: ```python isinstance(x, int | float) ``` ## Options - `target-version` ## References - [Python documentation: `isinstance`](https://docs.python.org/3/library/functions.html#isinstance) - [Python documentation: `issubclass`](https://docs.python.org/3/library/functions.html#issubclass) [PEP 604]: https://peps.python.org/pep-0604/ --- .../examples/cvat/recording-oracle/src/utils/annotations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/examples/cvat/recording-oracle/src/utils/annotations.py b/packages/examples/cvat/recording-oracle/src/utils/annotations.py index 2e897b3b93..fca0e874f3 100644 --- a/packages/examples/cvat/recording-oracle/src/utils/annotations.py +++ b/packages/examples/cvat/recording-oracle/src/utils/annotations.py @@ -99,7 +99,7 @@ def __init__( dst_label_cat = dm.LabelCategories(attributes=deepcopy(src_label_cat.attributes)) for dst_label in dst_labels: - assert isinstance(dst_label, (str, tuple)) + assert isinstance(dst_label, str | tuple) dst_parent = "" if isinstance(dst_label, tuple): From ba08199f657e0bdb18e0590316798eebb9e809bb Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sun, 18 Aug 2024 13:16:54 +0200 Subject: [PATCH 39/44] [Recording Oracle] Use pytest.raises with match argument --- .../tests/integration/chain/test_escrow.py | 6 ++---- .../tests/integration/chain/test_kvstore.py | 3 +-- .../tests/integration/chain/test_web3.py | 9 +++------ 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py index de1997e019..a72534a1ab 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py @@ -145,17 +145,15 @@ def test_store_results_invalid_url(self): escrow_address = create_escrow(self.w3) with patch("src.chain.escrow.get_web3") as mock_function: mock_function.return_value = self.w3 - with pytest.raises(EscrowClientError) as error: + with pytest.raises(EscrowClientError, match="Invalid URL: invalid_url"): store_results(self.w3.eth.chain_id, escrow_address, "invalid_url", DEFAULT_HASH) - assert str(error.exception) == "Invalid URL: invalid_url" def test_store_results_invalid_hash(self): escrow_address = create_escrow(self.w3) with patch("src.chain.escrow.get_web3") as mock_function: mock_function.return_value = self.w3 - with pytest.raises(EscrowClientError) as error: + with pytest.raises(EscrowClientError, match="Invalid empty hash"): store_results(self.w3.eth.chain_id, escrow_address, DEFAULT_MANIFEST_URL, "") - assert str(error.exception) == "Invalid empty hash" def test_get_reputation_oracle_address(self): escrow_address = create_escrow(self.w3) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py index e7435e53f9..08c7a8584b 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py @@ -84,9 +84,8 @@ def test_get_role_by_address(self): def test_get_role_by_address_invalid_escrow(self): with patch("src.chain.kvstore.get_web3") as mock_function: mock_function.return_value = self.w3 - with pytest.raises(KVStoreClientError) as error: + with pytest.raises(KVStoreClientError, match="Invalid address: invalid_address"): get_role_by_address(self.w3.eth.chain_id, "invalid_address") - assert str(error.exception) == "Invalid address: invalid_address" def test_get_role_by_address_invalid_address(self): create_escrow(self.w3) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py index 1933e6d3a5..c6a97fd696 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_web3.py @@ -55,9 +55,8 @@ def test_get_web3_localhost(self): assert w3.manager._provider.endpoint_uri == LocalhostConfig.rpc_api def test_get_web3_invalid_chain_id(self): - with pytest.raises(ValueError) as error: + with pytest.raises(ValueError, match="1234 is not in available list of networks."): get_web3(1234) - assert str(error.exception) == "1234 is not in available list of networks." def test_sign_message_polygon(self): with ( @@ -84,9 +83,8 @@ def test_sign_message_amoy(self): assert signed_message == SIGNATURE def test_sign_message_invalid_chain_id(self): - with pytest.raises(ValueError) as error: + with pytest.raises(ValueError, match="1234 is not in available list of networks."): sign_message(1234, "message") - assert str(error.exception) == "1234 is not in available list of networks." def test_recover_signer(self): with patch("src.chain.web3.get_web3") as mock_function: @@ -105,6 +103,5 @@ def test_validate_address(self): assert address == DEFAULT_GAS_PAYER def test_validate_address_invalid_address(self): - with pytest.raises(ValueError) as error: + with pytest.raises(ValueError, match="invalid_address is not a correct Web3 address"): validate_address("invalid_address") - assert str(error.exception) == "invalid_address is not a correct Web3 address" From 1aaedfd4aba93839917315980268e4f2dc3623d1 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sun, 18 Aug 2024 13:31:56 +0200 Subject: [PATCH 40/44] [Recording Oracle] Rewrite the last self.assert... --- .../recording-oracle/tests/integration/chain/test_escrow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py index a72534a1ab..2bcb156c87 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py @@ -128,7 +128,7 @@ def test_get_encrypted_escrow_manifest(self): downloaded_manifest_content = get_escrow_manifest( self.network_config.chain_id, self.escrow_address ) - self.assertDictEqual(downloaded_manifest_content, original_manifest_content) + assert downloaded_manifest_content == original_manifest_content def test_store_results(self): escrow_address = create_escrow(self.w3) From 97703acca57d039c61db410ac7345f941567b77c Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sun, 18 Aug 2024 13:46:44 +0200 Subject: [PATCH 41/44] [Recording Oracle] Add missing __init__.py modules --- packages/examples/cvat/recording-oracle/pyproject.toml | 2 ++ packages/examples/cvat/recording-oracle/src/core/__init__.py | 0 .../examples/cvat/recording-oracle/src/handlers/__init__.py | 0 packages/examples/cvat/recording-oracle/src/models/__init__.py | 0 .../examples/cvat/recording-oracle/src/validation/__init__.py | 0 5 files changed, 2 insertions(+) create mode 100644 packages/examples/cvat/recording-oracle/src/core/__init__.py create mode 100644 packages/examples/cvat/recording-oracle/src/handlers/__init__.py create mode 100644 packages/examples/cvat/recording-oracle/src/models/__init__.py create mode 100644 packages/examples/cvat/recording-oracle/src/validation/__init__.py diff --git a/packages/examples/cvat/recording-oracle/pyproject.toml b/packages/examples/cvat/recording-oracle/pyproject.toml index 82fbbf1b4f..bedc8627f9 100644 --- a/packages/examples/cvat/recording-oracle/pyproject.toml +++ b/packages/examples/cvat/recording-oracle/pyproject.toml @@ -87,6 +87,8 @@ ignore = [ "S", # security "DTZ005", # allow datetimes without timezones ] +# alembic is not a package in a traditional sense, so putting __init__.py there doesn't make sense +"alembic/*" = ["INP001"] [tool.ruff.lint.pep8-naming] classmethod-decorators = [ diff --git a/packages/examples/cvat/recording-oracle/src/core/__init__.py b/packages/examples/cvat/recording-oracle/src/core/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/examples/cvat/recording-oracle/src/handlers/__init__.py b/packages/examples/cvat/recording-oracle/src/handlers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/examples/cvat/recording-oracle/src/models/__init__.py b/packages/examples/cvat/recording-oracle/src/models/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/examples/cvat/recording-oracle/src/validation/__init__.py b/packages/examples/cvat/recording-oracle/src/validation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From 0022ab1cc7b5b1d1dde3dd09a85b43d25879596d Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Sun, 18 Aug 2024 13:49:02 +0200 Subject: [PATCH 42/44] [Recording Oracle] Unsafe autofix TID252: relative-imports ## What it does Checks for relative imports. ## Why is this bad? Absolute imports, or relative imports from siblings, are recommended by [PEP 8]: > Absolute imports are recommended, as they are usually more readable and tend to be better behaved... > ```python > import mypkg.sibling > from mypkg import sibling > from mypkg.sibling import example > ``` > However, explicit relative imports are an acceptable alternative to absolute imports, > especially when dealing with complex package layouts where using absolute imports would be > unnecessarily verbose: > ```python > from . import sibling > from .sibling import example > ``` ## Example ```python from .. import foo ``` Use instead: ```python from mypkg import foo ``` ## Options - `lint.flake8-tidy-imports.ban-relative-imports` [PEP 8]: https://peps.python.org/pep-0008/#imports --- .../cvat/recording-oracle/src/validation/dataset_comparison.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py index add6157a8c..a4ce1059dc 100644 --- a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py +++ b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py @@ -10,8 +10,7 @@ from src.core.config import Config from src.core.validation_errors import TooFewGtError - -from .annotation_matching import ( +from src.validation.annotation_matching import ( Bbox, MatchResult, Point, From 55146477f811e1d390526d7ed063e3669af7eec7 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Mon, 19 Aug 2024 13:10:08 +0200 Subject: [PATCH 43/44] [Recording Oracle] Fix/ignore remaining errors --- .../cvat/recording-oracle/alembic/env.py | 2 +- .../cvat/recording-oracle/pyproject.toml | 23 ++++++++++++++++++ .../examples/cvat/recording-oracle/run.py | 2 +- .../cvat/recording-oracle/src/core/config.py | 22 +++++++++-------- .../src/core/oracle_events.py | 5 +--- .../handlers/process_intermediate_results.py | 24 ++++++++++++------- .../src/handlers/validation.py | 4 ++-- .../recording-oracle/src/schemas/__init__.py | 3 --- .../recording-oracle/src/services/cloud/s3.py | 3 +-- .../src/services/cloud/types.py | 15 +++++------- .../recording-oracle/src/services/webhook.py | 2 +- .../recording-oracle/src/utils/annotations.py | 7 +++--- .../cvat/recording-oracle/src/utils/net.py | 2 +- .../recording-oracle/src/utils/webhooks.py | 2 +- .../src/validation/dataset_comparison.py | 7 +++--- .../services/cloud/test_client_service.py | 4 ++-- .../services/test_webhook_service.py | 2 +- 17 files changed, 77 insertions(+), 52 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/alembic/env.py b/packages/examples/cvat/recording-oracle/alembic/env.py index e810187f35..1e85b134f0 100644 --- a/packages/examples/cvat/recording-oracle/alembic/env.py +++ b/packages/examples/cvat/recording-oracle/alembic/env.py @@ -16,7 +16,7 @@ if config.config_file_name is not None: fileConfig(config.config_file_name) -from src.db import Base +from src.db import Base # noqa: E402 # add your model's MetaData object here # for 'autogenerate' support diff --git a/packages/examples/cvat/recording-oracle/pyproject.toml b/packages/examples/cvat/recording-oracle/pyproject.toml index bedc8627f9..b6bfcb956e 100644 --- a/packages/examples/cvat/recording-oracle/pyproject.toml +++ b/packages/examples/cvat/recording-oracle/pyproject.toml @@ -71,6 +71,29 @@ ignore = [ "G001", # Forbid str.format for logging "PTH123", # Checks for uses of `os.path.splitext` "D", # Docstrings + "N806", # Variable in function should be lowercase + "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar` + "SLF001", # Private member accessed + "F811", # Redefinition of unused + "RUF005", # Consider iterable unpacking instead of concatenation + "A002", # Argument is shadowing a Python builtin + "N818", # Exception name should be named with an Error suffix + "TRY002", # Create your own exception + "ANN003", # Missing type annotation for `**kwargs` + "ANN204", # Missing return type annotation for special method + "ERA001", # Found commented-out code + "N801", # Class name should use CapWords convention + "PLR0915", # Too many statements + "F401", # Imported but unused + "PLR2004", # Magic value used in comparison, consider replacing with a constant variable + "ANN002", # Missing type annotation for `*args` + "TRY300", # Consider moving this statement to an `else` block + "C901", # Function is too complex + "PLW2901", # Variable overwritten by assignment target + "PTH118", # Prefer pathlib instead of os.path + "PTH119", # `os.path.basename()` should be replaced by `Path.name` + "PTH122", # `os.path.splitext()` should be replaced by `Path.suffix`, `Path.stem`, and `Path.parent` + "PTH207", # Replace `glob` with `Path.glob` or `Path.rglob` ] diff --git a/packages/examples/cvat/recording-oracle/run.py b/packages/examples/cvat/recording-oracle/run.py index 5b59b70a51..2f09b1a09f 100644 --- a/packages/examples/cvat/recording-oracle/run.py +++ b/packages/examples/cvat/recording-oracle/run.py @@ -11,7 +11,7 @@ uvicorn.run( app="src:app", - host="0.0.0.0", + host="0.0.0.0", # noqa: S104 port=int(Config.port), workers=Config.workers_amount, reload=is_dev, diff --git a/packages/examples/cvat/recording-oracle/src/core/config.py b/packages/examples/cvat/recording-oracle/src/core/config.py index c3dae16ebf..db5534ac4c 100644 --- a/packages/examples/cvat/recording-oracle/src/core/config.py +++ b/packages/examples/cvat/recording-oracle/src/core/config.py @@ -16,7 +16,7 @@ from src.utils.net import is_ipv4 dotenv_path = os.getenv("DOTENV_PATH", None) -if dotenv_path and not os.path.exists(dotenv_path): +if dotenv_path and not os.path.exists(dotenv_path): # noqa: PTH110 raise FileNotFoundError(dotenv_path) load_dotenv(dotenv_path) @@ -30,7 +30,7 @@ def validate(cls) -> None: class Postgres: port = os.environ.get("PG_PORT", "5434") - host = os.environ.get("PG_HOST", "0.0.0.0") + host = os.environ.get("PG_HOST", "0.0.0.0") # noqa: S104 user = os.environ.get("PG_USER", "admin") password = os.environ.get("PG_PASSWORD", "admin") database = os.environ.get("PG_DB", "recording_oracle") @@ -122,8 +122,7 @@ def provider_endpoint_url(cls) -> str: def bucket_url(cls) -> str: if is_ipv4(cls.endpoint_url): return f"{cls.get_scheme()}{cls.endpoint_url}/{cls.data_bucket_name}/" - else: - return f"{cls.get_scheme()}{cls.data_bucket_name}.{cls.endpoint_url}/" + return f"{cls.get_scheme()}{cls.data_bucket_name}.{cls.endpoint_url}/" class StorageConfig(IStorageConfig): @@ -194,7 +193,7 @@ class ValidationConfig: Each such job will be accepted "blindly", as we can't validate the annotations. """ - max_escrow_iterations = int(os.getenv("MAX_ESCROW_ITERATIONS", 0)) + max_escrow_iterations = int(os.getenv("MAX_ESCROW_ITERATIONS", "0")) """ Maximum escrow annotation-validation iterations. After this, the escrow is finished automatically. @@ -217,7 +216,7 @@ def validate(cls) -> None: if cls.pgp_private_key: try: Encryption(cls.pgp_private_key, passphrase=cls.pgp_passphrase) - except Exception as ex: + except Exception as ex: # noqa: BLE001 # Possible reasons: # - private key is invalid # - private key is locked but no passphrase is provided @@ -253,9 +252,12 @@ def validate(cls) -> None: attr_or_method.validate() @classmethod - def get_network_configs(cls, only_configured: bool = True) -> Iterable[_NetworkConfig]: + def get_network_configs(cls, *, only_configured: bool = True) -> Iterable[_NetworkConfig]: for attr_or_method in cls.__dict__: attr_or_method = getattr(cls, attr_or_method) - if inspect.isclass(attr_or_method) and issubclass(attr_or_method, _NetworkConfig): - if not only_configured or attr_or_method.is_configured(): - yield attr_or_method + if ( + inspect.isclass(attr_or_method) + and issubclass(attr_or_method, _NetworkConfig) + and (not only_configured or attr_or_method.is_configured()) + ): + yield attr_or_method diff --git a/packages/examples/cvat/recording-oracle/src/core/oracle_events.py b/packages/examples/cvat/recording-oracle/src/core/oracle_events.py index 7ef349cd55..04ce7c49db 100644 --- a/packages/examples/cvat/recording-oracle/src/core/oracle_events.py +++ b/packages/examples/cvat/recording-oracle/src/core/oracle_events.py @@ -4,10 +4,7 @@ from src.core.types import ExchangeOracleEventTypes, OracleWebhookTypes, RecordingOracleEventTypes -EventTypeTag = Union[ - ExchangeOracleEventTypes, - RecordingOracleEventTypes, -] +EventTypeTag = ExchangeOracleEventTypes | RecordingOracleEventTypes class OracleEvent(BaseModel): diff --git a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py index 84747ad42e..1a22b9f7e1 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py @@ -167,7 +167,7 @@ def _parse_gt(self): ) ) - def _load_job_dataset(self, job_id: int, job_dataset_path: Path) -> dm.Dataset: + def _load_job_dataset(self, job_id: int, job_dataset_path: Path) -> dm.Dataset: # noqa: ARG002 manifest = self._require_field(self.manifest) return dm.Dataset.import_from( @@ -353,7 +353,11 @@ def _get_gt_weights(self, *, job_cvat_id: int, job_gt_dataset: dm.Dataset) -> di return weights def _gt_key_to_sample_id( - self, gt_key: str, *, job_cvat_id: int, job_gt_dataset: dm.Dataset + self, + gt_key: str, + *, + job_cvat_id: int, # noqa: ARG002 + job_gt_dataset: dm.Dataset, # noqa: ARG002 ) -> str | None: return gt_key @@ -524,7 +528,7 @@ def _download_task_meta(self): return boxes_to_points_mapping, roi_filenames, rois, gt_dataset, points_dataset - def _make_gt_dataset_for_job(self, job_id: int, job_dataset: dm.Dataset) -> dm.Dataset: + def _make_gt_dataset_for_job(self, job_id: int, job_dataset: dm.Dataset) -> dm.Dataset: # noqa: ARG002 job_gt_dataset = dm.Dataset(categories=self._gt_dataset.categories(), media_type=dm.Image) for job_sample in job_dataset: @@ -734,7 +738,7 @@ def _load_job_dataset(self, job_id: int, job_dataset_path: Path) -> dm.Dataset: return updated_dataset - def _make_gt_dataset_for_job(self, job_id: int, job_dataset: dm.Dataset) -> dm.Dataset: + def _make_gt_dataset_for_job(self, job_id: int, job_dataset: dm.Dataset) -> dm.Dataset: # noqa: ARG002 job_label_cat: dm.LabelCategories = job_dataset.categories()[dm.AnnotationType.label] assert len(job_label_cat) == 2 job_skeleton_label_id, job_skeleton_label = next( @@ -835,8 +839,8 @@ class _LabelId(NamedTuple): def _get_gt_dataset_label_id(self, job_gt_dataset: dm.Dataset) -> _LabelId: label_cat: dm.LabelCategories = job_gt_dataset.categories()[dm.AnnotationType.label] assert len(label_cat) == 2 - job_skeleton_label = next(l for l in label_cat if not l.parent) - job_point_label = next(l for l in label_cat if l.parent) + job_skeleton_label = next(label for label in label_cat if not label.parent) + job_point_label = next(label for label in label_cat if label.parent) return self._LabelId( *next( @@ -849,7 +853,11 @@ def _get_gt_dataset_label_id(self, job_gt_dataset: dm.Dataset) -> _LabelId: ) def _gt_key_to_sample_id( - self, gt_key: str, *, job_cvat_id: int, job_gt_dataset: dm.Dataset + self, + gt_key: str, + *, + job_cvat_id: int, # noqa: ARG002 + job_gt_dataset: dm.Dataset, ) -> str | None: parsed_gt_key = self._parse_gt_key(gt_key) job_label_id = self._get_gt_dataset_label_id(job_gt_dataset) @@ -907,7 +915,7 @@ def _compute_gt_stats_update( return updated_gt_stats -def process_intermediate_results( +def process_intermediate_results( # noqa: PLR0912 session: Session, *, escrow_address: str, diff --git a/packages/examples/cvat/recording-oracle/src/handlers/validation.py b/packages/examples/cvat/recording-oracle/src/handlers/validation.py index 10a5b03e42..06830f02d8 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/validation.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/validation.py @@ -94,7 +94,7 @@ def _download_results(self): self._download_results_meta() self._download_annotations() - ValidationResult = Union[ValidationSuccess, ValidationFailure] + ValidationResult = ValidationSuccess | ValidationFailure def _process_annotation_results(self) -> ValidationResult: assert self.annotation_meta is not None @@ -159,7 +159,7 @@ def _handle_validation_result(self, validation_result: ValidationResult): escrow.store_results( chain_id, escrow_address, - Config.storage_config.bucket_url() + os.path.dirname(recor_merged_annotations_path), + Config.storage_config.bucket_url() + os.path.dirname(recor_merged_annotations_path), # noqa: PTH120 compute_resulting_annotations_hash(validation_result.resulting_annotations), ) diff --git a/packages/examples/cvat/recording-oracle/src/schemas/__init__.py b/packages/examples/cvat/recording-oracle/src/schemas/__init__.py index 787e0573bf..e26cbee23e 100644 --- a/packages/examples/cvat/recording-oracle/src/schemas/__init__.py +++ b/packages/examples/cvat/recording-oracle/src/schemas/__init__.py @@ -1,8 +1,5 @@ -# pylint: disable=too-few-public-methods """Schema for API input&output""" -from typing import List, Optional - from pydantic import BaseModel diff --git a/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py b/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py index ccb5557e7d..19b570216a 100644 --- a/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py +++ b/packages/examples/cvat/recording-oracle/src/services/cloud/s3.py @@ -49,8 +49,7 @@ def file_exists(self, key: str, *, bucket: str | None = None) -> bool: except ClientError as e: if e.response["Error"]["Code"] == "404": return False - else: - raise + raise def download_file(self, key: str, *, bucket: str | None = None) -> bytes: bucket = unquote(bucket) if bucket else self._bucket diff --git a/packages/examples/cvat/recording-oracle/src/services/cloud/types.py b/packages/examples/cvat/recording-oracle/src/services/cloud/types.py index c41e7c0de8..8a4c318966 100644 --- a/packages/examples/cvat/recording-oracle/src/services/cloud/types.py +++ b/packages/examples/cvat/recording-oracle/src/services/cloud/types.py @@ -45,9 +45,7 @@ def from_storage_config(cls, config: type[IStorageConfig]) -> BucketCredentials "Invalid storage configuration. The access_key/secret_key pair" f"cannot be specified with {config.provider} provider" ) - elif ( - bool(config.access_key) ^ bool(config.secret_key) - ) and config.provider.lower() == "aws": + if (bool(config.access_key) ^ bool(config.secret_key)) and config.provider.lower() == "aws": raise ValueError( "Invalid storage configuration. " "Either none or both access_key and secret_key must be specified for an AWS storage" @@ -99,7 +97,7 @@ def from_url(cls, url: str) -> BucketAccessInfo: bucket_name=parsed_url.netloc.split(".")[0], path=parsed_url.path.lstrip("/"), ) - elif parsed_url.netloc.endswith(DEFAULT_GCS_HOST): + if parsed_url.netloc.endswith(DEFAULT_GCS_HOST): # Google Cloud Storage (GCS) bucket # Virtual hosted-style is expected: # https://BUCKET_NAME.storage.googleapis.com/OBJECT_NAME @@ -109,7 +107,7 @@ def from_url(cls, url: str) -> BucketAccessInfo: host_url=f"{parsed_url.scheme}://{DEFAULT_GCS_HOST}", path=parsed_url.path.lstrip("/"), ) - elif Config.features.enable_custom_cloud_host: + if Config.features.enable_custom_cloud_host: if is_ipv4(parsed_url.netloc): host = parsed_url.netloc bucket_name, path = parsed_url.path.lstrip("/").split("/", maxsplit=1) @@ -124,8 +122,7 @@ def from_url(cls, url: str) -> BucketAccessInfo: bucket_name=bucket_name, path=path, ) - else: - raise ValueError(f"{parsed_url.netloc} cloud provider is not supported.") + raise ValueError(f"{parsed_url.netloc} cloud provider is not supported.") @classmethod def _from_dict(cls, data: dict) -> BucketAccessInfo: @@ -176,9 +173,9 @@ def from_bucket_url(cls, bucket_url: manifest.BucketUrl) -> BucketAccessInfo: def parse_obj(cls, data: str | type[IStorageConfig] | manifest.BucketUrl) -> BucketAccessInfo: if isinstance(data, manifest.BucketUrlBase): return cls.from_bucket_url(data) - elif isinstance(data, str): + if isinstance(data, str): return cls.from_url(data) - elif isclass(data) and issubclass(data, IStorageConfig): + if isclass(data) and issubclass(data, IStorageConfig): return cls.from_storage_config(data) raise TypeError(f"Unsupported data type ({type(data)}) was provided") diff --git a/packages/examples/cvat/recording-oracle/src/services/webhook.py b/packages/examples/cvat/recording-oracle/src/services/webhook.py index 6a2623b184..41e9ca1c10 100644 --- a/packages/examples/cvat/recording-oracle/src/services/webhook.py +++ b/packages/examples/cvat/recording-oracle/src/services/webhook.py @@ -59,7 +59,7 @@ def create_webhook( if self.direction == OracleWebhookDirectionTags.incoming and not signature: raise ValueError("Webhook signature must be specified for incoming events") - elif self.direction == OracleWebhookDirectionTags.outgoing and signature: + if self.direction == OracleWebhookDirectionTags.outgoing and signature: raise ValueError("Webhook signature must not be specified for outgoing events") if signature: diff --git a/packages/examples/cvat/recording-oracle/src/utils/annotations.py b/packages/examples/cvat/recording-oracle/src/utils/annotations.py index fca0e874f3..ea138a703b 100644 --- a/packages/examples/cvat/recording-oracle/src/utils/annotations.py +++ b/packages/examples/cvat/recording-oracle/src/utils/annotations.py @@ -1,3 +1,4 @@ +from argparse import ArgumentParser from collections.abc import Iterable from copy import deepcopy @@ -38,7 +39,7 @@ def shift_ann( ] ) else: - raise AssertionError(f"Unsupported annotation type '{ann.type}'") + raise TypeError(f"Unsupported annotation type '{ann.type}'") return shifted_ann @@ -65,7 +66,7 @@ class ProjectLabels(dm.ItemTransform): """ @classmethod - def build_cmdline_parser(cls, **kwargs): + def build_cmdline_parser(cls, **kwargs) -> ArgumentParser: parser = super().build_cmdline_parser(**kwargs) parser.add_argument( "-l", @@ -76,7 +77,7 @@ def build_cmdline_parser(cls, **kwargs): ) return parser - def __init__( + def __init__( # noqa: PLR0912 self, extractor: dm.IExtractor, dst_labels: Iterable[str | tuple[str, str]] | dm.LabelCategories, diff --git a/packages/examples/cvat/recording-oracle/src/utils/net.py b/packages/examples/cvat/recording-oracle/src/utils/net.py index 360dafaebb..04ad799510 100644 --- a/packages/examples/cvat/recording-oracle/src/utils/net.py +++ b/packages/examples/cvat/recording-oracle/src/utils/net.py @@ -1,7 +1,7 @@ import ipaddress -def is_ipv4(addr: str, allow_port: bool = True) -> bool: +def is_ipv4(addr: str, *, allow_port: bool = True) -> bool: try: if allow_port: addr = addr.split(":", maxsplit=1)[0] diff --git a/packages/examples/cvat/recording-oracle/src/utils/webhooks.py b/packages/examples/cvat/recording-oracle/src/utils/webhooks.py index 97a74f7348..a1bf128887 100644 --- a/packages/examples/cvat/recording-oracle/src/utils/webhooks.py +++ b/packages/examples/cvat/recording-oracle/src/utils/webhooks.py @@ -28,7 +28,7 @@ def prepare_outgoing_webhook_body( def prepare_signed_message( - escrow_address: str, + escrow_address: str, # noqa: ARG001 chain_id: Networks, message: str | None = None, body: dict | None = None, diff --git a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py index a4ce1059dc..f8c87216dd 100644 --- a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py +++ b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py @@ -1,7 +1,7 @@ from __future__ import annotations import itertools -from abc import ABCMeta, abstractmethod +from abc import ABC, ABCMeta, abstractmethod from typing import TYPE_CHECKING import datumaro as dm @@ -28,6 +28,7 @@ class SimilarityFunction(metaclass=ABCMeta): "A function to compute similarity between 2 annotations" + @abstractmethod def __call__(self, gt_ann: dm.Annotation, ds_ann: dm.Annotation) -> float: ... @@ -354,12 +355,12 @@ def _compute_oks( return 0 if visibility_a is None: - visibility_a = np.full(len(p1), True) + visibility_a = np.full(len(p1), fill_value=True) else: visibility_a = np.asarray(visibility_a, dtype=bool) if visibility_b is None: - visibility_b = np.full(len(p2), True) + visibility_b = np.full(len(p2), fill_value=True) else: visibility_b = np.asarray(visibility_b, dtype=bool) diff --git a/packages/examples/cvat/recording-oracle/tests/integration/services/cloud/test_client_service.py b/packages/examples/cvat/recording-oracle/tests/integration/services/cloud/test_client_service.py index 19fabef7be..b26712d279 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/services/cloud/test_client_service.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/services/cloud/test_client_service.py @@ -68,7 +68,7 @@ def test_degenerate_file_operations(self): client.remove_file(invalid_file, bucket=self.bucket_name) def test_degenerate_client(self): - with pytest.raises(EndpointConnectionError): + with pytest.raises(EndpointConnectionError): # noqa: PT012 invalid_client = S3Client( endpoint_url="http://not.an.url:1234", access_key=self.access_key, @@ -76,5 +76,5 @@ def test_degenerate_client(self): ) invalid_client.create_file("test.txt", bucket=self.bucket_name) - with pytest.raises(ValueError): + with pytest.raises(ValueError): # noqa: PT011 S3Client(endpoint_url="nonsense-stuff") diff --git a/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py b/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py index 08012f66f1..e7f5d8b828 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/services/test_webhook_service.py @@ -56,7 +56,7 @@ def test_create_webhook(self): def _test_none_webhook_argument(self, argument_name, error_type): kwargs = dict(**self.webhook_kwargs) kwargs[argument_name] = None - with pytest.raises(error_type): + with pytest.raises(error_type): # noqa: PT012 inbox.create_webhook(**kwargs) self.session.commit() From 2ebb83dbdfbb21960a09dcdedad28355ad9bf9f8 Mon Sep 17 00:00:00 2001 From: Arseny Boykov <36469655+Bobronium@users.noreply.github.com> Date: Mon, 26 Aug 2024 11:06:33 +0200 Subject: [PATCH 44/44] [Recording Oracle] Apply changes from similar [Exchange Oracle] PR --- .../cvat/recording-oracle/pyproject.toml | 34 +++++++++++++------ .../src/core/oracle_events.py | 2 -- .../cvat/recording-oracle/src/core/storage.py | 5 ++- .../handlers/process_intermediate_results.py | 11 +++--- .../src/handlers/validation.py | 1 - .../src/validation/dataset_comparison.py | 2 +- .../tests/integration/chain/test_kvstore.py | 6 ++-- .../recording-oracle/tests/utils/constants.py | 10 ++++-- 8 files changed, 48 insertions(+), 23 deletions(-) diff --git a/packages/examples/cvat/recording-oracle/pyproject.toml b/packages/examples/cvat/recording-oracle/pyproject.toml index b6bfcb956e..2e504f6678 100644 --- a/packages/examples/cvat/recording-oracle/pyproject.toml +++ b/packages/examples/cvat/recording-oracle/pyproject.toml @@ -41,19 +41,34 @@ unfixable = [ "RUF005", # messes up concantenation with numpy structures ] ignore = [ - "ANN101", # method args annotations (mypy will take care of that) + "W191", # Rules conflicting with ruff format (https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules) + "E111", # | + "E114", # | + "E117", # | + "D206", # | + "D300", # | + "Q000", # | + "Q001", # | + "Q002", # | + "Q003", # | + "COM812", # | + "COM819", # | + "ISC001", # | + "ISC002", # | + "ANN101", # Method args annotations (mypy will take care of that) "ANN001", # | "ANN202", # | "ANN201", # | "ANN401", # | "ANN102", # | "RUF001", # Allow cyrillic letters in comments - "COM812", # Trailing comas are handled by ruff format - "ISC001", # Recommended to be disabled when using ruff format + "B904", # Raise from: modern pythons preserve previous exceptions - "EM", # Forbids using literal strings in exceptions. I think it's ok - "TRY003", # Forbids long messages in exceptions, again, it's ok - "G004", # Logging statement uses f-string — it's ok, f-strings are fast + "EM", # Forbids using literal strings in exceptions. + # Sujested way of dealing with exceptions increases verbosity + # while giving little to no benefit in readability + "TRY003", # | + "G004", # Forbids using f-strings in logging. This project doesn't rely on lazy % formatting when using logging. "A003", # Class attribute `id` is shadowing a Python builtin — it's ok in class body "FIX001", # Forbids using TODOs, but TODOs are useful "FIX002", # | @@ -66,8 +81,6 @@ ignore = [ # Want to resolve eventually, but not now: "S101", # Allow asserts (there are too many of them right now to fix) "TRY401", # Checks for excesive logging of exception objects - "E501", # Too long lines - "PTH118", # Prefer pathlib instead of os.path "G001", # Forbid str.format for logging "PTH123", # Checks for uses of `os.path.splitext` "D", # Docstrings @@ -84,7 +97,6 @@ ignore = [ "ERA001", # Found commented-out code "N801", # Class name should use CapWords convention "PLR0915", # Too many statements - "F401", # Imported but unused "PLR2004", # Magic value used in comparison, consider replacing with a constant variable "ANN002", # Missing type annotation for `*args` "TRY300", # Consider moving this statement to an `else` block @@ -112,6 +124,7 @@ ignore = [ ] # alembic is not a package in a traditional sense, so putting __init__.py there doesn't make sense "alembic/*" = ["INP001"] +"__init__.py" = ["F401"] [tool.ruff.lint.pep8-naming] classmethod-decorators = [ @@ -119,13 +132,14 @@ classmethod-decorators = [ ] [tool.ruff.lint.pylint] -max-args = 9 # in the name of stupidity and lazyness, please god forgive me +max-args = 9 # Lower number might be beneficial to reduce cognitive load. Consider using data containers. [tool.ruff.lint.isort] forced-separate = ["tests"] [tool.ruff.lint.flake8-tidy-imports] ban-relative-imports = "all" + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" diff --git a/packages/examples/cvat/recording-oracle/src/core/oracle_events.py b/packages/examples/cvat/recording-oracle/src/core/oracle_events.py index 04ce7c49db..721e3f0647 100644 --- a/packages/examples/cvat/recording-oracle/src/core/oracle_events.py +++ b/packages/examples/cvat/recording-oracle/src/core/oracle_events.py @@ -1,5 +1,3 @@ -from typing import Union - from pydantic import BaseModel from src.core.types import ExchangeOracleEventTypes, OracleWebhookTypes, RecordingOracleEventTypes diff --git a/packages/examples/cvat/recording-oracle/src/core/storage.py b/packages/examples/cvat/recording-oracle/src/core/storage.py index 6c5c53dbe2..939be882fd 100644 --- a/packages/examples/cvat/recording-oracle/src/core/storage.py +++ b/packages/examples/cvat/recording-oracle/src/core/storage.py @@ -7,4 +7,7 @@ def compose_data_bucket_filename(escrow_address: str, chain_id: Networks, filena def compose_results_bucket_filename(escrow_address: str, chain_id: Networks, filename: str) -> str: - return f"{escrow_address}@{chain_id}{Config.exchange_oracle_storage_config.results_dir_suffix}/{filename}" + return ( + f"{escrow_address}@{chain_id}{Config.exchange_oracle_storage_config.results_dir_suffix}" + f"/{filename}" + ) diff --git a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py index 1a22b9f7e1..17d9e2522f 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py @@ -1021,7 +1021,8 @@ def process_intermediate_results( # noqa: PLR0912 escrow_iteration = task.iteration if escrow_iteration and Config.validation.max_escrow_iterations <= escrow_iteration: logger.info( - f"Validation for escrow_address={escrow_address}: too many iterations, stopping annotation" + f"Validation for escrow_address={escrow_address}:" + f" too many iterations, stopping annotation" ) should_complete = True @@ -1038,15 +1039,17 @@ def process_intermediate_results( # noqa: PLR0912 ): logger.info( f"Validation for escrow_address={escrow_address}: " - f"too many assignments have insufficient GT for validation ({unverifiable_jobs_count} of {total_jobs} ({unverifiable_jobs_count / total_jobs * 100:.2f}%)), " - "stopping annotation" + f"too many assignments have insufficient GT for validation " + f"({unverifiable_jobs_count} of {total_jobs} " + f"({unverifiable_jobs_count / total_jobs * 100:.2f}%)), stopping annotation" ) should_complete = True elif len(rejected_jobs) == unverifiable_jobs_count: if unverifiable_jobs_count: logger.info( f"Validation for escrow_address={escrow_address}: " - f"only unverifiable assignments left ({unverifiable_jobs_count}), stopping annotation" + f"only unverifiable assignments left ({unverifiable_jobs_count})," + f" stopping annotation" ) should_complete = True diff --git a/packages/examples/cvat/recording-oracle/src/handlers/validation.py b/packages/examples/cvat/recording-oracle/src/handlers/validation.py index 06830f02d8..b6be970315 100644 --- a/packages/examples/cvat/recording-oracle/src/handlers/validation.py +++ b/packages/examples/cvat/recording-oracle/src/handlers/validation.py @@ -2,7 +2,6 @@ import os from collections import Counter from logging import Logger -from typing import Union from sqlalchemy.orm import Session diff --git a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py index f8c87216dd..168d01c820 100644 --- a/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py +++ b/packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py @@ -1,7 +1,7 @@ from __future__ import annotations import itertools -from abc import ABC, ABCMeta, abstractmethod +from abc import ABCMeta, abstractmethod from typing import TYPE_CHECKING import datumaro as dm diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py index 08c7a8584b..081221c3d0 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py @@ -162,7 +162,8 @@ def set_file_url_and_hash(url: str, key: str): register_in_kvstore() mock_set_file_url_and_hash.assert_not_called() - # check that public key URL and hash will be updated in KVStore if previous hash is outdated/corrupted + # check that public key URL and hash will be updated in KVStore + # if previous hash is outdated/corrupted with patch( "human_protocol_sdk.kvstore.KVStoreClient.set_file_url_and_hash", Mock() ) as mock_set_file_url_and_hash: @@ -172,7 +173,8 @@ def set_file_url_and_hash(url: str, key: str): mock_set_file_url_and_hash.assert_called_once() assert store["public_key_hash"] != "corrupted_hash" - # check that a new public key URL will be written to KVStore when an outdated URL is stored there + # check that a new public key URL will be written to KVStore + # when an outdated URL is stored there with ( patch( "src.core.config.Config.encryption_config.pgp_public_key_url", diff --git a/packages/examples/cvat/recording-oracle/tests/utils/constants.py b/packages/examples/cvat/recording-oracle/tests/utils/constants.py index 839008f892..2df6f087da 100644 --- a/packages/examples/cvat/recording-oracle/tests/utils/constants.py +++ b/packages/examples/cvat/recording-oracle/tests/utils/constants.py @@ -17,14 +17,20 @@ DEFAULT_MANIFEST_URL = "http://host.docker.internal:9000/manifests/manifest.json" DEFAULT_HASH = "test" -SIGNATURE = "0xa0c5626301e3c198cb91356e492890c0c28db8c37044846134939246911a693c4d7116d04aa4bc40a41077493868b8dd533d30980f6addb28d1b3610a84cb4091c" +SIGNATURE = ( + "0xa0c5626301e3c198cb91356e492890c0c28db8c37044846134939246911a693c" + "4d7116d04aa4bc40a41077493868b8dd533d30980f6addb28d1b3610a84cb4091c" +) WEBHOOK_MESSAGE = { "escrow_address": "0xFE776895f6b00AA53969b20119a4777Ed920676a", "chain_id": 80002, } -WEBHOOK_MESSAGE_SIGNED = "0xfeef93fdb26b9b855da432ca3ccc4425366e656401d3a4f67c2f0cab053fe34c2c7d0ce026d47f9fa100c05c12945f94c60dcb63ed94124fa4c60e888c4156281c" +WEBHOOK_MESSAGE_SIGNED = ( + "0xfeef93fdb26b9b855da432ca3ccc4425366e656401d3a4f67c2f0cab053fe34c" + "2c7d0ce026d47f9fa100c05c12945f94c60dcb63ed94124fa4c60e888c4156281c" +) JOB_REQUESTER_ID = "9001"