From 81d5667b4ed3fd4e0c6c3a408227354322a24551 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Mon, 2 Mar 2026 14:36:26 +0100 Subject: [PATCH 1/9] Bump third-party dependencies - Google Benchmark: v1.9.4 -> v1.9.5 - CUTLASS: v4.3.5 -> v4.4.1 - DLPack: v1.2 -> v1.3 - pybind11: v3.0.1 -> v3.0.2 - black: 25.12.0 -> 26.1.0 Signed-off-by: Joaquin Anton Guirao --- .github/workflows/lint.yml | 2 +- docker/Dockerfile | 2 +- docker/Dockerfile.build.aarch64-linux | 2 +- third_party/README.rst | 10 +++++----- third_party/benchmark | 2 +- third_party/cutlass | 2 +- third_party/dlpack | 2 +- third_party/pybind11 | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c6674b5d336..72b3cddbc78 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-python@v6 with: python-version: '3.10' - - run: pip install flake8 bandit "black[jupyter]"==25.12.0 + - run: pip install flake8 bandit "black[jupyter]"==26.1.0 # keep black invocations separated so each invocation will pick appropriate configuration file # from the top dir used for it - run: black --check --verbose ${{ env.PYTHON_SECURITY_LINT_PATHS }} ${{ env.PYTHON_LINT_PATHS }} ${{ env.AUTOGRAPH_LINT_PATHS }} diff --git a/docker/Dockerfile b/docker/Dockerfile index 90484779135..cdb7ecdab54 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -50,7 +50,7 @@ ${LIBRARY_PATH} RUN ln -s /opt/python/cp${PYV}* /opt/python/v # install Python bindings and patch it to use the clang we have here -RUN pip install future setuptools wheel clang==14.0 flake8 bandit astunparse gast optree numpy "black[jupyter]"==25.12.0 nvtx makefun && \ +RUN pip install future setuptools wheel clang==14.0 flake8 bandit astunparse gast optree numpy "black[jupyter]"==26.1.0 nvtx makefun && \ PY_CLANG_PATH=$(echo $(pip show clang) | sed 's/.*Location: \(.*\) Requires.*/\1/')/clang/cindex.py && \ LIBCLANG_PATH=/usr/lib64/libclang.so && \ sed -i "s|library_file = None|library_file = \"${LIBCLANG_PATH}\"|" ${PY_CLANG_PATH} && \ diff --git a/docker/Dockerfile.build.aarch64-linux b/docker/Dockerfile.build.aarch64-linux index b81f06cd645..3034708b1eb 100644 --- a/docker/Dockerfile.build.aarch64-linux +++ b/docker/Dockerfile.build.aarch64-linux @@ -52,7 +52,7 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/ curl -O https://bootstrap.pypa.io/get-pip.py; \ fi && python3 get-pip.py && rm get-pip.py && \ # decouple libclang and clang installation so libclang changes are not overriden by clang - pip install clang==14.0 && pip install libclang==14.0.1 flake8 bandit "black[jupyter]"==25.12.0 && \ + pip install clang==14.0 && pip install libclang==14.0.1 flake8 bandit "black[jupyter]"==26.1.0 && \ rm -rf /root/.cache/pip/ && \ cd /tmp && git clone https://github.com/NixOS/patchelf && cd patchelf && \ ./bootstrap.sh && ./configure --prefix=/usr/ && make -j install && cd / && rm -rf /tmp/patchelf && \ diff --git a/third_party/README.rst b/third_party/README.rst index 1f9da5ac97c..d0cc929cf8f 100644 --- a/third_party/README.rst +++ b/third_party/README.rst @@ -5,25 +5,25 @@ This part of the repository contains extra dependencies required to build DALI, +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ | Repository | Version | License | +==================================================================+===========================================================================================================================+===================================================================================================================+ -| `Google Benchmark `_ | `1.9.4 `_ | `Apache License 2.0 `_ | +| `Google Benchmark `_ | `1.9.5 `_ | `Apache License 2.0 `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ | `Boost Preprocessor `_ | `1.89.0 `_ | `Boost Software License 1.0 `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ | `COCO API `_ | `Top-of-tree (Feb 20, 2020) `_ | `BSD 2-Clause License `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ -| `CUTLASS `_ | `4.3.5 `_ | `BSD 3-Clause License `_ | +| `CUTLASS `_ | `4.4.1 `_ | `BSD 3-Clause License `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ | `CV-CUDA `_ | `0.16.0 `_ | `Apache License 2.0 `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ -| `DLPack `_ | `1.2 `_ | `Apache License 2.0 `_ | +| `DLPack `_ | `1.3 `_ | `Apache License 2.0 `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ | `FFTS `_ | `Custom fork top-of-tree (Sep 6, 2024) `_ | `BSD 3-Clause License `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ | `GoogleTest `_ | `1.17.0 `_ | `BSD 3-Clause License `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ -| `pybind11 `_ | `3.0.1 `_ | `BSD 3-Clause License `_ | +| `pybind11 `_ | `3.0.2 `_ | `BSD 3-Clause License `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ | `RapidJSON `_ | `Top-of-tree (Feb 5, 2025) `_ | `MIT License, BSD 3-Clause License, JSON License `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ -| `black `_ | `25.12.0 `_ | `MIT License `_ | +| `black `_ | `26.1.0 `_ | `MIT License `_ | +------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------+ diff --git a/third_party/benchmark b/third_party/benchmark index eddb0241389..192ef10025e 160000 --- a/third_party/benchmark +++ b/third_party/benchmark @@ -1 +1 @@ -Subproject commit eddb0241389718a23a42db6af5f0164b6e0139af +Subproject commit 192ef10025eb2c4cdd392bc502f0c852196baa48 diff --git a/third_party/cutlass b/third_party/cutlass index f3fde58372d..4370102f9da 160000 --- a/third_party/cutlass +++ b/third_party/cutlass @@ -1 +1 @@ -Subproject commit f3fde58372d33e9a5650ba7b80fc48b3b49d40c8 +Subproject commit 4370102f9dacab813282e1d67722fceb0b90a019 diff --git a/third_party/dlpack b/third_party/dlpack index 93c8f2a3c77..84d107bf416 160000 --- a/third_party/dlpack +++ b/third_party/dlpack @@ -1 +1 @@ -Subproject commit 93c8f2a3c774b84af6f652b1992c48164fae60fc +Subproject commit 84d107bf416c6bab9ae68ad285876600d230490d diff --git a/third_party/pybind11 b/third_party/pybind11 index f5fbe867d2d..45fab4087ea 160000 --- a/third_party/pybind11 +++ b/third_party/pybind11 @@ -1 +1 @@ -Subproject commit f5fbe867d2d26e4a0a9177a51f6e568868ad3dc8 +Subproject commit 45fab4087eaaff234227a10cf7845e8b07f28a98 From e03de1d881716efd4a273c0a1dbb0b09b73de832 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Mon, 2 Mar 2026 17:01:38 +0100 Subject: [PATCH 2/9] Bump conda dependencies - black/black-jupyter: 25.12.0 -> 26.1.0 - protobuf/libprotobuf/libprotobuf-static: 6.31.1 -> 6.33.5 dm-tree blocker removed (replaced by optree in #6225) Signed-off-by: Joaquin Anton Guirao --- conda/dali_native_libs/recipe/meta.yaml | 6 +++--- conda/dali_python_bindings/recipe/meta.yaml | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/conda/dali_native_libs/recipe/meta.yaml b/conda/dali_native_libs/recipe/meta.yaml index c1d3263d33e..9d06dc6bcf4 100644 --- a/conda/dali_native_libs/recipe/meta.yaml +++ b/conda/dali_native_libs/recipe/meta.yaml @@ -73,8 +73,8 @@ requirements: - python-clang - git-lfs host: - - protobuf =6.31.1 - - libprotobuf-static =6.31.1 + - protobuf =6.33.5 + - libprotobuf-static =6.33.5 - libjpeg-turbo - dali-opencv - dali-ffmpeg @@ -107,7 +107,7 @@ requirements: # dependency to install the right version on the libabseil (as protobuf depends on # libprotobuf-static and a newer version of libprotobuf-static may be available than # the protobuf was build with) - - libprotobuf =6.31.1 + - libprotobuf =6.33.5 - cfitsio - nvidia-nvimagecodec-cuda{{ environ.get('CUDA_VERSION', '') | replace(".","") }} - libnvcomp diff --git a/conda/dali_python_bindings/recipe/meta.yaml b/conda/dali_python_bindings/recipe/meta.yaml index 7866f3c64ac..7e1320e9d62 100644 --- a/conda/dali_python_bindings/recipe/meta.yaml +++ b/conda/dali_python_bindings/recipe/meta.yaml @@ -1,5 +1,5 @@ # (C) Copyright IBM Corp. 2019. All Rights Reserved. -# Copyright (c) 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -72,11 +72,11 @@ requirements: - patchelf - python-clang - git-lfs - - black =25.12.0 - - black-jupyter =25.12.0 + - black =26.1.0 + - black-jupyter =26.1.0 - bandit host: - - libprotobuf-static =6.31.1 + - libprotobuf-static =6.33.5 - dali-opencv - dali-ffmpeg - python @@ -97,7 +97,7 @@ requirements: # dependency to install the right version on the libabseil (as protobuf depends on # libprotobuf-static and a newer version of libprotobuf-static may be available than # the protobuf was build with) - - libprotobuf =6.31.1 + - libprotobuf =6.33.5 - future - astunparse >=1.6.0 - gast >=0.3.3 From 77d57a3350e52de109af449f8f49d6b80436ee2b Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Mon, 2 Mar 2026 17:10:28 +0100 Subject: [PATCH 3/9] Bump packaging upper limit to 26.0 Signed-off-by: Joaquin Anton Guirao --- dali/python/setup.py.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dali/python/setup.py.in b/dali/python/setup.py.in index ff67251bb1a..46a1cfc47fe 100644 --- a/dali/python/setup.py.in +++ b/dali/python/setup.py.in @@ -90,7 +90,7 @@ For more details please check the # 1.16 or later on python 3.12 due to import six.moves 'six >= 1.16, <= 1.17', 'optree', - 'packaging <= 25.0', + 'packaging <= 26.0', 'numpy', 'nvtx', 'makefun', From 32d095c8aacf6e033989b8cff81533bb2c530d82 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Tue, 3 Mar 2026 08:52:06 +0100 Subject: [PATCH 4/9] bump black required version to 26 Signed-off-by: Joaquin Anton Guirao --- docs/pyproject.toml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/pyproject.toml b/docs/pyproject.toml index 43e2bbd680b..d9d86b1cc73 100644 --- a/docs/pyproject.toml +++ b/docs/pyproject.toml @@ -1,6 +1,6 @@ [tool.black] line-length = 80 target-version = ['py38', 'py39', 'py310', 'py311'] -required-version = '25' +required-version = '26' workers = 32 extend-exclude = ".*/examples/use_cases/.*" diff --git a/pyproject.toml b/pyproject.toml index 8a5ab94a2d6..d039f4d74ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.black] line-length = 100 target-version = ['py38', 'py39', 'py310', 'py311'] -required-version = '25' +required-version = '26' workers = 32 extend-exclude = ".*/docs/examples/use_cases/.*" From aa8b39c5bcf9eb6a03bee315a16deaabc4289091 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Tue, 3 Mar 2026 19:46:05 +0100 Subject: [PATCH 5/9] black 26 autoformat Signed-off-by: Joaquin Anton Guirao --- .../dali/_autograph/converters/call_trees.py | 1 - .../dali/_autograph/converters/directives.py | 1 - .../converters/list_comprehensions.py | 1 - .../converters/return_statements.py | 1 - .../nvidia/dali/_autograph/core/ag_ctx.py | 1 - .../nvidia/dali/_autograph/core/config_lib.py | 1 - .../dali/_autograph/core/function_wrappers.py | 1 - .../nvidia/dali/_autograph/impl/conversion.py | 1 - .../_autograph/operators/data_structures.py | 1 - .../_autograph/operators/dispatch_context.py | 1 - .../dali/_autograph/operators/py_builtins.py | 1 - .../nvidia/dali/_autograph/pyct/gast_util.py | 1 + .../dali/_autograph/pyct/inspect_utils.py | 1 - .../dali/_autograph/pyct/origin_info.py | 1 + .../nvidia/dali/_autograph/pyct/parser.py | 7 +-- .../_autograph/pyct/static_analysis/annos.py | 1 - .../dali/_autograph/pyct/testing/codegen.py | 1 + .../nvidia/dali/_autograph/utils/tf_stack.py | 1 - dali/python/nvidia/dali/_multiproc/pool.py | 1 - .../nvidia/dali/_multiproc/shared_batch.py | 1 - dali/python/nvidia/dali/_utils/dali_trace.py | 1 - .../dali/_utils/external_source_impl.py | 8 +--- dali/python/nvidia/dali/backend.py | 1 - .../dali/experimental/dynamic/_device.py | 1 - .../dali/experimental/dynamic/_exceptions.py | 1 - .../dali/experimental/dynamic/_tensor.py | 2 +- dali/python/nvidia/dali/external_source.py | 4 +- dali/python/nvidia/dali/fn/__init__.py | 1 - dali/python/nvidia/dali/ops/__init__.py | 1 - dali/python/nvidia/dali/ops/_docs.py | 1 - .../dali/ops/_operators/python_function.py | 1 - dali/python/nvidia/dali/ops/_signatures.py | 19 +++----- dali/python/nvidia/dali/pipeline.py | 6 +-- .../nvidia/dali/plugin/base_iterator.py | 6 +-- .../nvidia/dali/plugin/jax/integration.py | 1 - .../nvidia/dali/plugin/numba/__init__.py | 1 - dali/python/nvidia/dali/plugin/tf.py | 11 ++--- dali/python/nvidia/dali/reducers.py | 1 - .../pyct/static_analysis/test_activity.py | 1 - .../pyct/static_analysis/test_liveness.py | 1 - .../test_reaching_definitions.py | 1 - dali/test/python/autograph/pyct/test_anno.py | 1 - .../python/autograph/pyct/test_ast_util.py | 40 ++++------------ .../autograph/pyct/test_inspect_utils.py | 24 ++++------ .../test/python/autograph/pyct/test_loader.py | 6 +-- .../test/python/autograph/pyct/test_parser.py | 12 ++--- .../autograph/pyct/test_pretty_printer.py | 6 +-- .../checkpointing/test_dali_checkpointing.py | 7 +-- dali/test/python/decoder/test_video.py | 2 +- .../python/experimental_mode/test_docs.py | 1 - .../experimental_mode/test_standalone_op.py | 4 +- .../python/jax_plugin/test_jax_operator.py | 1 - .../python/ndd_vs_fn/ndd_vs_fn_test_utils.py | 36 ++++---------- dali/test/python/ndd_vs_fn/test_ndd_vs_fn.py | 1 - .../ndd_vs_fn/test_ndd_vs_fn_1darray.py | 1 - .../ndd_vs_fn/test_ndd_vs_fn_coverage.py | 1 - .../ndd_vs_fn/test_ndd_vs_fn_noinput.py | 1 - .../python/ndd_vs_fn/test_ndd_vs_fn_random.py | 1 - .../ndd_vs_fn/test_ndd_vs_fn_sequence.py | 1 - dali/test/python/operator_1/test_debayer.py | 2 +- dali/test/python/operator_1/test_laplacian.py | 3 +- .../test/python/operator_1/test_multipaste.py | 1 - dali/test/python/operator_1/test_one_hot.py | 3 +- .../operator_1/test_operator_exception.py | 48 +++++++------------ .../operator_1/test_operator_origin_trace.py | 1 - .../operator_1/test_peek_image_shape.py | 2 +- dali/test/python/operator_1/test_slice.py | 8 ++-- .../python/operator_2/test_random_choice.py | 2 +- .../operator_2/test_random_object_bbox.py | 1 - dali/test/python/operator_2/test_rotate.py | 7 ++- .../python/operator_2/test_spectrogram.py | 1 - dali/test/python/reader/test_numpy.py | 3 +- dali/test/python/sequences_test_utils.py | 1 - ...test_RN50_external_source_parallel_data.py | 1 - dali/test/python/test_dali_cpu_only.py | 1 - .../test/python/test_dali_tf_dataset_eager.py | 1 - .../test/python/test_dali_tf_dataset_mnist.py | 1 - dali/test/python/test_dali_tf_exec2.py | 1 - dali/test/python/test_external_source_cupy.py | 2 - dali/test/python/test_external_source_dali.py | 2 +- .../python/test_external_source_parallel.py | 2 +- dali/test/python/test_fw_iterators.py | 8 ++-- dali/test/python/test_pipeline.py | 2 +- dali/test/python/test_pipeline_debug.py | 1 - dali/test/python/test_plugin_manager.py | 2 +- dali/test/python/test_tensor_formatting.py | 1 - dali/test/python/test_utils.py | 1 - dali/test/python/test_video_pipeline.py | 2 +- .../jax_operator_multi_gpu_process_1.py | 1 - .../frameworks/jax/flax-basic_example.ipynb | 1 - .../frameworks/jax/jax-basic_example.ipynb | 1 - .../frameworks/jax/jax-multigpu_example.ipynb | 4 -- docs/examples/frameworks/jax/model.py | 1 - .../jax/pax_examples/dali_pax_example.py | 1 - .../jax/pax_examples/dali_pax_input.py | 1 - .../paddle/paddle-basic_example.ipynb | 1 - .../paddle/paddle-various-readers.ipynb | 1 - .../pytorch/pytorch-basic_example.ipynb | 1 - .../pytorch/pytorch-various-readers.ipynb | 1 - .../image_processing/3d_transforms.ipynb | 2 +- .../pipeline_mode.ipynb | 1 - docs/examples/legacy_getting_started.ipynb | 1 - .../pipeline_mode.ipynb | 2 +- .../images/decode_and_transfrom_from_json.py | 1 - .../videos/decode_and_transform_from_json.py | 1 - internal_tools/hw_decoder_bench.py | 1 - internal_tools/spelling/common.py | 1 - 107 files changed, 104 insertions(+), 273 deletions(-) diff --git a/dali/python/nvidia/dali/_autograph/converters/call_trees.py b/dali/python/nvidia/dali/_autograph/converters/call_trees.py index 43f651e8d06..b33c74a4020 100644 --- a/dali/python/nvidia/dali/_autograph/converters/call_trees.py +++ b/dali/python/nvidia/dali/_autograph/converters/call_trees.py @@ -29,7 +29,6 @@ from nvidia.dali._autograph.pyct import templates from nvidia.dali._autograph.utils import ag_logging - # TODO(mdan): Rename to FunctionCallsTransformer. diff --git a/dali/python/nvidia/dali/_autograph/converters/directives.py b/dali/python/nvidia/dali/_autograph/converters/directives.py index 762baf75962..f7034531df8 100644 --- a/dali/python/nvidia/dali/_autograph/converters/directives.py +++ b/dali/python/nvidia/dali/_autograph/converters/directives.py @@ -35,7 +35,6 @@ from nvidia.dali._autograph.lang import directives from nvidia.dali._autograph.pyct import anno - STATIC_VALUE = "static_value" """Used for AST annotations, see visit_Name.""" diff --git a/dali/python/nvidia/dali/_autograph/converters/list_comprehensions.py b/dali/python/nvidia/dali/_autograph/converters/list_comprehensions.py index a7e0414d439..9f236fe9f74 100644 --- a/dali/python/nvidia/dali/_autograph/converters/list_comprehensions.py +++ b/dali/python/nvidia/dali/_autograph/converters/list_comprehensions.py @@ -31,7 +31,6 @@ from nvidia.dali._autograph.core import converter from nvidia.dali._autograph.pyct import templates - # TODO(mdan): This should covert directly to operator calls. diff --git a/dali/python/nvidia/dali/_autograph/converters/return_statements.py b/dali/python/nvidia/dali/_autograph/converters/return_statements.py index b04b3073621..09e1b3899aa 100644 --- a/dali/python/nvidia/dali/_autograph/converters/return_statements.py +++ b/dali/python/nvidia/dali/_autograph/converters/return_statements.py @@ -24,7 +24,6 @@ from nvidia.dali._autograph.pyct.static_analysis import activity from nvidia.dali._autograph.pyct.static_analysis.annos import NodeAnno - BODY_DEFINITELY_RETURNS = "BODY_DEFINITELY_RETURNS" ORELSE_DEFINITELY_RETURNS = "ORELSE_DEFINITELY_RETURNS" STMT_DEFINITELY_RETURNS = "STMT_DEFINITELY_RETURNS" diff --git a/dali/python/nvidia/dali/_autograph/core/ag_ctx.py b/dali/python/nvidia/dali/_autograph/core/ag_ctx.py index d03bc49878e..a6af573376d 100644 --- a/dali/python/nvidia/dali/_autograph/core/ag_ctx.py +++ b/dali/python/nvidia/dali/_autograph/core/ag_ctx.py @@ -21,7 +21,6 @@ from nvidia.dali._autograph.utils import ag_logging from nvidia.dali._autograph.utils.all_utils import export_symbol - stacks = threading.local() diff --git a/dali/python/nvidia/dali/_autograph/core/config_lib.py b/dali/python/nvidia/dali/_autograph/core/config_lib.py index d953778aa62..07e5e2b9b17 100644 --- a/dali/python/nvidia/dali/_autograph/core/config_lib.py +++ b/dali/python/nvidia/dali/_autograph/core/config_lib.py @@ -16,7 +16,6 @@ import enum - # TODO(mdan): For better performance, allow each rule to take a set names. diff --git a/dali/python/nvidia/dali/_autograph/core/function_wrappers.py b/dali/python/nvidia/dali/_autograph/core/function_wrappers.py index 2fa45f233f5..d04afc51048 100644 --- a/dali/python/nvidia/dali/_autograph/core/function_wrappers.py +++ b/dali/python/nvidia/dali/_autograph/core/function_wrappers.py @@ -18,7 +18,6 @@ from nvidia.dali._autograph.core import converter from nvidia.dali._autograph.operators import variables - # TODO(mdan): Move this into operators - it represents a function definition. diff --git a/dali/python/nvidia/dali/_autograph/impl/conversion.py b/dali/python/nvidia/dali/_autograph/impl/conversion.py index 7c69ff75061..5c540d02ae4 100644 --- a/dali/python/nvidia/dali/_autograph/impl/conversion.py +++ b/dali/python/nvidia/dali/_autograph/impl/conversion.py @@ -24,7 +24,6 @@ from nvidia.dali._autograph.pyct import inspect_utils from nvidia.dali._autograph.utils import ag_logging as logging - _ALLOWLIST_CACHE = cache.UnboundInstanceCache() diff --git a/dali/python/nvidia/dali/_autograph/operators/data_structures.py b/dali/python/nvidia/dali/_autograph/operators/data_structures.py index cc12a88227f..a01262c1987 100644 --- a/dali/python/nvidia/dali/_autograph/operators/data_structures.py +++ b/dali/python/nvidia/dali/_autograph/operators/data_structures.py @@ -18,7 +18,6 @@ from nvidia.dali._autograph.utils import hooks - # TODO(mdan): Once control flow supports objects, repackage as a class. diff --git a/dali/python/nvidia/dali/_autograph/operators/dispatch_context.py b/dali/python/nvidia/dali/_autograph/operators/dispatch_context.py index 3e81f454da8..b1904f90a65 100644 --- a/dali/python/nvidia/dali/_autograph/operators/dispatch_context.py +++ b/dali/python/nvidia/dali/_autograph/operators/dispatch_context.py @@ -16,7 +16,6 @@ import collections - # TODO(mdan): This is where macro override controls fit. diff --git a/dali/python/nvidia/dali/_autograph/operators/py_builtins.py b/dali/python/nvidia/dali/_autograph/operators/py_builtins.py index ccb582060a0..a0af76d865b 100644 --- a/dali/python/nvidia/dali/_autograph/operators/py_builtins.py +++ b/dali/python/nvidia/dali/_autograph/operators/py_builtins.py @@ -21,7 +21,6 @@ from nvidia.dali._autograph.utils import hooks - UNSPECIFIED = object() diff --git a/dali/python/nvidia/dali/_autograph/pyct/gast_util.py b/dali/python/nvidia/dali/_autograph/pyct/gast_util.py index 107f626416e..614dffd6d76 100644 --- a/dali/python/nvidia/dali/_autograph/pyct/gast_util.py +++ b/dali/python/nvidia/dali/_autograph/pyct/gast_util.py @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================== """Gast compatibility library. Supports 0.2.2 and 0.3.2.""" + # TODO(mdan): Remove this file once it's safe to break compatibility. import functools diff --git a/dali/python/nvidia/dali/_autograph/pyct/inspect_utils.py b/dali/python/nvidia/dali/_autograph/pyct/inspect_utils.py index d8b1dd00007..e12faa8df2b 100644 --- a/dali/python/nvidia/dali/_autograph/pyct/inspect_utils.py +++ b/dali/python/nvidia/dali/_autograph/pyct/inspect_utils.py @@ -26,7 +26,6 @@ import six - # This lock seems to help avoid linecache concurrency errors. _linecache_lock = threading.Lock() diff --git a/dali/python/nvidia/dali/_autograph/pyct/origin_info.py b/dali/python/nvidia/dali/_autograph/pyct/origin_info.py index db3e438aec5..f2879373367 100644 --- a/dali/python/nvidia/dali/_autograph/pyct/origin_info.py +++ b/dali/python/nvidia/dali/_autograph/pyct/origin_info.py @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================== """Container for origin source code information before AutoGraph compilation.""" + import collections import difflib import inspect diff --git a/dali/python/nvidia/dali/_autograph/pyct/parser.py b/dali/python/nvidia/dali/_autograph/pyct/parser.py index 589dfe55027..87c3a0ee73b 100644 --- a/dali/python/nvidia/dali/_autograph/pyct/parser.py +++ b/dali/python/nvidia/dali/_autograph/pyct/parser.py @@ -32,11 +32,8 @@ from nvidia.dali._autograph.pyct import errors from nvidia.dali._autograph.pyct import inspect_utils - -PY2_PREAMBLE = textwrap.dedent( - """ -""" -) +PY2_PREAMBLE = textwrap.dedent(""" +""") PY3_PREAMBLE = "" MAX_SIZE = 0 diff --git a/dali/python/nvidia/dali/_autograph/pyct/static_analysis/annos.py b/dali/python/nvidia/dali/_autograph/pyct/static_analysis/annos.py index ae003bc6940..805bef4a91f 100644 --- a/dali/python/nvidia/dali/_autograph/pyct/static_analysis/annos.py +++ b/dali/python/nvidia/dali/_autograph/pyct/static_analysis/annos.py @@ -16,7 +16,6 @@ from enum import Enum - # TODO(mdan): Remove. diff --git a/dali/python/nvidia/dali/_autograph/pyct/testing/codegen.py b/dali/python/nvidia/dali/_autograph/pyct/testing/codegen.py index 2847b3fc472..2c002e8af1c 100644 --- a/dali/python/nvidia/dali/_autograph/pyct/testing/codegen.py +++ b/dali/python/nvidia/dali/_autograph/pyct/testing/codegen.py @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================== """Random code generation for testing/fuzzing.""" + # pylint: disable=invalid-name import random import string diff --git a/dali/python/nvidia/dali/_autograph/utils/tf_stack.py b/dali/python/nvidia/dali/_autograph/utils/tf_stack.py index f9cf48fe472..d86b46028d6 100644 --- a/dali/python/nvidia/dali/_autograph/utils/tf_stack.py +++ b/dali/python/nvidia/dali/_autograph/utils/tf_stack.py @@ -34,7 +34,6 @@ from types import ModuleType from typing import List, Union - # Generally such lookups should be done using `threading.local()`. See # https://blogs.gnome.org/jamesh/2008/06/11/tls-python/ for a detailed # explanation of why. However the transform stacks are expected to be empty diff --git a/dali/python/nvidia/dali/_multiproc/pool.py b/dali/python/nvidia/dali/_multiproc/pool.py index 2b24d88acb7..f85551fb1a2 100644 --- a/dali/python/nvidia/dali/_multiproc/pool.py +++ b/dali/python/nvidia/dali/_multiproc/pool.py @@ -36,7 +36,6 @@ ) from nvidia.dali._multiproc.shared_queue import ShmQueue - """ A pipeline with parallel external sources creates `WorkerPool` to parallelize sources computation. Each external source in the pipeline has its own `ShmChunkManager` with a view on shm chunks diff --git a/dali/python/nvidia/dali/_multiproc/shared_batch.py b/dali/python/nvidia/dali/_multiproc/shared_batch.py index 4b371847ab3..951731b70f7 100644 --- a/dali/python/nvidia/dali/_multiproc/shared_batch.py +++ b/dali/python/nvidia/dali/_multiproc/shared_batch.py @@ -21,7 +21,6 @@ ) import pickle # nosec B403 - np = None diff --git a/dali/python/nvidia/dali/_utils/dali_trace.py b/dali/python/nvidia/dali/_utils/dali_trace.py index 5f249315b90..811623fe402 100644 --- a/dali/python/nvidia/dali/_utils/dali_trace.py +++ b/dali/python/nvidia/dali/_utils/dali_trace.py @@ -17,7 +17,6 @@ from nvidia.dali._autograph.utils.tf_stack import get_frame_map, get_frame_filter from nvidia.dali._autograph import is_frame_ag_call_entrypoint, is_frame_ag_call_unconverted - _origin_trace_enabled = True # Processing options, mainly for debugging purposes _collapse_ag_frames = True diff --git a/dali/python/nvidia/dali/_utils/external_source_impl.py b/dali/python/nvidia/dali/_utils/external_source_impl.py index e8e80050642..591295e2df0 100644 --- a/dali/python/nvidia/dali/_utils/external_source_impl.py +++ b/dali/python/nvidia/dali/_utils/external_source_impl.py @@ -227,14 +227,10 @@ def _cycle_enabled(cycle): return False if cycle is True or cycle == "quiet" or cycle == "raise": return True - raise ValueError( - """Invalid value {} for the argument `cycle`. Valid values are + raise ValueError("""Invalid value {} for the argument `cycle`. Valid values are - "no", False or None - cycling disabled - "quiet", True - quietly rewind the data - - "raise" - raise StopIteration on each rewind.""".format( - repr(cycle) - ) - ) + - "raise" - raise StopIteration on each rewind.""".format(repr(cycle))) def accepted_arg_count(callable): diff --git a/dali/python/nvidia/dali/backend.py b/dali/python/nvidia/dali/backend.py index df4dfcc3842..fe03ad654f0 100644 --- a/dali/python/nvidia/dali/backend.py +++ b/dali/python/nvidia/dali/backend.py @@ -38,7 +38,6 @@ import atexit import gc - _ExecutorType.__bool__ = lambda self: self.value != 0 _ExecutorType.__and__ = lambda x, y: _ExecutorType(x.value & y.value) _ExecutorType.__or__ = lambda x, y: _ExecutorType(x.value | y.value) diff --git a/dali/python/nvidia/dali/experimental/dynamic/_device.py b/dali/python/nvidia/dali/experimental/dynamic/_device.py index c6466d25498..ab333c62213 100644 --- a/dali/python/nvidia/dali/experimental/dynamic/_device.py +++ b/dali/python/nvidia/dali/experimental/dynamic/_device.py @@ -16,7 +16,6 @@ from threading import local from typing import NoReturn, TypeAlias, Union - try: import torch # type: ignore diff --git a/dali/python/nvidia/dali/experimental/dynamic/_exceptions.py b/dali/python/nvidia/dali/experimental/dynamic/_exceptions.py index d5e81d77f20..79ecd224b77 100644 --- a/dali/python/nvidia/dali/experimental/dynamic/_exceptions.py +++ b/dali/python/nvidia/dali/experimental/dynamic/_exceptions.py @@ -19,7 +19,6 @@ from ._eval_mode import EvalMode - T = TypeVar("T", bound=BaseException) CallStack: TypeAlias = Sequence[tuple[types.CodeType, int]] diff --git a/dali/python/nvidia/dali/experimental/dynamic/_tensor.py b/dali/python/nvidia/dali/experimental/dynamic/_tensor.py index 9f3fafe3357..fe3eb2c035f 100644 --- a/dali/python/nvidia/dali/experimental/dynamic/_tensor.py +++ b/dali/python/nvidia/dali/experimental/dynamic/_tensor.py @@ -253,7 +253,7 @@ def __init__( elif arr.dtype == np.float64: arr = arr.astype(np.float32) elif arr.dtype == object: - (arr, converted_dtype_id) = _try_convert_enums(arr) + arr, converted_dtype_id = _try_convert_enums(arr) self._storage = _backend.TensorCPU(arr, layout, False) if converted_dtype_id is not None: self._storage.reinterpret(converted_dtype_id) diff --git a/dali/python/nvidia/dali/external_source.py b/dali/python/nvidia/dali/external_source.py index 83be51ca19a..1befb1f1c69 100644 --- a/dali/python/nvidia/dali/external_source.py +++ b/dali/python/nvidia/dali/external_source.py @@ -92,7 +92,7 @@ def to_numpy(x): inputs = [] checked = False for datum in data: - (is_dlpack, is_gpu_data) = _b.CheckDLPackCapsule(datum) + is_dlpack, is_gpu_data = _b.CheckDLPackCapsule(datum) if not is_dlpack and not checked: _check_data_batch(data, batch_size, layout) checked = True @@ -117,7 +117,7 @@ def to_numpy(x): ), "Mixed input types are not support, all need to reside on the CPU or GPU" data = inputs else: - (is_dlpack, is_gpu_data) = _b.CheckDLPackCapsule(data) + is_dlpack, is_gpu_data = _b.CheckDLPackCapsule(data) if not is_dlpack: _check_data_batch(data, batch_size, layout) if hasattr(data, "__cuda_array_interface__"): diff --git a/dali/python/nvidia/dali/fn/__init__.py b/dali/python/nvidia/dali/fn/__init__.py index ea4f8027b35..0c44ca37071 100644 --- a/dali/python/nvidia/dali/fn/__init__.py +++ b/dali/python/nvidia/dali/fn/__init__.py @@ -19,7 +19,6 @@ from nvidia.dali.external_source import external_source from nvidia.dali._utils import dali_trace as _dali_trace - _special_case_mapping = {"b_box": "bbox", "mx_net": "mxnet", "tf_record": "tfrecord"} diff --git a/dali/python/nvidia/dali/ops/__init__.py b/dali/python/nvidia/dali/ops/__init__.py index 31ed86faa3e..7cb0080ff1d 100644 --- a/dali/python/nvidia/dali/ops/__init__.py +++ b/dali/python/nvidia/dali/ops/__init__.py @@ -894,7 +894,6 @@ def _promote_scalar_constant(value, input_device): _generate_input_desc, ) - # Discover and generate bindings for all regular operators. _load_ops() diff --git a/dali/python/nvidia/dali/ops/_docs.py b/dali/python/nvidia/dali/ops/_docs.py index cb19e487589..3758b71cc75 100644 --- a/dali/python/nvidia/dali/ops/_docs.py +++ b/dali/python/nvidia/dali/ops/_docs.py @@ -19,7 +19,6 @@ from nvidia.dali.ops import _registry, _names - _MAX_INPUT_SPELLED_OUT = 5 diff --git a/dali/python/nvidia/dali/ops/_operators/python_function.py b/dali/python/nvidia/dali/ops/_operators/python_function.py index 9cca3cfcea8..718d336f386 100644 --- a/dali/python/nvidia/dali/ops/_operators/python_function.py +++ b/dali/python/nvidia/dali/ops/_operators/python_function.py @@ -20,7 +20,6 @@ from nvidia.dali.pipeline import Pipeline as _Pipeline from nvidia.dali.types import CUDAStream as _CUDAStream - cupy = None diff --git a/dali/python/nvidia/dali/ops/_signatures.py b/dali/python/nvidia/dali/ops/_signatures.py index de764a1c6fb..e0fb2b5a2cf 100644 --- a/dali/python/nvidia/dali/ops/_signatures.py +++ b/dali/python/nvidia/dali/ops/_signatures.py @@ -29,7 +29,6 @@ from nvidia.dali import types as _types from nvidia.dali.ops import _docs, _names, _registry - Api = Literal["fn", "ops", "dynamic"] @@ -647,8 +646,7 @@ def _gen_ops_signature(schema, schema_name, cls_name): if schema.MaxNumInput() == 0 else _gen_ops_call_signature_with_inputs(schema, schema_name) ) - return inspect_repr_fixups( - f""" + return inspect_repr_fixups(f""" class {cls_name}: \"""{_docs._docstring_generator_class(schema_name)} \""" @@ -658,8 +656,7 @@ def __init__{_call_signature(schema, "ops", include_inputs=False, include_kwargs ... {signature} -""" - ) +""") def _gen_dynamic_call_signature_no_input(schema: _b.OpSchema, **kwargs): @@ -777,13 +774,11 @@ def next_epoch( def _gen_dynamic_cls_signature(schema: _b.OpSchema, schema_name: str, op_name: str): - call_template = string.Template( - """ + call_template = string.Template(""" @overload def __call__$signature: ... - """ - ) + """) call_overloads = ( call_template.substitute(signature=signature) for signature in _gen_dynamic_call_signature( @@ -816,14 +811,12 @@ def __init__{_call_signature( def _gen_dynamic_fun_signature(schema: _b.OpSchema, schema_name: str, op_name: str): - template = string.Template( - """ + template = string.Template(""" @overload def $fn_name$signature: \"""$doc \""" -""" - ) +""") doc = _docs._docstring_generator_fn(schema_name, api="dynamic") overloads = ( diff --git a/dali/python/nvidia/dali/pipeline.py b/dali/python/nvidia/dali/pipeline.py index 830f388ce83..21f3f1f93ac 100644 --- a/dali/python/nvidia/dali/pipeline.py +++ b/dali/python/nvidia/dali/pipeline.py @@ -1554,8 +1554,7 @@ def my_pipe(): A tuple of `TensorList` objects for respective pipeline outputs """ if len(pipeline_inputs) > 0 and not self._are_pipeline_inputs_possible(): - raise RuntimeError( - f""" + raise RuntimeError(f""" When using pipeline_inputs named arguments, either `prefetch_queue_depth` in Pipeline constructor shall be set to 1 (for both devices) or `exec_pipelined` shall be set to False. @@ -1564,8 +1563,7 @@ def my_pipe(): Please set the `prefetch_queue_depth` or `exec_pipelined` argument in the Pipeline constructor properly or provide inputs to DALI Pipeline via another mean (e.g. `feed_input` function or `source` argument in the `fn.external_source` - operator.)""" - ) + operator.)""") self.build() for inp_name, inp_value in pipeline_inputs.items(): self.feed_input(inp_name, inp_value) diff --git a/dali/python/nvidia/dali/plugin/base_iterator.py b/dali/python/nvidia/dali/plugin/base_iterator.py index 241a8f2910e..3c61db5a662 100644 --- a/dali/python/nvidia/dali/plugin/base_iterator.py +++ b/dali/python/nvidia/dali/plugin/base_iterator.py @@ -573,10 +573,8 @@ def reset(self): with p._check_api_type_scope(types.PipelineAPIType.ITERATOR): p.schedule_run() else: - logging.warning( - "DALI iterator does not support resetting while epoch is not finished. \ - Ignoring..." - ) + logging.warning("DALI iterator does not support resetting while epoch is not finished. \ + Ignoring...") def next(self): """ diff --git a/dali/python/nvidia/dali/plugin/jax/integration.py b/dali/python/nvidia/dali/plugin/jax/integration.py index 72f537b96b6..fee78617b6b 100644 --- a/dali/python/nvidia/dali/plugin/jax/integration.py +++ b/dali/python/nvidia/dali/plugin/jax/integration.py @@ -18,7 +18,6 @@ from nvidia.dali.backend import TensorGPU from packaging.version import Version - _jax_has_old_dlpack = Version(jax.__version__) < Version("0.4.16") diff --git a/dali/python/nvidia/dali/plugin/numba/__init__.py b/dali/python/nvidia/dali/plugin/numba/__init__.py index 361df926854..7ac96e88d53 100644 --- a/dali/python/nvidia/dali/plugin/numba/__init__.py +++ b/dali/python/nvidia/dali/plugin/numba/__init__.py @@ -25,7 +25,6 @@ import numba as nb import importlib - _to_numpy = { dali_types.BOOL: "bool_", dali_types.UINT8: "uint8", diff --git a/dali/python/nvidia/dali/plugin/tf.py b/dali/python/nvidia/dali/plugin/tf.py index cc39b3356d5..44b698a5e12 100644 --- a/dali/python/nvidia/dali/plugin/tf.py +++ b/dali/python/nvidia/dali/plugin/tf.py @@ -35,15 +35,12 @@ _dali_tf_module = dali_tf_plugin.load_dali_tf_plugin() _dali_tf = _dali_tf_module.dali -_dali_tf.__doc__ = ( - _dali_tf.__doc__ - + """ +_dali_tf.__doc__ = _dali_tf.__doc__ + """ Please keep in mind that TensorFlow allocates almost all available device memory by default. This might cause errors in DALI due to insufficient memory. On how to change this behavior please look into the TensorFlow documentation, as it may differ based on your use case. """ -) _experimental_dataset_docstring = """Experimental variant of :class:`~nvidia.dali.plugin.tf.DALIDataset`. This dataset adds support for input tf.data.Datasets. @@ -229,10 +226,8 @@ def DALIIteratorWrapper( cpu_prefetch_queue_depth = prefetch_queue_depth["cpu_size"] gpu_prefetch_queue_depth = prefetch_queue_depth["gpu_size"] if exec_dynamic: - raise ValueError( - """Separated queues are incompatible with the default execution model. - Please set ``exec_dynamic=False`` in the pipeline constructor or @pipeline_def.""" - ) + raise ValueError("""Separated queues are incompatible with the default execution model. + Please set ``exec_dynamic=False`` in the pipeline constructor or @pipeline_def.""") elif type(prefetch_queue_depth) is int: exec_separated = False cpu_prefetch_queue_depth = -1 # dummy: wont' be used diff --git a/dali/python/nvidia/dali/reducers.py b/dali/python/nvidia/dali/reducers.py index db195a38bc9..087dba384a2 100644 --- a/dali/python/nvidia/dali/reducers.py +++ b/dali/python/nvidia/dali/reducers.py @@ -19,7 +19,6 @@ import marshal import importlib - # Don't allow any reformatters turning it into regular `def` function. # The whole point of this object is to have # properties (name) specific to lambda. diff --git a/dali/test/python/autograph/pyct/static_analysis/test_activity.py b/dali/test/python/autograph/pyct/static_analysis/test_activity.py index 77078001cae..70223f9588d 100644 --- a/dali/test/python/autograph/pyct/static_analysis/test_activity.py +++ b/dali/test/python/autograph/pyct/static_analysis/test_activity.py @@ -26,7 +26,6 @@ from nvidia.dali._autograph.pyct.static_analysis import activity from nvidia.dali._autograph.pyct.static_analysis import annos - QN = qual_names.QN NodeAnno = annos.NodeAnno diff --git a/dali/test/python/autograph/pyct/static_analysis/test_liveness.py b/dali/test/python/autograph/pyct/static_analysis/test_liveness.py index b3d8902ded7..96d3591f8fc 100644 --- a/dali/test/python/autograph/pyct/static_analysis/test_liveness.py +++ b/dali/test/python/autograph/pyct/static_analysis/test_liveness.py @@ -26,7 +26,6 @@ from nvidia.dali._autograph.pyct.static_analysis import liveness from nvidia.dali._autograph.pyct.static_analysis import reaching_fndefs - global_a = 7 global_b = 17 diff --git a/dali/test/python/autograph/pyct/static_analysis/test_reaching_definitions.py b/dali/test/python/autograph/pyct/static_analysis/test_reaching_definitions.py index 5d99f6fbe4c..adce80d7d11 100644 --- a/dali/test/python/autograph/pyct/static_analysis/test_reaching_definitions.py +++ b/dali/test/python/autograph/pyct/static_analysis/test_reaching_definitions.py @@ -25,7 +25,6 @@ from nvidia.dali._autograph.pyct.static_analysis import activity from nvidia.dali._autograph.pyct.static_analysis import reaching_definitions - global_a = 7 global_b = 17 diff --git a/dali/test/python/autograph/pyct/test_anno.py b/dali/test/python/autograph/pyct/test_anno.py index acd2792c8fa..13246472dee 100644 --- a/dali/test/python/autograph/pyct/test_anno.py +++ b/dali/test/python/autograph/pyct/test_anno.py @@ -19,7 +19,6 @@ from nvidia.dali._autograph.pyct import anno - # TODO(mdan): Consider strong types instead of primitives. diff --git a/dali/test/python/autograph/pyct/test_ast_util.py b/dali/test/python/autograph/pyct/test_ast_util.py index 70bd8b12430..d1d26a743c5 100644 --- a/dali/test/python/autograph/pyct/test_ast_util.py +++ b/dali/test/python/autograph/pyct/test_ast_util.py @@ -100,28 +100,20 @@ def test_rename_symbols_function(self): self.assertEqual(source.strip(), "def f1():\n pass") def test_copy_clean(self): - node = parser.parse( - textwrap.dedent( - """ + node = parser.parse(textwrap.dedent(""" def f(a): return a + 1 - """ - ) - ) + """)) setattr(node, "__foo", "bar") new_node = ast_util.copy_clean(node) self.assertIsNot(new_node, node) self.assertFalse(hasattr(new_node, "__foo")) def test_copy_clean_preserves_annotations(self): - node = parser.parse( - textwrap.dedent( - """ + node = parser.parse(textwrap.dedent(""" def f(a): return a + 1 - """ - ) - ) + """)) anno.setanno(node, "foo", "bar") anno.setanno(node, "baz", 1) new_node = ast_util.copy_clean(node, preserve_annos={"foo"}) @@ -210,30 +202,18 @@ def f(a): self.assertEqual(child_a, child_b) def test_parallel_walk_inconsistent_trees(self): - node_1 = parser.parse( - textwrap.dedent( - """ + node_1 = parser.parse(textwrap.dedent(""" def f(a): return a + 1 - """ - ) - ) - node_2 = parser.parse( - textwrap.dedent( - """ + """)) + node_2 = parser.parse(textwrap.dedent(""" def f(a): return a + (a * 2) - """ - ) - ) - node_3 = parser.parse( - textwrap.dedent( - """ + """)) + node_3 = parser.parse(textwrap.dedent(""" def f(a): return a + 2 - """ - ) - ) + """)) with self.assertRaises(ValueError): for _ in ast_util.parallel_walk(node_1, node_2): pass diff --git a/dali/test/python/autograph/pyct/test_inspect_utils.py b/dali/test/python/autograph/pyct/test_inspect_utils.py index 3f4b8351ac8..f8c2a12b055 100644 --- a/dali/test/python/autograph/pyct/test_inspect_utils.py +++ b/dali/test/python/autograph/pyct/test_inspect_utils.py @@ -179,13 +179,11 @@ def wrapper(*args, **kwargs): return wrapper - expected = textwrap.dedent( - """ + expected = textwrap.dedent(""" @functools.wraps(f) def wrapper(*args, **kwargs): return f(*args, **kwargs) - """ - ) + """) @wrapper_decorator def test_fn(a): @@ -195,13 +193,11 @@ def test_fn(a): self.assertSourceIdentical(inspect_utils.getimmediatesource(test_fn), expected) def test_getimmediatesource_functools_wrapper_different_module(self): - expected = textwrap.dedent( - """ + expected = textwrap.dedent(""" @functools.wraps(f) def wrapper(*args, **kwargs): return f(*args, **kwargs) - """ - ) + """) @decorators.wrapping_decorator def test_fn(a): @@ -211,12 +207,10 @@ def test_fn(a): self.assertSourceIdentical(inspect_utils.getimmediatesource(test_fn), expected) def test_getimmediatesource_normal_decorator_different_module(self): - expected = textwrap.dedent( - """ + expected = textwrap.dedent(""" def standalone_wrapper(*args, **kwargs): return f(*args, **kwargs) - """ - ) + """) @decorators.standalone_decorator def test_fn(a): @@ -226,12 +220,10 @@ def test_fn(a): self.assertSourceIdentical(inspect_utils.getimmediatesource(test_fn), expected) def test_getimmediatesource_normal_functional_decorator_different_module(self): - expected = textwrap.dedent( - """ + expected = textwrap.dedent(""" def functional_wrapper(*args, **kwargs): return f(*args, **kwargs) - """ - ) + """) @decorators.functional_decorator() def test_fn(a): diff --git a/dali/test/python/autograph/pyct/test_loader.py b/dali/test/python/autograph/pyct/test_loader.py index bcd33d2c4e2..0c574cfe923 100644 --- a/dali/test/python/autograph/pyct/test_loader.py +++ b/dali/test/python/autograph/pyct/test_loader.py @@ -99,14 +99,12 @@ def f(a): self.assertAstMatches(node, temp_output.read()) def test_load_source(self): - test_source = textwrap.dedent( - """ + test_source = textwrap.dedent(""" # coding=utf-8 def f(a): '日本語 Δθₜ ← Δθₜ₋₁ + ∇Q(sₜ, aₜ)(rₜ + γₜ₊₁ max Q(⋅))' return a + 1 - """ - ) + """) module, _ = loader.load_source(test_source, delete_on_exit=True) self.assertEqual(module.f(1), 2) self.assertEqual(module.f.__doc__, "日本語 Δθₜ ← Δθₜ₋₁ + ∇Q(sₜ, aₜ)(rₜ + γₜ₊₁ max Q(⋅))") diff --git a/dali/test/python/autograph/pyct/test_parser.py b/dali/test/python/autograph/pyct/test_parser.py index bc77dd045f0..b27acfda248 100644 --- a/dali/test/python/autograph/pyct/test_parser.py +++ b/dali/test/python/autograph/pyct/test_parser.py @@ -223,11 +223,9 @@ def f(): def test_parse_multiline_strings(self): def f(): - print( - """ + print(""" multiline -string""" - ) +string""") node, _ = parser.parse_entity(f, future_features=()) self.assertEqual("f", node.name) @@ -342,15 +340,13 @@ def test_unparse(self): source = parser.unparse(node, indentation=" ") self.assertEqual( - textwrap.dedent( - """ + textwrap.dedent(""" # coding=utf-8 if 1: a = b else: a = 'c' - """ - ).strip(), + """).strip(), source.strip(), ) diff --git a/dali/test/python/autograph/pyct/test_pretty_printer.py b/dali/test/python/autograph/pyct/test_pretty_printer.py index 35c2f996dad..0d6f62e81d5 100644 --- a/dali/test/python/autograph/pyct/test_pretty_printer.py +++ b/dali/test/python/autograph/pyct/test_pretty_printer.py @@ -23,12 +23,10 @@ class PrettyPrinterTest(unittest.TestCase): def test_unicode_bytes(self): - source = textwrap.dedent( - """ + source = textwrap.dedent(""" def f(): return b'b', u'u', 'depends_py2_py3' - """ - ) + """) node = ast.parse(source) self.assertIsNotNone(pretty_printer.fmt(node)) diff --git a/dali/test/python/checkpointing/test_dali_checkpointing.py b/dali/test/python/checkpointing/test_dali_checkpointing.py index cd7a7a9009a..1b9a613ef27 100644 --- a/dali/test/python/checkpointing/test_dali_checkpointing.py +++ b/dali/test/python/checkpointing/test_dali_checkpointing.py @@ -36,7 +36,6 @@ from nvidia.dali.auto_aug import trivial_augment as ta from reader.test_numpy import is_gds_supported - reader_signed_off = create_sign_off_registry() random_signed_off = create_sign_off_registry() @@ -529,12 +528,10 @@ def test_nemo_asr_reader( manifest = tempfile.NamedTemporaryFile("w") for i, f in enumerate(wav_files): - manifest.write( - f'{{"audio_filepath": "{f}", \ + manifest.write(f'{{"audio_filepath": "{f}", \ "offset": {i / 1000}, \ "duration": {0.3 + i / 100}, \ - "text": "o{"o" * i}"}}\n' - ) + "text": "o{"o" * i}"}}\n') manifest.flush() check_reader_checkpointing( diff --git a/dali/test/python/decoder/test_video.py b/dali/test/python/decoder/test_video.py index 356dcea2fe4..1afc621fe69 100644 --- a/dali/test/python/decoder/test_video.py +++ b/dali/test/python/decoder/test_video.py @@ -884,7 +884,7 @@ def decoder_pipeline(): return videos, frame_no pipe = decoder_pipeline(batch_size=batch_size, num_threads=2, device_id=0) - (out, frame_no) = pipe.run() + out, frame_no = pipe.run() assert len(out) > 0, f"No output from decoder pipeline for {codec}" for i in range(batch_size): diff --git a/dali/test/python/experimental_mode/test_docs.py b/dali/test/python/experimental_mode/test_docs.py index 27fd53a14e0..adb8872ad98 100644 --- a/dali/test/python/experimental_mode/test_docs.py +++ b/dali/test/python/experimental_mode/test_docs.py @@ -15,7 +15,6 @@ import nvidia.dali.experimental.dynamic as ndd import re - _graph_regex = re.compile(r".*(^|[^A-Za-z0-9_])[Gg]raph([ .,)]|$).*") diff --git a/dali/test/python/experimental_mode/test_standalone_op.py b/dali/test/python/experimental_mode/test_standalone_op.py index b9fbdb59b24..6d41d250ff3 100644 --- a/dali/test/python/experimental_mode/test_standalone_op.py +++ b/dali/test/python/experimental_mode/test_standalone_op.py @@ -156,8 +156,8 @@ def test_standalone_op_reader(): ws.SetStream(stream) op.SetupAndRun(ws) # get the output - (out_img, out_label) = ws.GetOutputs() - (ref_img, ref_label) = pipe.run() + out_img, out_label = ws.GetOutputs() + ref_img, ref_label = pipe.run() for i in range(batch_size): assert np.array_equal(out_img[i], ref_img[i]) assert np.array_equal(out_label[i], ref_label[i]) diff --git a/dali/test/python/jax_plugin/test_jax_operator.py b/dali/test/python/jax_plugin/test_jax_operator.py index 3d364caf5ad..0cf1c20de60 100644 --- a/dali/test/python/jax_plugin/test_jax_operator.py +++ b/dali/test/python/jax_plugin/test_jax_operator.py @@ -24,7 +24,6 @@ from nose_utils import assert_raises from test_utils import get_dali_extra_path, check_batch, restrict_python_version - test_data_root = get_dali_extra_path() images_dir = os.path.join(test_data_root, "db", "single", "jpeg") diff --git a/dali/test/python/ndd_vs_fn/ndd_vs_fn_test_utils.py b/dali/test/python/ndd_vs_fn/ndd_vs_fn_test_utils.py index 1a291e62022..008020226eb 100644 --- a/dali/test/python/ndd_vs_fn/ndd_vs_fn_test_utils.py +++ b/dali/test/python/ndd_vs_fn/ndd_vs_fn_test_utils.py @@ -296,44 +296,34 @@ def _cmp_metadata( # Validate device pipe_is_gpu = isinstance(pipe_out, TensorListGPU) ndd_is_gpu = ndd_out.device.device_type == "gpu" - assert ( - pipe_is_gpu == ndd_is_gpu - ), f"""[Output {output_idx}] Device mismatch: + assert pipe_is_gpu == ndd_is_gpu, f"""[Output {output_idx}] Device mismatch: Pipeline: {'gpu' if pipe_is_gpu else 'cpu'} NDD: {ndd_out.device.device_type}""" # Validate batch size pipe_batch_size = len(pipe_out.shape()) ndd_batch_size = ndd_out.batch_size - assert ( - pipe_batch_size == ndd_batch_size - ), f"""[Output {output_idx}] Batch size mismatch: + assert pipe_batch_size == ndd_batch_size, f"""[Output {output_idx}] Batch size mismatch: Pipeline: {pipe_batch_size} NDD: {ndd_batch_size}""" # Validate data type pipe_dtype = pipe_out.dtype ndd_dtype = ndd_out.dtype.type_id - assert ( - pipe_dtype == ndd_dtype - ), f"""[Output {output_idx}] Data type mismatch: + assert pipe_dtype == ndd_dtype, f"""[Output {output_idx}] Data type mismatch: Pipeline: {pipe_dtype} NDD: {ndd_dtype}""" # Validate layout if available pipe_layout = pipe_out.layout() if hasattr(pipe_out, "layout") and pipe_out.layout() else None ndd_layout = ndd_out.layout if hasattr(ndd_out, "layout") else None - assert ( - pipe_layout == ndd_layout - ), f"""[Output {output_idx}] Layout mismatch: + assert pipe_layout == ndd_layout, f"""[Output {output_idx}] Layout mismatch: Pipeline: {pipe_layout} NDD: {ndd_layout}""" # Validate individual sample shapes for i, (psh, nddsh) in enumerate(zip(pipe_out.shape(), ndd_out.shape, strict=True)): - assert ( - psh == nddsh - ), f"""[Output {output_idx}, Sample {i}] Shape mismatch: + assert psh == nddsh, f"""[Output {output_idx}, Sample {i}] Shape mismatch: Pipeline: {psh} NDD: {nddsh}""" @@ -375,9 +365,7 @@ def compare( if isinstance(ndd_out, tuple): pipe_out_len = len(pipe_out) ndd_out_len = len(ndd_out) - assert ( - pipe_out_len == ndd_out_len - ), f"""Number of outputs mismatch: + assert pipe_out_len == ndd_out_len, f"""Number of outputs mismatch: Pipeline: {pipe_out_len} NDD: {ndd_out_len}""" @@ -616,10 +604,8 @@ def get_fn_operator(operator_name: str): try: fn_operator = get_nested_attr(fn, operator_name) except AttributeError: - raise AttributeError( - f"""Couldn't find operator {operator_name} in fn module. - Please check the string with operator name specification.""" - ) + raise AttributeError(f"""Couldn't find operator {operator_name} in fn module. + Please check the string with operator name specification.""") return fn_operator @@ -628,10 +614,8 @@ def get_ndd_operator(operator_name: str): try: ndd_operator = get_nested_attr(ndd, operator_name) except AttributeError: - raise AttributeError( - f"""Couldn't find operator {operator_name} in ndd module. - Please check the string with operator name specification.""" - ) + raise AttributeError(f"""Couldn't find operator {operator_name} in ndd module. + Please check the string with operator name specification.""") return ndd_operator diff --git a/dali/test/python/ndd_vs_fn/test_ndd_vs_fn.py b/dali/test/python/ndd_vs_fn/test_ndd_vs_fn.py index 83c879ee621..406a7d1aec2 100644 --- a/dali/test/python/ndd_vs_fn/test_ndd_vs_fn.py +++ b/dali/test/python/ndd_vs_fn/test_ndd_vs_fn.py @@ -42,7 +42,6 @@ N_ITERATIONS, ) - """ This module tests DALI operators by comparing fn API with ndd (dynamic) API outputs. diff --git a/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_1darray.py b/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_1darray.py index 8ff04560ae6..1777318edd1 100644 --- a/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_1darray.py +++ b/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_1darray.py @@ -22,7 +22,6 @@ generate_data, ) - ARRAY_1D_OPERATORS = [ OperatorTestConfig("power_spectrum"), OperatorTestConfig("preemphasis_filter"), diff --git a/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_coverage.py b/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_coverage.py index c91e0f0e1cb..22eb3204dec 100644 --- a/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_coverage.py +++ b/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_coverage.py @@ -15,7 +15,6 @@ from ndd_vs_fn_test_utils import sign_off from nvidia.dali.experimental.dynamic._ops import _all_ops - excluded_operators = [ "readers.VideoResize", # TODO(michalz): add manual tests "experimental.readers.Fits", # TODO(michalz): add tests diff --git a/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_noinput.py b/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_noinput.py index f70b926eaf9..91409795c74 100644 --- a/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_noinput.py +++ b/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_noinput.py @@ -26,7 +26,6 @@ ) from test_ndd_vs_fn_coverage import sign_off - NO_INPUT_OPERATORS = [ OperatorTestConfig("transforms.translation", {"offset": (2, 3)}), OperatorTestConfig("transforms.scale", {"scale": (2, 3)}), diff --git a/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_random.py b/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_random.py index 1c276c0720f..aacd707b107 100644 --- a/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_random.py +++ b/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_random.py @@ -34,7 +34,6 @@ import nvidia.dali.fn as fn import nvidia.dali.experimental.dynamic as ndd - RANDOM_OPERATORS_1D_ARRAY = [ OperatorTestConfig("random.choice"), OperatorTestConfig("random.normal"), diff --git a/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_sequence.py b/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_sequence.py index 5327ef4526a..32ef41005c9 100644 --- a/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_sequence.py +++ b/dali/test/python/ndd_vs_fn/test_ndd_vs_fn_sequence.py @@ -24,7 +24,6 @@ generate_data, ) - SEQUENCE_OPERATORS = [ OperatorTestConfig("cast", {"dtype": types.INT32}), OperatorTestConfig("copy"), diff --git a/dali/test/python/operator_1/test_debayer.py b/dali/test/python/operator_1/test_debayer.py index 6f880f50e73..e7bdfb28cd7 100644 --- a/dali/test/python/operator_1/test_debayer.py +++ b/dali/test/python/operator_1/test_debayer.py @@ -134,7 +134,7 @@ def get_test_data(cls, dtype): ) ) def test_debayer_fixed_pattern(self, i, args): - (batch_size, pattern, device, debayer_op) = args + batch_size, pattern, device, debayer_op = args num_iterations = 3 test_hwc_single_channel_input = i % 2 == 1 bayered_imgs, npp_baseline = self.get_test_data(np.uint8) diff --git a/dali/test/python/operator_1/test_laplacian.py b/dali/test/python/operator_1/test_laplacian.py index 085976037f3..23504a18f89 100644 --- a/dali/test/python/operator_1/test_laplacian.py +++ b/dali/test/python/operator_1/test_laplacian.py @@ -25,7 +25,6 @@ from sequences_test_utils import video_suite_helper, ArgCb from test_utils import get_dali_extra_path, check_batch, RandomlyShapedDataIterator - data_root = get_dali_extra_path() images_dir = os.path.join(data_root, "db", "single", "jpeg") @@ -107,7 +106,7 @@ def get_cv2_kernel(win_size, smoothing): return sum(outer(*ws) for ws in windows) pipe = pipeline(num_threads=4, batch_size=batch_size, device_id=0) - (kernels, scales) = pipe.run() + kernels, scales = pipe.run() if device == "gpu": kernels = kernels.as_cpu() kernels = [np.array(ker)[(slice(1, -1),) * num_dims] for ker in kernels] diff --git a/dali/test/python/operator_1/test_multipaste.py b/dali/test/python/operator_1/test_multipaste.py index 43df99b9f31..d15e00f27ca 100644 --- a/dali/test/python/operator_1/test_multipaste.py +++ b/dali/test/python/operator_1/test_multipaste.py @@ -19,7 +19,6 @@ from nose_utils import assert_raises from nose2.tools import params, cartesian_params - DEBUG_LVL = 0 SHOW_IMAGES = False diff --git a/dali/test/python/operator_1/test_one_hot.py b/dali/test/python/operator_1/test_one_hot.py index ff52397ce20..8898f2d9592 100644 --- a/dali/test/python/operator_1/test_one_hot.py +++ b/dali/test/python/operator_1/test_one_hot.py @@ -20,7 +20,6 @@ from nose_utils import raises import numpy as np - num_classes = 20 batch_size = 10 @@ -122,7 +121,7 @@ def check_one_hot_operator( axis_name=axis_name, device=device, ) - (outputs, input_batch) = pipeline.run() + outputs, input_batch = pipeline.run() if device == "gpu": input_batch = input_batch.as_cpu() input_batch = list(map(np.array, input_batch)) diff --git a/dali/test/python/operator_1/test_operator_exception.py b/dali/test/python/operator_1/test_operator_exception.py index c7808e77256..99ae3777204 100644 --- a/dali/test/python/operator_1/test_operator_exception.py +++ b/dali/test/python/operator_1/test_operator_exception.py @@ -33,8 +33,7 @@ def pipe(): with assert_raises( RuntimeError, - glob=( - """Critical error when building pipeline: + glob=("""Critical error when building pipeline: Error in CPU operator `nvidia.dali.fn.throw_exception`, which was used in the pipeline definition with the following traceback: @@ -44,8 +43,7 @@ def pipe(): encountered: Error in constructor -Current pipeline object is no longer valid.""" - ), +Current pipeline object is no longer valid."""), ): p = pipe() p.run() @@ -59,8 +57,7 @@ def pipe(): with assert_raises( error, - glob=( - """Critical error in pipeline: + glob=("""Critical error in pipeline: Error in CPU operator `nvidia.dali.fn.throw_exception`, which was used in the pipeline definition with the following traceback: @@ -70,8 +67,7 @@ def pipe(): encountered: Test message -Current pipeline object is no longer valid.""" - ), +Current pipeline object is no longer valid."""), ): p = pipe() p.run() @@ -95,8 +91,7 @@ def pipe(): with assert_raises( error_type, - glob=( - """Critical error in pipeline: + glob=("""Critical error in pipeline: Error in CPU operator `nvidia.dali.fn.throw_exception`, which was used in the pipeline definition with the following traceback: @@ -106,8 +101,7 @@ def pipe(): encountered: Test message -Current pipeline object is no longer valid.""" - ), +Current pipeline object is no longer valid."""), ): p = pipe() p.run() @@ -120,8 +114,7 @@ def pipe(): with assert_raises( RuntimeError, - glob=( - """Critical error in pipeline: + glob=("""Critical error in pipeline: Error in CPU operator `nvidia.dali.fn.throw_exception`, which was used in the pipeline definition with the following traceback: @@ -131,8 +124,7 @@ def pipe(): encountered: Unknown critical error. -Current pipeline object is no longer valid.""" - ), +Current pipeline object is no longer valid."""), ): p = pipe() p.run() @@ -147,8 +139,7 @@ def pipe(): with assert_raises( RuntimeError, - glob=( - """Critical error in pipeline: + glob=("""Critical error in pipeline: Error in CPU operator `nvidia.dali.math.add`, which was used in the pipeline definition with the following traceback: @@ -162,8 +153,7 @@ def pipe(): 3 x 2 (d=1, belonging to sample_idx=0) *C++ context: *broadcasting* -Current pipeline object is no longer valid.""" - ), +Current pipeline object is no longer valid."""), ): p = pipe() p.run() @@ -178,8 +168,7 @@ def pipe(): with assert_raises( RuntimeError, - glob=( - """Critical error in pipeline: + glob=("""Critical error in pipeline: Error in CPU operator `nvidia.dali.math.atan2`, which was used in the pipeline definition with the following traceback: @@ -193,8 +182,7 @@ def pipe(): 3 x 2 (d=1, belonging to sample_idx=0) *C++ context: *broadcasting* -Current pipeline object is no longer valid.""" - ), +Current pipeline object is no longer valid."""), ): p = pipe() p.run() @@ -213,8 +201,7 @@ def non_scalar_condition(): with assert_raises( RuntimeError, - glob=( - """Critical error in pipeline: + glob=("""Critical error in pipeline: Error in CPU operator `nvidia.dali.fn._conditional.split`, which was used in the pipeline definition with the following traceback: @@ -223,8 +210,7 @@ def non_scalar_condition(): encountered: -*Assert on "dim == 0" failed: Conditions inside `if` statements are restricted to scalar*""" - ), +*Assert on "dim == 0" failed: Conditions inside `if` statements are restricted to scalar*"""), ): pipe = non_scalar_condition() pipe.run() @@ -242,8 +228,7 @@ def non_scalar_condition(): with assert_raises( RuntimeError, - glob=( - """Critical error in pipeline: + glob=("""Critical error in pipeline: Error in CPU operator `nvidia.dali.fn._conditional.merge`, which was used in the pipeline definition with the following traceback: @@ -252,8 +237,7 @@ def non_scalar_condition(): encountered: -*Assert on "base_input.type() == input.type()" failed: Divergent data found*""" - ), +*Assert on "base_input.type() == input.type()" failed: Divergent data found*"""), ): pipe = non_scalar_condition() pipe.run() diff --git a/dali/test/python/operator_1/test_operator_origin_trace.py b/dali/test/python/operator_1/test_operator_origin_trace.py index 9fc23207588..7c7c5bfce39 100644 --- a/dali/test/python/operator_1/test_operator_origin_trace.py +++ b/dali/test/python/operator_1/test_operator_origin_trace.py @@ -25,7 +25,6 @@ from test_utils import load_test_operator_plugin from nvidia.dali.pipeline.experimental import pipeline_def as pipeline_def_experimental - load_test_operator_plugin() diff --git a/dali/test/python/operator_1/test_peek_image_shape.py b/dali/test/python/operator_1/test_peek_image_shape.py index e185bbab4ce..a6335248da0 100644 --- a/dali/test/python/operator_1/test_peek_image_shape.py +++ b/dali/test/python/operator_1/test_peek_image_shape.py @@ -36,7 +36,7 @@ def run_decode(data_path, out_type): length = pipe.reader_meta(name="reader")["epoch_size"] while samples < length: samples += batch_size - (images, decoded_shape, raw_shape) = pipe.run() + images, decoded_shape, raw_shape = pipe.run() for i in range(batch_size): # as we are asking for a particular color space it may # differ from the source image, so don't compare it diff --git a/dali/test/python/operator_1/test_slice.py b/dali/test/python/operator_1/test_slice.py index 70b2666c199..a78235c010d 100644 --- a/dali/test/python/operator_1/test_slice.py +++ b/dali/test/python/operator_1/test_slice.py @@ -124,7 +124,7 @@ def iter_setup(self): data = self.iterator.next() self.feed_input(self.data, data, layout=self.layout) - (crop_pos, crop_size) = self.pos_size_iter.next() + crop_pos, crop_size = self.pos_size_iter.next() self.feed_input(self.crop_pos, crop_pos) self.feed_input(self.crop_size, crop_size) @@ -185,7 +185,7 @@ def define_graph(self): return images def iter_setup(self): - (crop_pos, crop_size) = self.pos_size_iter.next() + crop_pos, crop_size = self.pos_size_iter.next() self.feed_input(self.crop_pos, crop_pos) self.feed_input(self.crop_size, crop_size) @@ -360,7 +360,7 @@ def iter_setup(self): data = self.iterator.next() self.feed_input(self.data, data, layout=self.layout) - (crop_pos, crop_size) = self.pos_size_iter.next() + crop_pos, crop_size = self.pos_size_iter.next() self.feed_input(self.crop_pos, crop_pos) self.feed_input(self.crop_size, crop_size) @@ -405,7 +405,7 @@ def define_graph(self): return out def iter_setup(self): - (crop_pos, crop_size) = self.pos_size_iter.next() + crop_pos, crop_size = self.pos_size_iter.next() self.feed_input(self.crop_pos, crop_pos) self.feed_input(self.crop_size, crop_size) diff --git a/dali/test/python/operator_2/test_random_choice.py b/dali/test/python/operator_2/test_random_choice.py index b431aaed1ea..b1bc87a56e3 100644 --- a/dali/test/python/operator_2/test_random_choice.py +++ b/dali/test/python/operator_2/test_random_choice.py @@ -289,7 +289,7 @@ def choice_pipeline(): return interp, interp_as_int, imgs pipe = choice_pipeline() - (interp, interp_as_int, imgs) = pipe.run() + interp, interp_as_int, imgs = pipe.run() assert interp.dtype == types.DALIDataType.INTERP_TYPE for i in range(batch_size): check_sample( diff --git a/dali/test/python/operator_2/test_random_object_bbox.py b/dali/test/python/operator_2/test_random_object_bbox.py index 5a5d1981f36..2435a862bc5 100644 --- a/dali/test/python/operator_2/test_random_object_bbox.py +++ b/dali/test/python/operator_2/test_random_object_bbox.py @@ -24,7 +24,6 @@ import random from test_utils import check_batch, np_type_to_dali - np.random.seed(1234) random.seed(1234) diff --git a/dali/test/python/operator_2/test_rotate.py b/dali/test/python/operator_2/test_rotate.py index b98f0dc9499..318f5d6c297 100644 --- a/dali/test/python/operator_2/test_rotate.py +++ b/dali/test/python/operator_2/test_rotate.py @@ -32,7 +32,6 @@ sequence_suite_helper, ) - test_data_root = os.environ["DALI_EXTRA_PATH"] caffe_db_folder = os.path.join(test_data_root, "db", "lmdb") @@ -40,7 +39,7 @@ def get_output_size(angle, input_size, parity_correction=True): cosa = abs(math.cos(angle)) sina = abs(math.sin(angle)) - (h, w) = input_size[0:2] + h, w = input_size[0:2] eps = 1e-2 out_w = int(math.ceil(w * cosa + h * sina - eps)) out_h = int(math.ceil(h * cosa + w * sina - eps)) @@ -107,8 +106,8 @@ def get_3d_output_size(angle, axis, input_size, parity_correction=False): def get_transform(angle, input_size, output_size): cosa = math.cos(angle) sina = math.sin(angle) - (out_h, out_w) = output_size[0:2] - (in_h, in_w) = input_size[0:2] + out_h, out_w = output_size[0:2] + in_h, in_w = input_size[0:2] t1 = np.array([[1, 0, -out_w * 0.5], [0, 1, -out_h * 0.5], [0, 0, 1]]) r = np.array([[cosa, -sina, 0], [sina, cosa, 0], [0, 0, 1]]) t2 = np.array([[1, 0, in_w * 0.5], [0, 1, in_h * 0.5], [0, 0, 1]]) diff --git a/dali/test/python/operator_2/test_spectrogram.py b/dali/test/python/operator_2/test_spectrogram.py index a0cdffdaaa8..9b9ca2c82c8 100644 --- a/dali/test/python/operator_2/test_spectrogram.py +++ b/dali/test/python/operator_2/test_spectrogram.py @@ -28,7 +28,6 @@ from nose2.tools import params from nose_utils import assert_raises - audio_files = get_files("db/audio/wav", "wav") diff --git a/dali/test/python/reader/test_numpy.py b/dali/test/python/reader/test_numpy.py index f443033c156..643511ca491 100644 --- a/dali/test/python/reader/test_numpy.py +++ b/dali/test/python/reader/test_numpy.py @@ -27,7 +27,6 @@ from nose2.tools import params, cartesian_params from test_utils import compare_pipelines, to_array - gds_data_root = "/scratch/" if not os.path.isdir(gds_data_root): gds_data_root = os.getcwd() + "/scratch/" @@ -1082,7 +1081,7 @@ def test_shuffling(shuffling, pad_last_batch): pipe.set_outputs(data_cpu, data_gpu) for _ in range(num_samples // batch_size * 2): - (cpu_arr, gpu_arr) = pipe.run() + cpu_arr, gpu_arr = pipe.run() assert_array_equal(to_array(cpu_arr), to_array(gpu_arr)) diff --git a/dali/test/python/sequences_test_utils.py b/dali/test/python/sequences_test_utils.py index e02f663f7f9..4304e2f5401 100644 --- a/dali/test/python/sequences_test_utils.py +++ b/dali/test/python/sequences_test_utils.py @@ -24,7 +24,6 @@ from test_utils import get_dali_extra_path, check_batch - data_root = get_dali_extra_path() vid_file = os.path.join(data_root, "db", "video", "sintel", "sintel_trailer-720p.mp4") diff --git a/dali/test/python/test_RN50_external_source_parallel_data.py b/dali/test/python/test_RN50_external_source_parallel_data.py index 937597b4d99..5518fb6bff4 100644 --- a/dali/test/python/test_RN50_external_source_parallel_data.py +++ b/dali/test/python/test_RN50_external_source_parallel_data.py @@ -24,7 +24,6 @@ get_pipe_factories, ) - # This test requires significant amount of shared memory to be able to pass # the batches between worker processes and the main process. If running in docker # make sure that -shm-size is big enough. diff --git a/dali/test/python/test_dali_cpu_only.py b/dali/test/python/test_dali_cpu_only.py index 4b2bfd2e170..888fb6597ec 100644 --- a/dali/test/python/test_dali_cpu_only.py +++ b/dali/test/python/test_dali_cpu_only.py @@ -39,7 +39,6 @@ from test_utils import get_dali_extra_path, get_files, module_functions from webdataset_base import generate_temp_index_file as generate_temp_wds_index - data_root = get_dali_extra_path() images_dir = os.path.join(data_root, "db", "single", "jpeg") audio_files = get_files(os.path.join("db", "audio", "wav"), "wav") diff --git a/dali/test/python/test_dali_tf_dataset_eager.py b/dali/test/python/test_dali_tf_dataset_eager.py index ee3b15ec716..a8e64e4422b 100644 --- a/dali/test/python/test_dali_tf_dataset_eager.py +++ b/dali/test/python/test_dali_tf_dataset_eager.py @@ -49,7 +49,6 @@ import random as random import itertools - tf.compat.v1.enable_eager_execution() diff --git a/dali/test/python/test_dali_tf_dataset_mnist.py b/dali/test/python/test_dali_tf_dataset_mnist.py index 556bb4fafde..c2adaed757a 100644 --- a/dali/test/python/test_dali_tf_dataset_mnist.py +++ b/dali/test/python/test_dali_tf_dataset_mnist.py @@ -24,7 +24,6 @@ from packaging.version import Version from nose_utils import SkipTest - TARGET = 0.8 BATCH_SIZE = 64 DROPOUT = 0.2 diff --git a/dali/test/python/test_dali_tf_exec2.py b/dali/test/python/test_dali_tf_exec2.py index 3c015cf3e50..4593bdc3e98 100644 --- a/dali/test/python/test_dali_tf_exec2.py +++ b/dali/test/python/test_dali_tf_exec2.py @@ -23,7 +23,6 @@ from test_utils_tensorflow import skip_inputs_for_incompatible_tf from test_utils import get_dali_extra_path - test_data_root = get_dali_extra_path() lmdb_folder = os.path.join(test_data_root, "db", "lmdb") diff --git a/dali/test/python/test_external_source_cupy.py b/dali/test/python/test_external_source_cupy.py index 14a06b82536..23759b02cf9 100644 --- a/dali/test/python/test_external_source_cupy.py +++ b/dali/test/python/test_external_source_cupy.py @@ -28,14 +28,12 @@ from nvidia.dali.tensors import TensorGPU import numpy as np - use_cupy() # extra tests, GPU-specific import cupy as cp # noqa:E402 - we need to call this after use_cupy() - assert nvidia.dali.types._is_cupy_array(cp.array([1, 2, 3])), "CuPy array not recognized" diff --git a/dali/test/python/test_external_source_dali.py b/dali/test/python/test_external_source_dali.py index 5854cd86794..0037fce921c 100644 --- a/dali/test/python/test_external_source_dali.py +++ b/dali/test/python/test_external_source_dali.py @@ -885,7 +885,7 @@ def test_pipe(): return src_0, src_1, src_2 pipe = test_pipe() - (out0, out1, out2) = pipe.run() + out0, out1, out2 = pipe.run() np.array_equal(np.array(out0.as_tensor()), np.array([0, 1, 2, 3])) np.array_equal(np.array(out1.as_tensor()), np.array([2, 3, 4, 5])) np.array_equal(np.array(out2.as_tensor()), np.array([42, 42, 42, 42])) diff --git a/dali/test/python/test_external_source_parallel.py b/dali/test/python/test_external_source_parallel.py index f74346a1b9f..2aa6bd31ccd 100644 --- a/dali/test/python/test_external_source_parallel.py +++ b/dali/test/python/test_external_source_parallel.py @@ -640,7 +640,7 @@ def pipeline(): i = 0 while True: try: - (sample_outs, batch_outs, iter_outs) = pipe.run() + sample_outs, batch_outs, iter_outs = pipe.run() assert len(sample_outs) == len( batch_outs ), f"Batch length mismatch: sample: {len(sample_outs)}, batch: {len(batch_outs)}" diff --git a/dali/test/python/test_fw_iterators.py b/dali/test/python/test_fw_iterators.py index 13d8f8cbb39..d8104b7ba5a 100644 --- a/dali/test/python/test_fw_iterators.py +++ b/dali/test/python/test_fw_iterators.py @@ -542,7 +542,7 @@ def check_mxnet_iterator_pass_reader_name( epoch_counter, rounded_shard_size, ) - (ids, sample_counter, per_gpu_counter, epoch_counter, rounded_shard_size) = ret + ids, sample_counter, per_gpu_counter, epoch_counter, rounded_shard_size = ret @attr("mxnet") @@ -854,7 +854,7 @@ def check_gluon_iterator_pass_reader_name( epoch_counter, rounded_shard_size, ) - (ids, sample_counter, per_gpu_counter, epoch_counter, rounded_shard_size) = ret + ids, sample_counter, per_gpu_counter, epoch_counter, rounded_shard_size = ret @attr("gluon") @@ -1560,7 +1560,7 @@ def check_pytorch_iterator_pass_reader_name( epoch_counter, rounded_shard_size, ) - (ids, sample_counter, per_gpu_counter, epoch_counter, rounded_shard_size) = ret + ids, sample_counter, per_gpu_counter, epoch_counter, rounded_shard_size = ret @attr("pytorch") @@ -1842,7 +1842,7 @@ def check_paddle_iterator_pass_reader_name( epoch_counter, rounded_shard_size, ) - (ids, sample_counter, per_gpu_counter, epoch_counter, rounded_shard_size) = ret + ids, sample_counter, per_gpu_counter, epoch_counter, rounded_shard_size = ret @attr("paddle") diff --git a/dali/test/python/test_pipeline.py b/dali/test/python/test_pipeline.py index a2b4b636f38..1f3a915ff9a 100644 --- a/dali/test/python/test_pipeline.py +++ b/dali/test/python/test_pipeline.py @@ -1521,7 +1521,7 @@ def iter_setup(self): data = self.iterator.next() self.feed_input(self.data, data, layout=self.layout) - (crop_pos, crop_size) = self.pos_size_iter.next() + crop_pos, crop_size = self.pos_size_iter.next() self.feed_input(self.crop_pos, crop_pos) self.feed_input(self.crop_size, crop_size) diff --git a/dali/test/python/test_pipeline_debug.py b/dali/test/python/test_pipeline_debug.py index 3cdf5cf3930..d175c999c53 100644 --- a/dali/test/python/test_pipeline_debug.py +++ b/dali/test/python/test_pipeline_debug.py @@ -31,7 +31,6 @@ _impl_uninitialized, ) - file_root = os.path.join(get_dali_extra_path(), "db/single/jpeg") diff --git a/dali/test/python/test_plugin_manager.py b/dali/test/python/test_plugin_manager.py index e58a1ea77a7..04b8d1d8303 100644 --- a/dali/test/python/test_plugin_manager.py +++ b/dali/test/python/test_plugin_manager.py @@ -65,7 +65,7 @@ def define_graph(self): return (self.images, custom_dummy_out) def iter_setup(self): - (images, labels) = iterator.next() + images, labels = iterator.next() self.feed_input(self.images, images) diff --git a/dali/test/python/test_tensor_formatting.py b/dali/test/python/test_tensor_formatting.py index c15a8c2bab4..ce6b972f580 100644 --- a/dali/test/python/test_tensor_formatting.py +++ b/dali/test/python/test_tensor_formatting.py @@ -20,7 +20,6 @@ # Import Dynamic API import nvidia.dali.experimental.dynamic as ndd - # Test case definitions for parameterized tests TENSOR_TEST_CASES = [ { diff --git a/dali/test/python/test_utils.py b/dali/test/python/test_utils.py index 05359828171..363d699c94f 100644 --- a/dali/test/python/test_utils.py +++ b/dali/test/python/test_utils.py @@ -29,7 +29,6 @@ from packaging.version import Version from nose_utils import SkipTest - is_of_supported_var = None diff --git a/dali/test/python/test_video_pipeline.py b/dali/test/python/test_video_pipeline.py index 63dff541e9e..01c5d2121d2 100644 --- a/dali/test/python/test_video_pipeline.py +++ b/dali/test/python/test_video_pipeline.py @@ -583,7 +583,7 @@ def video_decoder_pipeline(): pipe = video_decoder_pipeline(batch_size=batch_size, num_threads=1, device_id=0) for _ in range(iter): - (a, b) = pipe.run() + a, b = pipe.run() a = a.as_cpu().as_array() b = b.as_cpu().as_array() absdiff = np.abs(a.astype(int) - b.astype(int)) diff --git a/docs/examples/custom_operations/jax_operator_multi_gpu_process_1.py b/docs/examples/custom_operations/jax_operator_multi_gpu_process_1.py index 9dcc19a1a8c..c2d3365fc65 100644 --- a/docs/examples/custom_operations/jax_operator_multi_gpu_process_1.py +++ b/docs/examples/custom_operations/jax_operator_multi_gpu_process_1.py @@ -28,7 +28,6 @@ from nvidia.dali.plugin.jax import data_iterator from nvidia.dali.plugin.jax.fn import jax_function - os.environ["CUDA_VISIBLE_DEVICES"] = "1" jax.distributed.initialize( diff --git a/docs/examples/frameworks/jax/flax-basic_example.ipynb b/docs/examples/frameworks/jax/flax-basic_example.ipynb index 1fba536fdfe..acba0864124 100644 --- a/docs/examples/frameworks/jax/flax-basic_example.ipynb +++ b/docs/examples/frameworks/jax/flax-basic_example.ipynb @@ -65,7 +65,6 @@ "\n", "from nvidia.dali.plugin.jax import data_iterator\n", "\n", - "\n", "batch_size = 50\n", "image_size = 28\n", "num_classes = 10\n", diff --git a/docs/examples/frameworks/jax/jax-basic_example.ipynb b/docs/examples/frameworks/jax/jax-basic_example.ipynb index f0c5314b342..dd33cb32cc2 100644 --- a/docs/examples/frameworks/jax/jax-basic_example.ipynb +++ b/docs/examples/frameworks/jax/jax-basic_example.ipynb @@ -64,7 +64,6 @@ "import nvidia.dali.fn as fn\n", "import nvidia.dali.types as types\n", "\n", - "\n", "batch_size = 200\n", "image_size = 28\n", "num_classes = 10\n", diff --git a/docs/examples/frameworks/jax/jax-multigpu_example.ipynb b/docs/examples/frameworks/jax/jax-multigpu_example.ipynb index c1514625726..3bdaef98e65 100644 --- a/docs/examples/frameworks/jax/jax-multigpu_example.ipynb +++ b/docs/examples/frameworks/jax/jax-multigpu_example.ipynb @@ -42,7 +42,6 @@ "from jax.sharding import PositionalSharding, Mesh\n", "from jax.experimental import mesh_utils\n", "\n", - "\n", "mesh = mesh_utils.create_device_mesh((jax.device_count(), 1))\n", "sharding = PositionalSharding(mesh)\n", "\n", @@ -73,7 +72,6 @@ "import nvidia.dali.fn as fn\n", "import nvidia.dali.types as types\n", "\n", - "\n", "image_size = 28\n", "num_classes = 10\n", "\n", @@ -355,7 +353,6 @@ "import jax.numpy as jnp\n", "from model import init_model, accuracy\n", "\n", - "\n", "model = init_model()\n", "model = jax.tree_map(lambda x: jnp.array([x] * jax.device_count()), model)" ] @@ -401,7 +398,6 @@ "source": [ "from model import update_parallel\n", "\n", - "\n", "for epoch in range(num_epochs):\n", " for it, batch in enumerate(training_iterator):\n", " model = update_parallel(model, batch)\n", diff --git a/docs/examples/frameworks/jax/model.py b/docs/examples/frameworks/jax/model.py index c1712d89d8e..689cfe5d960 100644 --- a/docs/examples/frameworks/jax/model.py +++ b/docs/examples/frameworks/jax/model.py @@ -18,7 +18,6 @@ from jax.scipy.special import logsumexp import numpy.random as npr - layers = [784, 1024, 1024, 10] diff --git a/docs/examples/frameworks/jax/pax_examples/dali_pax_example.py b/docs/examples/frameworks/jax/pax_examples/dali_pax_example.py index acf8419a644..ffe2571a1ee 100644 --- a/docs/examples/frameworks/jax/pax_examples/dali_pax_example.py +++ b/docs/examples/frameworks/jax/pax_examples/dali_pax_example.py @@ -40,7 +40,6 @@ from pax_examples.dali_pax_input import MnistDaliInput - NestedMap = py_utils.NestedMap template_field = base_layer.template_field JTensor = pytypes.JTensor diff --git a/docs/examples/frameworks/jax/pax_examples/dali_pax_input.py b/docs/examples/frameworks/jax/pax_examples/dali_pax_input.py index 87ae3959ae3..8458e4e7ddc 100644 --- a/docs/examples/frameworks/jax/pax_examples/dali_pax_input.py +++ b/docs/examples/frameworks/jax/pax_examples/dali_pax_input.py @@ -20,7 +20,6 @@ import nvidia.dali.fn as fn import nvidia.dali.types as types - training_data_path = os.path.join( os.environ["DALI_EXTRA_PATH"], "db/MNIST/training/" ) diff --git a/docs/examples/frameworks/paddle/paddle-basic_example.ipynb b/docs/examples/frameworks/paddle/paddle-basic_example.ipynb index 6bdd44c0ab2..2d4fc636078 100644 --- a/docs/examples/frameworks/paddle/paddle-basic_example.ipynb +++ b/docs/examples/frameworks/paddle/paddle-basic_example.ipynb @@ -114,7 +114,6 @@ "import numpy as np\n", "from nvidia.dali.plugin.paddle import DALIGenericIterator\n", "\n", - "\n", "label_range = (0, 999)\n", "pipes = [\n", " caffe_pipeline(\n", diff --git a/docs/examples/frameworks/paddle/paddle-various-readers.ipynb b/docs/examples/frameworks/paddle/paddle-various-readers.ipynb index 9937374dbfc..29d14c4dbc8 100644 --- a/docs/examples/frameworks/paddle/paddle-various-readers.ipynb +++ b/docs/examples/frameworks/paddle/paddle-various-readers.ipynb @@ -265,7 +265,6 @@ "import numpy as np\n", "from nvidia.dali.plugin.paddle import DALIGenericIterator\n", "\n", - "\n", "pipe_types = [\n", " [mxnet_reader_pipeline, (0, 999)],\n", " [caffe_reader_pipeline, (0, 999)],\n", diff --git a/docs/examples/frameworks/pytorch/pytorch-basic_example.ipynb b/docs/examples/frameworks/pytorch/pytorch-basic_example.ipynb index 5b64d0fa6ba..7621a6d7f4b 100644 --- a/docs/examples/frameworks/pytorch/pytorch-basic_example.ipynb +++ b/docs/examples/frameworks/pytorch/pytorch-basic_example.ipynb @@ -115,7 +115,6 @@ "import numpy as np\n", "from nvidia.dali.plugin.pytorch import DALIGenericIterator\n", "\n", - "\n", "label_range = (0, 999)\n", "pipes = [\n", " caffe_pipeline(\n", diff --git a/docs/examples/frameworks/pytorch/pytorch-various-readers.ipynb b/docs/examples/frameworks/pytorch/pytorch-various-readers.ipynb index 3c9844a9e99..ac932af0fb9 100644 --- a/docs/examples/frameworks/pytorch/pytorch-various-readers.ipynb +++ b/docs/examples/frameworks/pytorch/pytorch-various-readers.ipynb @@ -265,7 +265,6 @@ "import numpy as np\n", "from nvidia.dali.plugin.pytorch import DALIGenericIterator\n", "\n", - "\n", "pipe_types = [\n", " [mxnet_reader_pipeline, (0, 999)],\n", " [caffe_reader_pipeline, (0, 999)],\n", diff --git a/docs/examples/image_processing/3d_transforms.ipynb b/docs/examples/image_processing/3d_transforms.ipynb index 8b7bd4ddfc9..5164431ee01 100644 --- a/docs/examples/image_processing/3d_transforms.ipynb +++ b/docs/examples/image_processing/3d_transforms.ipynb @@ -164,7 +164,7 @@ "\n", "def perspective(volume, max_size):\n", " window_size = 300\n", - " (h, w) = volume.shape[1:3]\n", + " h, w = volume.shape[1:3]\n", " if h > w:\n", " w = window_size * w // h\n", " h = window_size\n", diff --git a/docs/examples/image_processing/color_space_conversion/pipeline_mode.ipynb b/docs/examples/image_processing/color_space_conversion/pipeline_mode.ipynb index 48e7e857040..6d954d12b1e 100644 --- a/docs/examples/image_processing/color_space_conversion/pipeline_mode.ipynb +++ b/docs/examples/image_processing/color_space_conversion/pipeline_mode.ipynb @@ -26,7 +26,6 @@ "import nvidia.dali.fn as fn\n", "import nvidia.dali.types as types\n", "\n", - "\n", "image_dir = \"../../data/images\"\n", "\n", "conversions = [\n", diff --git a/docs/examples/legacy_getting_started.ipynb b/docs/examples/legacy_getting_started.ipynb index ba08a475553..3d3d2f314a9 100644 --- a/docs/examples/legacy_getting_started.ipynb +++ b/docs/examples/legacy_getting_started.ipynb @@ -63,7 +63,6 @@ "import os.path\n", "import fnmatch\n", "\n", - "\n", "for root, dir, files in os.walk(\"data/images\"):\n", " depth = root.count(\"/\")\n", " ret = \"\"\n", diff --git a/docs/examples/sequence_processing/video/video_per_frame_arguments/pipeline_mode.ipynb b/docs/examples/sequence_processing/video/video_per_frame_arguments/pipeline_mode.ipynb index 7f52ad75342..118def02ad2 100644 --- a/docs/examples/sequence_processing/video/video_per_frame_arguments/pipeline_mode.ipynb +++ b/docs/examples/sequence_processing/video/video_per_frame_arguments/pipeline_mode.ipynb @@ -606,7 +606,7 @@ "source": [ "pipe = pipeline(batch_size=batch_size, device_id=0, num_threads=4)\n", "pipe.build()\n", - "(vid, coord_transformed) = pipe.run()\n", + "vid, coord_transformed = pipe.run()\n", "display_video(vid.as_cpu().as_array(), points=coord_transformed)" ] } diff --git a/docs/examples/zoo/images/decode_and_transfrom_from_json.py b/docs/examples/zoo/images/decode_and_transfrom_from_json.py index c5458a330ea..c41dc794b87 100644 --- a/docs/examples/zoo/images/decode_and_transfrom_from_json.py +++ b/docs/examples/zoo/images/decode_and_transfrom_from_json.py @@ -27,7 +27,6 @@ import nvidia.dali.types as types import nvidia.dali.plugin.pytorch as dali_pytorch - batch_size = 4 diff --git a/docs/examples/zoo/videos/decode_and_transform_from_json.py b/docs/examples/zoo/videos/decode_and_transform_from_json.py index beffba7c7fe..3e6283a0c62 100644 --- a/docs/examples/zoo/videos/decode_and_transform_from_json.py +++ b/docs/examples/zoo/videos/decode_and_transform_from_json.py @@ -29,7 +29,6 @@ import nvidia.dali.plugin.pytorch as dali_pytorch from nvidia.dali.plugin.base_iterator import LastBatchPolicy as LastBatchPolicy - batch_size = 4 diff --git a/internal_tools/hw_decoder_bench.py b/internal_tools/hw_decoder_bench.py index 151878fda7d..14200f15951 100644 --- a/internal_tools/hw_decoder_bench.py +++ b/internal_tools/hw_decoder_bench.py @@ -24,7 +24,6 @@ from nvidia.dali.pipeline import pipeline_def import nvidia.dali.experimental.dynamic as ndd - parser = argparse.ArgumentParser(description="DALI HW decoder benchmark") parser.add_argument("-b", dest="batch_size", help="batch size", default=1, type=int) parser.add_argument("-m", dest="minibatch_size", help="minibatch size", default=32, type=int) diff --git a/internal_tools/spelling/common.py b/internal_tools/spelling/common.py index 83c6425aaa3..6944ce00ccd 100644 --- a/internal_tools/spelling/common.py +++ b/internal_tools/spelling/common.py @@ -17,7 +17,6 @@ import re from dataclasses import dataclass - CWD = os.getcwd() CSPELL_DOCKER = "ghcr.io/streetsidesoftware/cspell:latest" CSPELL_COMMAND = f"docker run -it -v {CWD}:/workdir {CSPELL_DOCKER} \ From fbed4e7e7fb435cf4ba0d432fe0cb99e1ec17909 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 4 Mar 2026 08:08:30 +0100 Subject: [PATCH 6/9] Fix deprecated benchmark::internal::Benchmark usage for benchmark v1.9.5 --- dali/benchmark/caffe2_alexnet_bench.cc | 4 ++-- dali/benchmark/caffe_alexnet_bench.cc | 4 ++-- dali/benchmark/checkpointing_bench.cc | 2 +- dali/benchmark/copy_bench.cc | 2 +- dali/benchmark/crop_bench.cc | 4 ++-- dali/benchmark/crop_mirror_normalize_bench.cc | 4 ++-- dali/benchmark/decoder_bench.cc | 2 +- dali/benchmark/file_reader_alexnet_bench.cc | 2 +- dali/benchmark/file_reader_fast_forward_bench.cc | 2 +- dali/benchmark/flip_bench.cc | 2 +- dali/benchmark/gaussian_blur_bench.cc | 2 +- dali/benchmark/one_hot_bench.cc | 4 ++-- dali/benchmark/resnet50_bench.cc | 6 +++--- dali/benchmark/resnet50_nvjpeg_bench.cc | 2 +- dali/benchmark/slice_kernel_bench.cc | 6 +++--- dali/benchmark/slice_kernel_bench.cu | 6 +++--- dali/benchmark/thread_pool_bench.cc | 2 +- dali/benchmark/transpose_bench.cc | 2 +- dali/benchmark/transpose_cpu_bench.cc | 2 +- dali/benchmark/warp_affine_bench.cc | 4 ++-- 20 files changed, 32 insertions(+), 32 deletions(-) diff --git a/dali/benchmark/caffe2_alexnet_bench.cc b/dali/benchmark/caffe2_alexnet_bench.cc index d09b5ae1916..da6d2103d8d 100755 --- a/dali/benchmark/caffe2_alexnet_bench.cc +++ b/dali/benchmark/caffe2_alexnet_bench.cc @@ -130,7 +130,7 @@ BENCHMARK_DEFINE_F(C2Alexnet, Caffe2Pipe)(benchmark::State& st) { // NOLINT benchmark::Counter::kIsRate); } -static void PipeArgs(benchmark::internal::Benchmark *b) { +static void PipeArgs(benchmark::Benchmark *b) { for (int executor = 2; executor < 3; ++executor) { for (int fast_resize = 0; fast_resize < 2; ++fast_resize) { for (int batch_size = 128; batch_size <= 128; batch_size += 32) { @@ -258,7 +258,7 @@ BENCHMARK_DEFINE_F(C2Alexnet, HybridPipe)(benchmark::State& st) { // NOLINT benchmark::Counter::kIsRate); } -static void HybridPipeArgs(benchmark::internal::Benchmark *b) { +static void HybridPipeArgs(benchmark::Benchmark *b) { for (int executor = 0; executor < 3; ++executor) { for (int batch_size = 128; batch_size <= 128; batch_size += 32) { for (int num_thread = 1; num_thread <= 4; ++num_thread) { diff --git a/dali/benchmark/caffe_alexnet_bench.cc b/dali/benchmark/caffe_alexnet_bench.cc index bc1e1db789d..018a3fd032d 100755 --- a/dali/benchmark/caffe_alexnet_bench.cc +++ b/dali/benchmark/caffe_alexnet_bench.cc @@ -132,7 +132,7 @@ BENCHMARK_DEFINE_F(Alexnet, CaffePipe)(benchmark::State& st) { // NOLINT benchmark::Counter::kIsRate); } -static void PipeArgs(benchmark::internal::Benchmark *b) { +static void PipeArgs(benchmark::Benchmark *b) { for (int executor = 2; executor < 3; ++executor) { for (int fast_resize = 0; fast_resize < 2; ++fast_resize) { for (int batch_size = 128; batch_size <= 128; batch_size += 32) { @@ -260,7 +260,7 @@ BENCHMARK_DEFINE_F(Alexnet, HybridPipe)(benchmark::State& st) { // NOLINT benchmark::Counter::kIsRate); } -static void HybridPipeArgs(benchmark::internal::Benchmark *b) { +static void HybridPipeArgs(benchmark::Benchmark *b) { for (int executor = 0; executor < 3; ++executor) { for (int batch_size = 128; batch_size <= 128; batch_size += 32) { for (int num_thread = 1; num_thread <= 4; ++num_thread) { diff --git a/dali/benchmark/checkpointing_bench.cc b/dali/benchmark/checkpointing_bench.cc index 5e802c7cf28..91fc903e9bc 100755 --- a/dali/benchmark/checkpointing_bench.cc +++ b/dali/benchmark/checkpointing_bench.cc @@ -79,7 +79,7 @@ class CheckpointingOverhead : public DALIBenchmark { } }; -static void Args(benchmark::internal::Benchmark *b) { +static void Args(benchmark::Benchmark *b) { const std::vector policies = { CheckpointingPolicy::Disabled, CheckpointingPolicy::Enabled, diff --git a/dali/benchmark/copy_bench.cc b/dali/benchmark/copy_bench.cc index 276dd8010dd..ef0e21e2395 100644 --- a/dali/benchmark/copy_bench.cc +++ b/dali/benchmark/copy_bench.cc @@ -18,7 +18,7 @@ namespace dali { -static void CopyGPUArgs(benchmark::internal::Benchmark *b) { +static void CopyGPUArgs(benchmark::Benchmark *b) { for (int batch_size = 256; batch_size >= 1; batch_size /= 2) { for (int H = 2000; H >= 500; H /= 2) { int W = H; diff --git a/dali/benchmark/crop_bench.cc b/dali/benchmark/crop_bench.cc index 1e2dd5fccde..10fcad0ff99 100644 --- a/dali/benchmark/crop_bench.cc +++ b/dali/benchmark/crop_bench.cc @@ -18,7 +18,7 @@ namespace dali { -static void CropCPUArgs(benchmark::internal::Benchmark *b) { +static void CropCPUArgs(benchmark::Benchmark *b) { int batch_size = 1; for (int H = 1000; H >= 500; H /= 2) { int W = H, C = 3; @@ -54,7 +54,7 @@ BENCHMARK_REGISTER_F(OperatorBench, CropCPU)->Iterations(1000) ->UseRealTime() ->Apply(CropCPUArgs); -static void CropGPUArgs(benchmark::internal::Benchmark *b) { +static void CropGPUArgs(benchmark::Benchmark *b) { for (int batch_size = 128; batch_size >= 1; batch_size /= 4) { for (int H = 2000; H >= 500; H /= 2) { int W = H, C = 3; diff --git a/dali/benchmark/crop_mirror_normalize_bench.cc b/dali/benchmark/crop_mirror_normalize_bench.cc index 531c0883579..10963b42af6 100644 --- a/dali/benchmark/crop_mirror_normalize_bench.cc +++ b/dali/benchmark/crop_mirror_normalize_bench.cc @@ -18,7 +18,7 @@ namespace dali { -static void CropMirrorNormalizeCPUArgs(benchmark::internal::Benchmark *b) { +static void CropMirrorNormalizeCPUArgs(benchmark::Benchmark *b) { int batch_size = 8; int mean = 128, std = 1; for (auto &dtype : {DALI_FLOAT}) { @@ -76,7 +76,7 @@ BENCHMARK_REGISTER_F(OperatorBench, CropMirrorNormalizeCPU)->Iterations(500) ->UseRealTime() ->Apply(CropMirrorNormalizeCPUArgs); -static void CropMirrorNormalizeGPUArgs(benchmark::internal::Benchmark *b) { +static void CropMirrorNormalizeGPUArgs(benchmark::Benchmark *b) { int mean = 128, std = 1; for (int batch_size : {1, 8, 128, 256}) { for (int H = 1000; H >= 250; H /= 2) { diff --git a/dali/benchmark/decoder_bench.cc b/dali/benchmark/decoder_bench.cc index c8037ea6735..59902e641c3 100644 --- a/dali/benchmark/decoder_bench.cc +++ b/dali/benchmark/decoder_bench.cc @@ -85,7 +85,7 @@ class DecoderBench : public DALIBenchmark { } }; -static void PipeArgs(benchmark::internal::Benchmark *b) { +static void PipeArgs(benchmark::Benchmark *b) { for (int batch_size = 1; batch_size <= 128; batch_size <<= 1) { for (int num_thread = 1; num_thread <= 4; ++num_thread) { b->Args({batch_size, num_thread}); diff --git a/dali/benchmark/file_reader_alexnet_bench.cc b/dali/benchmark/file_reader_alexnet_bench.cc index 188757bcca3..7cc956016bc 100755 --- a/dali/benchmark/file_reader_alexnet_bench.cc +++ b/dali/benchmark/file_reader_alexnet_bench.cc @@ -131,7 +131,7 @@ BENCHMARK_DEFINE_F(FileReaderAlexnet, CaffePipe)(benchmark::State& st) { // NOLI benchmark::Counter::kIsRate); } -static void PipeArgs(benchmark::internal::Benchmark *b) { +static void PipeArgs(benchmark::Benchmark *b) { for (int executor = 2; executor < 3; ++executor) { for (int fast_resize = 0; fast_resize < 2; ++fast_resize) { for (int batch_size = 128; batch_size <= 128; batch_size += 32) { diff --git a/dali/benchmark/file_reader_fast_forward_bench.cc b/dali/benchmark/file_reader_fast_forward_bench.cc index 283ff303d77..45997d20c8c 100755 --- a/dali/benchmark/file_reader_fast_forward_bench.cc +++ b/dali/benchmark/file_reader_fast_forward_bench.cc @@ -90,7 +90,7 @@ BENCHMARK_DEFINE_F(FileReaderFastForward, FastForward)(benchmark::State& st) { / } } -static void Args(benchmark::internal::Benchmark *b) { +static void Args(benchmark::Benchmark *b) { for (int dataset_size = 100; dataset_size <= 1000000; dataset_size *= 100) { for (int frac = 0; frac <= 10; frac++) { // Without random shuffle diff --git a/dali/benchmark/flip_bench.cc b/dali/benchmark/flip_bench.cc index 61c69eeca44..c43b8d40ec9 100644 --- a/dali/benchmark/flip_bench.cc +++ b/dali/benchmark/flip_bench.cc @@ -18,7 +18,7 @@ namespace dali { -static void FlipGPUArgs(benchmark::internal::Benchmark *b) { +static void FlipGPUArgs(benchmark::Benchmark *b) { for (int batch_size = 256; batch_size >= 1; batch_size /= 2) { for (int H = 2000; H >= 500; H /= 2) { int W = H, C = 3; diff --git a/dali/benchmark/gaussian_blur_bench.cc b/dali/benchmark/gaussian_blur_bench.cc index 50a584d1086..d75eae82259 100644 --- a/dali/benchmark/gaussian_blur_bench.cc +++ b/dali/benchmark/gaussian_blur_bench.cc @@ -18,7 +18,7 @@ namespace dali { -static void GaussianBlurGPUArgs(benchmark::internal::Benchmark *b) { +static void GaussianBlurGPUArgs(benchmark::Benchmark *b) { for (int batch_size = 256; batch_size >= 1; batch_size /= 2) { for (int H = 1000; H >= 500; H /= 2) { int W = H, C = 3; diff --git a/dali/benchmark/one_hot_bench.cc b/dali/benchmark/one_hot_bench.cc index c34585f350a..db2cf65929d 100644 --- a/dali/benchmark/one_hot_bench.cc +++ b/dali/benchmark/one_hot_bench.cc @@ -18,7 +18,7 @@ namespace dali { -static void OneHotGPUAlternateAxis(benchmark::internal::Benchmark *b, int batch_size, int h, int w, +static void OneHotGPUAlternateAxis(benchmark::Benchmark *b, int batch_size, int h, int w, int num_classes) { int axis = -1; b->Args({batch_size, h, w, axis, num_classes}); @@ -26,7 +26,7 @@ static void OneHotGPUAlternateAxis(benchmark::internal::Benchmark *b, int batch_ b->Args({batch_size, h, w, axis, num_classes}); } -static void OneHotGPUArgs(benchmark::internal::Benchmark *b) { +static void OneHotGPUArgs(benchmark::Benchmark *b) { for (int batch_size = 256; batch_size >= 1; batch_size /= 2) { int h = 64, w = h; int num_classes = 500; diff --git a/dali/benchmark/resnet50_bench.cc b/dali/benchmark/resnet50_bench.cc index 778262fa7ee..3176833f001 100755 --- a/dali/benchmark/resnet50_bench.cc +++ b/dali/benchmark/resnet50_bench.cc @@ -134,7 +134,7 @@ BENCHMARK_DEFINE_F(RN50, C2Pipe)(benchmark::State& st) { // NOLINT benchmark::Counter::kIsRate); } -static void PipeArgs(benchmark::internal::Benchmark *b) { +static void PipeArgs(benchmark::Benchmark *b) { for (int executor = 2; executor < 3; ++executor) { for (int fast_resize = 0; fast_resize < 2; ++fast_resize) { for (int batch_size = 128; batch_size <= 128; batch_size += 32) { @@ -266,7 +266,7 @@ BENCHMARK_DEFINE_F(RN50, HybridPipe)(benchmark::State& st) { // NOLINT benchmark::Counter::kIsRate); } -static void HybridPipeArgs(benchmark::internal::Benchmark *b) { +static void HybridPipeArgs(benchmark::Benchmark *b) { for (int executor = 0; executor < 3; ++executor) { for (int batch_size = 128; batch_size <= 128; batch_size += 32) { for (int num_thread = 1; num_thread <= 4; ++num_thread) { @@ -375,7 +375,7 @@ BENCHMARK_DEFINE_F(RN50, nvJPEGPipe)(benchmark::State& st) { // NOLINT benchmark::Counter::kIsRate); } -static void nvjpegPipeArgs(benchmark::internal::Benchmark *b) { +static void nvjpegPipeArgs(benchmark::Benchmark *b) { for (int executor = 0; executor < 3; ++executor) { for (int batch_size = 128; batch_size <= 128; batch_size += 32) { for (int num_thread = 1; num_thread <= 4; ++num_thread) { diff --git a/dali/benchmark/resnet50_nvjpeg_bench.cc b/dali/benchmark/resnet50_nvjpeg_bench.cc index 3358446a1c8..2989ade0e5c 100755 --- a/dali/benchmark/resnet50_nvjpeg_bench.cc +++ b/dali/benchmark/resnet50_nvjpeg_bench.cc @@ -115,7 +115,7 @@ BENCHMARK_DEFINE_F(RealRN50, nvjpegPipe)(benchmark::State& st) { // NOLINT benchmark::Counter::kIsRate); } -static void PipeArgs(benchmark::internal::Benchmark *b) { +static void PipeArgs(benchmark::Benchmark *b) { for (int executor = 2; executor < 3; ++executor) { for (int batch_size = 128; batch_size <= 128; batch_size += 32) { for (int num_thread = 1; num_thread <= 4; ++num_thread) { diff --git a/dali/benchmark/slice_kernel_bench.cc b/dali/benchmark/slice_kernel_bench.cc index 7ee222696a6..71eaef470bd 100644 --- a/dali/benchmark/slice_kernel_bench.cc +++ b/dali/benchmark/slice_kernel_bench.cc @@ -76,7 +76,7 @@ class SliceBenchCPU : public DALIBenchmark { } }; -static void SliceKernelArgs_OnlySlice_CPU(benchmark::internal::Benchmark *b) { +static void SliceKernelArgs_OnlySlice_CPU(benchmark::Benchmark *b) { for (int H = 1000; H >= 500; H /= 2) { int W = H, C = 3; int crop_h = 9 * H / 10; @@ -94,7 +94,7 @@ BENCHMARK_REGISTER_F(SliceBenchCPU, Slice_CPU_OnlySlice)->Iterations(1000) ->UseRealTime() ->Apply(SliceKernelArgs_OnlySlice_CPU); -static void SliceKernelArgs_OnlyPad_CPU(benchmark::internal::Benchmark *b) { +static void SliceKernelArgs_OnlyPad_CPU(benchmark::Benchmark *b) { for (int H = 1000; H >= 500; H /= 2) { int W = H, C = 3; int crop_h = 9 * H / 10; @@ -114,7 +114,7 @@ BENCHMARK_REGISTER_F(SliceBenchCPU, Slice_OnlyPad_CPU)->Iterations(1000) ->UseRealTime() ->Apply(SliceKernelArgs_OnlyPad_CPU); -static void SliceKernelArgs_SliceAndPad_CPU(benchmark::internal::Benchmark *b) { +static void SliceKernelArgs_SliceAndPad_CPU(benchmark::Benchmark *b) { for (int H = 1000; H >= 500; H /= 2) { int W = H, C = 3; int crop_h = 9 * H / 10; diff --git a/dali/benchmark/slice_kernel_bench.cu b/dali/benchmark/slice_kernel_bench.cu index 82d55c87c12..8876be21d19 100644 --- a/dali/benchmark/slice_kernel_bench.cu +++ b/dali/benchmark/slice_kernel_bench.cu @@ -87,7 +87,7 @@ class SliceBenchGPU : public DALIBenchmark { } }; -static void SliceKernelArgs_GPU_OnlySlice(benchmark::internal::Benchmark *b) { +static void SliceKernelArgs_GPU_OnlySlice(benchmark::Benchmark *b) { for (int H = 1000; H >= 500; H /= 2) { int W = H, C = 3; int crop_h = 9 * H / 10; @@ -106,7 +106,7 @@ BENCHMARK_REGISTER_F(SliceBenchGPU, Slice_GPU_OnlySlice)->Iterations(1000) ->UseRealTime() ->Apply(SliceKernelArgs_GPU_OnlySlice); -static void SliceKernelArgs_GPU_OnlyPad(benchmark::internal::Benchmark *b) { +static void SliceKernelArgs_GPU_OnlyPad(benchmark::Benchmark *b) { for (int H = 1000; H >= 500; H /= 2) { int W = H, C = 3; int crop_h = 9 * H / 10; @@ -127,7 +127,7 @@ BENCHMARK_REGISTER_F(SliceBenchGPU, Slice_GPU_OnlyPad)->Iterations(1000) ->UseRealTime() ->Apply(SliceKernelArgs_GPU_OnlyPad); -static void SliceKernelArgs_GPU_SliceAndPad(benchmark::internal::Benchmark *b) { +static void SliceKernelArgs_GPU_SliceAndPad(benchmark::Benchmark *b) { for (int H = 1000; H >= 500; H /= 2) { int W = H, C = 3; int crop_h = 9 * H / 10; diff --git a/dali/benchmark/thread_pool_bench.cc b/dali/benchmark/thread_pool_bench.cc index 86ea6fb017c..20ac7003aec 100644 --- a/dali/benchmark/thread_pool_bench.cc +++ b/dali/benchmark/thread_pool_bench.cc @@ -19,7 +19,7 @@ namespace dali { class ThreadPoolBench : public DALIBenchmark {}; -static void ThreadPoolArgs(benchmark::internal::Benchmark *b) { +static void ThreadPoolArgs(benchmark::Benchmark *b) { int batch_size = 64; int work_size_min = 400; int work_size_max = 10000; diff --git a/dali/benchmark/transpose_bench.cc b/dali/benchmark/transpose_bench.cc index bd6aae4c116..05b983b194f 100644 --- a/dali/benchmark/transpose_bench.cc +++ b/dali/benchmark/transpose_bench.cc @@ -26,7 +26,7 @@ static std::vector permutations3[] = { {2, 1, 0}, }; -static void TransposeGPUArgs(benchmark::internal::Benchmark *b) { +static void TransposeGPUArgs(benchmark::Benchmark *b) { for (int batch_size = 256; batch_size >= 1; batch_size /= 2) { for (int H : dimensions) { for (int W : dimensions) { diff --git a/dali/benchmark/transpose_cpu_bench.cc b/dali/benchmark/transpose_cpu_bench.cc index da6b2e0d643..d4f4c56765c 100644 --- a/dali/benchmark/transpose_cpu_bench.cc +++ b/dali/benchmark/transpose_cpu_bench.cc @@ -52,7 +52,7 @@ std::tuple, std::vector> GetCase(int id) { return cases[id]; } -static void CustomArguments(benchmark::internal::Benchmark* b) { +static void CustomArguments(benchmark::Benchmark* b) { for (unsigned int i = 0; i < sizeof(cases) / sizeof(*cases); i++) { for (int scale = 1; scale <= 8; scale *= 2) { b->Args({i, scale}); diff --git a/dali/benchmark/warp_affine_bench.cc b/dali/benchmark/warp_affine_bench.cc index e63b00e89c2..8d9cb4614f9 100644 --- a/dali/benchmark/warp_affine_bench.cc +++ b/dali/benchmark/warp_affine_bench.cc @@ -18,7 +18,7 @@ namespace dali { -static void WarpAffineCPUArgs(benchmark::internal::Benchmark *b) { +static void WarpAffineCPUArgs(benchmark::Benchmark *b) { for (int batch_size = 16; batch_size >= 1; batch_size /= 2) { for (int H = 2048; H >= 256; H /= 2) { int W = H, C = 3; @@ -55,7 +55,7 @@ BENCHMARK_REGISTER_F(OperatorBench, WarpAffineCPU)->Iterations(50) ->Apply(WarpAffineCPUArgs); -static void WarpAffineGPUArgs(benchmark::internal::Benchmark *b) { +static void WarpAffineGPUArgs(benchmark::Benchmark *b) { for (int batch_size = 256; batch_size >= 1; batch_size /= 2) { for (int H = 2048; H >= 256; H /= 2) { int W = H, C = 3; From 354c500ee41b91d5912cbabab5242358edf857ce Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 4 Mar 2026 13:54:42 +0100 Subject: [PATCH 7/9] Fix nodiscard warnings on protobuf ParseFrom calls for protobuf bump --- dali/pipeline/operator/checkpointing/checkpoint.cc | 3 ++- dali/pipeline/operator/checkpointing/snapshot_serializer.cc | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dali/pipeline/operator/checkpointing/checkpoint.cc b/dali/pipeline/operator/checkpointing/checkpoint.cc index bc97928398c..db88c82ad82 100644 --- a/dali/pipeline/operator/checkpointing/checkpoint.cc +++ b/dali/pipeline/operator/checkpointing/checkpoint.cc @@ -93,7 +93,8 @@ std::string Checkpoint::SerializeToProtobuf(ExecutorBase &exec) const { void Checkpoint::DeserializeFromProtobuf(ExecutorBase &exec, std::string_view serialized_data) { Clear(); dali_proto::Checkpoint checkpoint; - checkpoint.ParseFromArray(serialized_data.data(), serialized_data.size()); + DALI_ENFORCE(checkpoint.ParseFromArray(serialized_data.data(), serialized_data.size()), + "Failed to deserialize checkpoint from protobuf."); for (int i = 0; i < checkpoint.cpts_size(); i++) { const auto &name = checkpoint.cpts(i).operator_name(); diff --git a/dali/pipeline/operator/checkpointing/snapshot_serializer.cc b/dali/pipeline/operator/checkpointing/snapshot_serializer.cc index d669be452f8..c845ed0ccea 100644 --- a/dali/pipeline/operator/checkpointing/snapshot_serializer.cc +++ b/dali/pipeline/operator/checkpointing/snapshot_serializer.cc @@ -59,7 +59,7 @@ std::string SerializeRNG(const std::vector &snapshot) { template std::vector DeserializeRNG(const std::string &data) { dali_proto::RNGSnapshotCPU proto_snapshot; - proto_snapshot.ParseFromString(data); + DALI_ENFORCE(proto_snapshot.ParseFromString(data), "Failed to deserialize RNG snapshot from protobuf."); std::vector snapshot; for (int i = 0; i < proto_snapshot.rng_size(); i++) snapshot.push_back(DeserializeFromString(proto_snapshot.rng(i))); @@ -96,7 +96,7 @@ std::string SnapshotSerializer::Serialize(const std::vector &snapsh template<> DLL_PUBLIC std::vector SnapshotSerializer::Deserialize(const std::string &data) { dali_proto::RNGSnapshotGPU proto_snapshot; - proto_snapshot.ParseFromString(data); + DALI_ENFORCE(proto_snapshot.ParseFromString(data), "Failed to deserialize GPU RNG snapshot from protobuf."); auto str = proto_snapshot.rng(); std::vector snapshot(str.size() / sizeof(curandState)); memcpy(snapshot.data(), str.data(), str.size()); @@ -115,7 +115,7 @@ std::string SnapshotSerializer::Serialize(const LoaderStateSnapshot &snapshot) { template<> DLL_PUBLIC LoaderStateSnapshot SnapshotSerializer::Deserialize(const std::string &data) { dali_proto::ReaderStateSnapshot proto_snapshot; - proto_snapshot.ParseFromString(data); + DALI_ENFORCE(proto_snapshot.ParseFromString(data), "Failed to deserialize loader state snapshot from protobuf."); return LoaderStateSnapshot { DeserializeFromString(proto_snapshot.loader_state().rng()), proto_snapshot.loader_state().current_epoch(), From 7b9b60eb040f22ee2e3f4217bb27e5a88944bcf4 Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Wed, 4 Mar 2026 15:48:44 +0100 Subject: [PATCH 8/9] Fix line length in snapshot_serializer.cc --- .../operator/checkpointing/snapshot_serializer.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dali/pipeline/operator/checkpointing/snapshot_serializer.cc b/dali/pipeline/operator/checkpointing/snapshot_serializer.cc index c845ed0ccea..001f137bec3 100644 --- a/dali/pipeline/operator/checkpointing/snapshot_serializer.cc +++ b/dali/pipeline/operator/checkpointing/snapshot_serializer.cc @@ -59,7 +59,8 @@ std::string SerializeRNG(const std::vector &snapshot) { template std::vector DeserializeRNG(const std::string &data) { dali_proto::RNGSnapshotCPU proto_snapshot; - DALI_ENFORCE(proto_snapshot.ParseFromString(data), "Failed to deserialize RNG snapshot from protobuf."); + DALI_ENFORCE(proto_snapshot.ParseFromString(data), + "Failed to deserialize RNG snapshot from protobuf."); std::vector snapshot; for (int i = 0; i < proto_snapshot.rng_size(); i++) snapshot.push_back(DeserializeFromString(proto_snapshot.rng(i))); @@ -96,7 +97,8 @@ std::string SnapshotSerializer::Serialize(const std::vector &snapsh template<> DLL_PUBLIC std::vector SnapshotSerializer::Deserialize(const std::string &data) { dali_proto::RNGSnapshotGPU proto_snapshot; - DALI_ENFORCE(proto_snapshot.ParseFromString(data), "Failed to deserialize GPU RNG snapshot from protobuf."); + DALI_ENFORCE(proto_snapshot.ParseFromString(data), + "Failed to deserialize GPU RNG snapshot from protobuf."); auto str = proto_snapshot.rng(); std::vector snapshot(str.size() / sizeof(curandState)); memcpy(snapshot.data(), str.data(), str.size()); @@ -115,7 +117,8 @@ std::string SnapshotSerializer::Serialize(const LoaderStateSnapshot &snapshot) { template<> DLL_PUBLIC LoaderStateSnapshot SnapshotSerializer::Deserialize(const std::string &data) { dali_proto::ReaderStateSnapshot proto_snapshot; - DALI_ENFORCE(proto_snapshot.ParseFromString(data), "Failed to deserialize loader state snapshot from protobuf."); + DALI_ENFORCE(proto_snapshot.ParseFromString(data), + "Failed to deserialize loader state snapshot from protobuf."); return LoaderStateSnapshot { DeserializeFromString(proto_snapshot.loader_state().rng()), proto_snapshot.loader_state().current_epoch(), From 1af942a85b4ac50afa3a9614a04a0cf1edc26e9c Mon Sep 17 00:00:00 2001 From: Joaquin Anton Guirao Date: Fri, 6 Mar 2026 08:51:16 +0100 Subject: [PATCH 9/9] Bump DALI_DEPS_VERSION Signed-off-by: Joaquin Anton Guirao --- DALI_DEPS_VERSION | 2 +- dali/test/python/decoder/test_video.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/DALI_DEPS_VERSION b/DALI_DEPS_VERSION index 3ac73e126c9..824148f9d09 100644 --- a/DALI_DEPS_VERSION +++ b/DALI_DEPS_VERSION @@ -1 +1 @@ -f689931ff3fe42aa4ae420d977a725ad5e344b5b +67cef5f4d1017999949e37f96a7359d8d05e8bbf diff --git a/dali/test/python/decoder/test_video.py b/dali/test/python/decoder/test_video.py index 1afc621fe69..2b2df4e0fd2 100644 --- a/dali/test/python/decoder/test_video.py +++ b/dali/test/python/decoder/test_video.py @@ -955,7 +955,7 @@ def video_pipe(): return videos, frame_idxs pipe = video_pipe(batch_size=batch_size, num_threads=2, device_id=0) - (out_videos, out_frame_idxs) = pipe.run() + out_videos, out_frame_idxs = pipe.run() for i in range(batch_size): idxs = out_frame_idxs.as_cpu().at(i) @@ -1013,7 +1013,7 @@ def video_pipe(): full_sequences = total_frames // (stride * sequence_length) padded_idxs = None for _ in range(full_sequences + 2): - (_, out_idxs) = pipe.run() + _, out_idxs = pipe.run() idxs = out_idxs.as_cpu().at(0).flatten() if -1 in idxs: padded_idxs = idxs @@ -1110,7 +1110,7 @@ def video_pipe(): num_full_seqs = total_frames // full_seq_stride padded_idxs = None for _ in range(num_full_seqs + 2): - (_, out_idxs) = pipe.run() + _, out_idxs = pipe.run() idxs = out_idxs.as_cpu().at(0).flatten() if idxs[0] == tail_start: padded_idxs = idxs @@ -1163,8 +1163,8 @@ def sequence_pipe(): pipe_s = scalar_pipe(batch_size=batch_size, num_threads=2, device_id=0, seed=42) pipe_q = sequence_pipe(batch_size=batch_size, num_threads=2, device_id=0, seed=42) - (_, out_scalar) = pipe_s.run() - (_, out_sequence) = pipe_q.run() + _, out_scalar = pipe_s.run() + _, out_sequence = pipe_q.run() for i in range(batch_size): scalar_idx = int(out_scalar.as_cpu().at(i)[0])