Fix CI, update test dependencies, and remove the compile command

.github/workflows/test-and-release.yml

@@ -1,91 +1,74 @@
-name: 'Test and release Yakut'
+name: 'Test & Release'
 on: [ push, pull_request ]
 
-# Ensures that only one workflow is running at a time
-concurrency:
-  group: ${{ github.workflow_sha }}
-  cancel-in-progress: true
-
 jobs:
-  yakut-test:
-    name: Test Yakut
+  test:
+    name: Test
     # https://docs.github.com/en/webhooks/webhook-events-and-payloads?actionType=edited#pull_request
     if: (github.event_name == 'push') || github.event.pull_request.head.repo.fork
     strategy:
       fail-fast: false
       matrix:
-        # The Windows NPcap runner is an ordinary Windows machine with the NPcap driver installed manually.
-        # We chose to do it this way because NPcap driver installation requires a reboot, which is difficult to
-        # automate. The NPcap driver is required for the Cyphal/UDP transport tests to work.
-        os: [ubuntu-22.04, windows-2019-npcap]
-        python: ['3.8', '3.9', '3.10', '3.11']
-        exclude:  # We don't test Windows with old Python versions because it takes too much effort.
-          - os: windows-2019-npcap
-            python: 3.8
-          - os: windows-2019-npcap
-            python: 3.9
+        os: [ ubuntu-latest ]
+        py: [ '3.10', '3.11', '3.12', '3.13' ]
+        # On Windows, we select the configurations we test manually because we only have a few runners,
+        # and because the infrastructure is hard to maintain using limited resources.
+        include:
+          - { os: win-pcap, py: '3.12' }
     runs-on: ${{ matrix.os }}
+    env:
+      FORCE_COLOR: 1
     steps:
-      - name: Check out
-        uses: actions/checkout@v3
-
-      - name: Install Python3
-        uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
         with:
-          python-version: ${{ matrix.python }}
+          submodules: true
 
-      - name: Log Python version
-        run: python --version
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.py }}
 
-      - name: Install dependencies
+      - name: Configure GNU/Linux
+        if: ${{ runner.os == 'Linux' }}
         # language=bash
         run: |
-          if [ "$RUNNER_OS" == "Linux" ]; then
-            sudo apt-get --ignore-missing update || true
-            sudo apt-get install -y linux-*-extra-$(uname -r) ncat
-            sudo apt-get install -y libsdl2-2.0-0   # For PySDL2. On Windows/macOS the binaries are pulled from PyPI.
-            sudo apt-get install -y libasound2-dev  # For RtMidi.
-          fi
-          git submodule update --init --recursive
-          python -m pip install --upgrade pip setuptools nox
-        shell: bash
+          python --version
+          sudo apt-get --ignore-missing update || true
+          sudo apt-get install -y linux-*-extra-$(uname -r) ncat
+          sudo apt-get install -y libsdl2-2.0-0   # For PySDL2. On Windows/macOS the binaries are pulled from PyPI.
+          sudo apt-get install -y libasound2-dev  # For RtMidi.
 
-      - name: Run build and test
-        # language=bash
-        run: |
-          nox --non-interactive --session test --python ${{ matrix.python }}
-          nox --non-interactive --session lint
-        shell: bash
-        env:
-          FORCE_COLOR: 1
+      # Only one statement per step to ensure the error codes are not ignored by PowerShell.
+      - run: python -m pip install --upgrade attrs pip setuptools nox
+      - run: nox --non-interactive --session test --python ${{ matrix.py }}
+      - run: nox --non-interactive --session lint
 
-      - name: Upload diagnostics
-        uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: (success() || failure())
         with:
           # The matrix is shown for convenience but this is fragile because the values may not be string-convertible.
           # Shall it break one day, feel free to remove the matrix from here.
-          # The job status is per matrix item, which is super convenient.
           name: ${{github.job}}-#${{strategy.job-index}}-${{job.status}}-${{join(matrix.*, ',')}}
           path: "**/*.log"
-          retention-days: 7
+          retention-days: 90
+          include-hidden-files: true
 
-  yakut-release:
-    name: Release Yakut
+  release:
+    name: Release
     runs-on: ubuntu-latest
     if: >
       (github.event_name == 'push') &&
       (contains(github.event.head_commit.message, '#release') || contains(github.ref, '/main'))
-    needs: yakut-test
+    needs: test
     steps:
       - name: Check out
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with:
+          submodules: true
 
       - name: Create distribution wheel
         # language=bash
         run: |
-          git submodule update --init --recursive
-          python -m pip install --upgrade pip setuptools wheel twine
+          python -m pip install --upgrade packaging pip setuptools wheel twine
           python setup.py sdist bdist_wheel
 
       - name: Get release version
@@ -99,7 +82,7 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN_YAKUT }}
 
       - name: Push version tag
-        uses: mathieudutour/github-tag-action@v6.1
+        uses: mathieudutour/github-tag-action@v6.2
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           custom_tag: ${{ env.yakut_version }}

.gitignore

@@ -53,6 +53,3 @@ coverage.xml
 
 # Compiled namespaces
 .*compiled
-/uavcan/
-/reg/
-/sirius_cyber_corp/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/deps/public_regulated_data_types"]
+	path = tests/deps/public_regulated_data_types
+	url = https://github.com/OpenCyphal/public_regulated_data_types

CONTRIBUTING.md

@@ -55,7 +55,7 @@ A more interactive approach is as follows:
 3. Change directory to `.nox/test-3-8/tmp`, here substitute `test-3-8` for the directory you have. 
    This is one of the environments that Nox creates for testing.
 4. Run `source ../bin/activate` to activate the virtualenv.
-5. `export PYTHONPATH=.compiled/`
+5. Optionally: `export PYCYPHAL_PATH=...compiled/`
 6. Run specific commands you need:
    `pytest ../../../yakut/whatever`, `mypy --strict ../../../yakut ../../../tests`, etc.
 
@@ -74,11 +74,10 @@ To look for manual tests in the codebase, please search for `def _main` under `t
 
 We recommend [JetBrains PyCharm](https://www.jetbrains.com/pycharm/) for development.
 
-The test suite stores compiled DSDL into `.compiled/` in the current working directory
-(when using Nox, the current working directory may be under a virtualenv private directory).
+The test suite stores compiled DSDL into whatever is pointed to by `PYCYPHAL_PATH`;
+the Noxfile overrides `PYCYPHAL_PATH` with a directory inside the venv.
 Make sure to mark it as a source directory to enable code completion and type analysis in the IDE
 (for PyCharm: right click -> Mark Directory As -> Sources Root).
-Alternatively, you can just compile DSDL manually directly in the project root.
 
 Configure the IDE to run Black on save.
 See the Black documentation for integration instructions.

README.md

@@ -1,4 +1,4 @@
-# Yakut
+# Yakut – Cyphal CLI tool
 
 <!--suppress CheckImageSize, HtmlDeprecatedAttribute -->
 <img src="/docs/opencyphal-favicon-512.png" alt="OpenCyphal logo" width=128 align=right>
@@ -23,8 +23,6 @@ Afterward do endeavor to read the docs: **`yakut --help`**
 
 Check for new versions every now and then: **`pip install --upgrade yakut`**
 
-Installation & configuration screencasts are available for [Windows](https://forum.opencyphal.org/t/screencast-of-installing-configuring-yakut/1197/2?u=pavel.kirienko), [GNU/Linux](https://forum.opencyphal.org/t/screencast-of-installing-configuring-yakut/1197/1?u=pavel.kirienko), and [macOS](https://www.youtube.com/watch?v=dQw4w9WgXcQ).
-
 ### Additional third-party tools
 
 Since Yakut heavily relies on YAML/JSON documents exchanged via stdin/stdout, [**`jq`**](https://stedolan.github.io/jq/) is often needed for any non-trivial usage of the tool, so consider installing it as well. Users of GNU/Linux will likely find it in the default software repositories (`pacman -S jq`, `apt install jq`, etc.).
@@ -33,9 +31,9 @@ Since Yakut heavily relies on YAML/JSON documents exchanged via stdin/stdout, [*
 
 Transport layer inspection tools:
 
+- [Wireshark](https://www.wireshark.org/) with [Cyphal plugins](https://github.com/OpenCyphal/wireshark_plugins)
+  (n.b.: some versions of Wireshark may label Cyphal/CAN captures as UAVCAN/CAN due to rebranding).
 - Cyphal/CAN on GNU/Linux (candump, canbusload, etc.): [`can-utils`](https://github.com/linux-can/can-utils)
-- Cyphal/UDP or Cyphal/CAN: [Wireshark](https://www.wireshark.org/)
-  (n.b.: Wireshark might label Cyphal captures as UAVCAN due to rebranding)
 
 ## Invoking commands
 

noxfile.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020 OpenCyphal
+# Copyright (c) OpenCyphal
 # This software is distributed under the terms of the MIT License.
 # Author: Pavel Kirienko <pavel@opencyphal.org>
 # type: ignore
@@ -14,8 +14,13 @@
 DEPS_DIR = ROOT_DIR / "tests" / "deps"
 assert DEPS_DIR.is_dir(), "Invalid configuration"
 
+CYPHAL_PATH = [
+    DEPS_DIR / "public_regulated_data_types",
+    ROOT_DIR / "tests" / "custom_data_types",
+]
 
-PYTHONS = ["3.8", "3.9", "3.10", "3.11"]
+
+PYTHONS = ["3.10", "3.11", "3.12", "3.13"]
 
 
 @nox.session(python=False)
@@ -50,9 +55,9 @@ def test(session):
 
     # Now we can install dependencies for the full integration test suite.
     session.install(
-        "pytest         ~= 7.4",
-        "pytest-asyncio ~= 0.21.0",
-        "coverage       ~= 7.4",
+        "pytest         ~= 8.3",
+        "pytest-asyncio ~= 0.26.0",
+        "coverage       ~= 7.8",
     )
 
     # The test suite generates a lot of temporary files, so we change the working directory.
@@ -82,13 +87,12 @@ def test(session):
         *session.posargs,
         env={
             "PYTHONPATH": str(DEPS_DIR),
-            "PATH": os.pathsep.join([session.env["PATH"], str(DEPS_DIR)]),
+            "PATH": os.pathsep.join([os.environ["PATH"], str(DEPS_DIR)]),
+            "CYPHAL_PATH": os.pathsep.join(map(str, CYPHAL_PATH)),
+            "PYCYPHAL_PATH": str(tmp_dir / ".compiled"),
+            "PYCYPHAL_LOGLEVEL": "ERROR",
         },
     )
-
-    # The coverage threshold is intentionally set low for interactive runs because when running locally
-    # in a reused virtualenv the DSDL compiler run may be skipped to save time, resulting in a reduced coverage.
-    # Some features are not available on Windows so the coverage threshold is set low for it.
     if session.posargs or session.interactive or sys.platform.startswith("win"):
         fail_under = 1
     else:
@@ -104,14 +108,14 @@ def test(session):
     #   1. It requires access to the code generated by the test suite.
     #   2. It has to be run separately per Python version we support.
     # If the interpreter is not CPython, this may need to be conditionally disabled.
-    session.install("mypy ~= 1.8")
-    session.run("mypy", "--strict", *map(str, src_dirs))
+    session.install("mypy ~= 1.15.0")
+    session.run("mypy", *map(str, src_dirs))
 
 
 @nox.session(reuse_venv=True)
 def lint(session):
-    session.install("pylint ~= 3.0.3")
+    session.install("pylint ~= 3.3.7")
     session.run("pylint", "yakut", "tests")
 
-    session.install("black ~= 23.12")
+    session.install("black ~= 25.1")
     session.run("black", "--check", ".")

setup.cfg

@@ -48,7 +48,7 @@ zip_safe             = False
 include_package_data = True
 packages             = find:
 install_requires =
-    pycyphal[transport-udp,transport-serial,transport-can-pythoncan] ~= 1.8
+    pycyphal[transport-udp,transport-serial,transport-can-pythoncan] ~= 1.20
     ruamel.yaml     <  0.18
     requests        ~= 2.27
     simplejson      ~= 3.17
@@ -100,9 +100,10 @@ log_file         = pytest.log
 log_file_level   = DEBUG
 # Unraisable exceptions are filtered because PyTest yields false-positives coming from PyCyphal.
 addopts          = --doctest-modules -v -p no:unraisableexception
-asyncio_mode     = auto
 filterwarnings =
     ignore:.*SDL2.*:UserWarning
+asyncio_mode                        = auto
+asyncio_default_fixture_loop_scope  = function
 
 # ---------------------------------------- MYPY ----------------------------------------
 [mypy]
@@ -114,15 +115,19 @@ disallow_untyped_defs  = True
 check_untyped_defs     = True
 no_implicit_optional   = True
 warn_redundant_casts   = True
-warn_unused_ignores    = True
+warn_unused_ignores    = False
 show_error_context     = True
-strict_equality        = True
+strict_equality        = False
+strict                 = False
 implicit_reexport      = False
 # We don't want MyPy to go checking generated code and its dependencies.
 follow_imports         = silent
 mypy_path =
     .compiled
 
+[mypy-nunavut_support]
+ignore_errors = True
+
 [mypy-pytest.*]
 ignore_missing_imports = True
 
@@ -167,6 +172,7 @@ exclude_lines =
 # ---------------------------------------- PYLINT ----------------------------------------
 [pylint.MASTER]
 fail-under=9.9
+ignore-paths=^.*/\.compiled/.*$
 
 [pylint.MESSAGES CONTROL]
 # Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED.

tests/cmd/accommodate.py

@@ -4,22 +4,18 @@
 
 from __future__ import annotations
 import time
-import typing
 from tests.subprocess import Subprocess, execute_cli
-from tests.dsdl import OUTPUT_DIR
 from tests.transport import TransportFactory
 
 
-def _unittest_accommodate_swarm(transport_factory: TransportFactory, compiled_dsdl: typing.Any) -> None:
-    _ = compiled_dsdl
+def _unittest_accommodate_swarm(transport_factory: TransportFactory) -> None:
     # We spawn a lot of processes here, which might strain the test system a little, so beware. I've tested it
     # with 120 processes and it made my workstation (24 GB RAM ~4 GHz Core i7) struggle to the point of being
     # unable to maintain sufficiently real-time operation for the test to pass. Hm.
     used_node_ids = list(range(5))
     pubs = [
         Subprocess.cli(
             f"--transport={transport_factory(idx).expression}",
-            f"--path={OUTPUT_DIR}",
             "pub",
             "--period=0.4",
             "--count=60",
@@ -29,7 +25,6 @@ def _unittest_accommodate_swarm(transport_factory: TransportFactory, compiled_ds
     time.sleep(5)  # Some time is required for the nodes to start.
     _, stdout, _ = execute_cli(
         "-v",
-        f"--path={OUTPUT_DIR}",
         f"--transport={transport_factory(None).expression}",
         "accommodate",
         timeout=100.0,
@@ -42,7 +37,6 @@ def _unittest_accommodate_swarm(transport_factory: TransportFactory, compiled_ds
 def _unittest_accommodate_loopback() -> None:
     _, stdout, _ = execute_cli(
         "-v",
-        f"--path={OUTPUT_DIR}",
         "accommodate",
         timeout=30.0,
         environment_variables={"YAKUT_TRANSPORT": "Loopback(None),Loopback(None)"},
@@ -53,7 +47,6 @@ def _unittest_accommodate_loopback() -> None:
 def _unittest_accommodate_udp_localhost() -> None:
     _, stdout, _ = execute_cli(
         "-v",
-        f"--path={OUTPUT_DIR}",
         "accommodate",
         timeout=30.0,
         environment_variables={"YAKUT_TRANSPORT": 'UDP("127.0.0.1",None)'},