From 38cd48068ca062ab9db38380bbfea97c288c97a6 Mon Sep 17 00:00:00 2001 From: Leonardo Pinheiro Date: Sun, 9 Mar 2025 15:25:25 +1000 Subject: [PATCH 1/9] update powershell fname and args --- .../src/autogen_ext/code_executors/_common.py | 2 + .../code_executors/local/__init__.py | 63 ++++++++++++++----- 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/python/packages/autogen-ext/src/autogen_ext/code_executors/_common.py b/python/packages/autogen-ext/src/autogen_ext/code_executors/_common.py index ce656e93f357..4b1259ef04ee 100644 --- a/python/packages/autogen-ext/src/autogen_ext/code_executors/_common.py +++ b/python/packages/autogen-ext/src/autogen_ext/code_executors/_common.py @@ -158,6 +158,8 @@ def lang_to_cmd(lang: str) -> str: return lang if lang in ["shell"]: return "sh" + if lang in ["pwsh", "powershell", "ps1"]: + return "pwsh" else: raise ValueError(f"Unsupported language: {lang}") diff --git a/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py b/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py index aefd366c3bad..85ef3b052ef0 100644 --- a/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py +++ b/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py @@ -302,26 +302,33 @@ async def execute_code_blocks( async def _execute_code_dont_check_setup( self, code_blocks: List[CodeBlock], cancellation_token: CancellationToken ) -> CommandLineCodeResult: + """ + Execute the provided code blocks in the local command line without re-checking setup. + Returns a CommandLineCodeResult indicating success or failure. + """ logs_all: str = "" file_names: List[Path] = [] exitcode = 0 + for code_block in code_blocks: lang, code = code_block.language, code_block.code lang = lang.lower() + # Remove pip output where possible code = silence_pip(code, lang) + # Normalize python variants to "python" if lang in PYTHON_VARIANTS: lang = "python" + # Abort if not supported if lang not in self.SUPPORTED_LANGUAGES: - # In case the language is not supported, we return an error message. exitcode = 1 logs_all += "\n" + f"unknown language {lang}" break + # Try extracting a filename (if present) try: - # Check if there is a filename comment filename = get_file_name_from_content(code, self._work_dir) except ValueError: return CommandLineCodeResult( @@ -330,32 +337,58 @@ async def _execute_code_dont_check_setup( code_file=None, ) + # If no filename is found, create one if filename is None: - # create a file with an automatically generated name code_hash = sha256(code.encode()).hexdigest() - filename = f"tmp_code_{code_hash}.{'py' if lang.startswith('python') else lang}" + if lang.startswith("python"): + ext = "py" + elif lang in ["pwsh", "powershell", "ps1"]: + ext = "ps1" + else: + ext = lang + + filename = f"tmp_code_{code_hash}.{ext}" written_file = (self._work_dir / filename).resolve() with written_file.open("w", encoding="utf-8") as f: f.write(code) file_names.append(written_file) + # Build environment env = os.environ.copy() - if self._virtual_env_context: virtual_env_exe_abs_path = os.path.abspath(self._virtual_env_context.env_exe) virtual_env_bin_abs_path = os.path.abspath(self._virtual_env_context.bin_path) env["PATH"] = f"{virtual_env_bin_abs_path}{os.pathsep}{env['PATH']}" - program = virtual_env_exe_abs_path if lang.startswith("python") else lang_to_cmd(lang) + # Decide how to invoke the script + if lang == "python": + program = ( + self._virtual_env_context.env_exe + if self._virtual_env_context + else sys.executable + ) + extra_args = [str(written_file.absolute())] + elif lang in {"pwsh", "powershell", "ps1"}: + # Use powershell/pwsh for .ps1 scripts with proper flags + program = "powershell" + extra_args = [ + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(written_file.absolute()), + ] else: - program = sys.executable if lang.startswith("python") else lang_to_cmd(lang) + # Shell commands (bash, sh, etc.) + program = lang_to_cmd(lang) + extra_args = [str(written_file.absolute())] - # Wrap in a task to make it cancellable + # Create a subprocess and run task = asyncio.create_task( asyncio.create_subprocess_exec( program, - str(written_file.absolute()), + *extra_args, cwd=self._work_dir, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, @@ -363,31 +396,27 @@ async def _execute_code_dont_check_setup( ) ) cancellation_token.link_future(task) + try: proc = await task stdout, stderr = await asyncio.wait_for(proc.communicate(), self._timeout) exitcode = proc.returncode or 0 - except asyncio.TimeoutError: - logs_all += "\n Timeout" - # Same exit code as the timeout command on linux. + logs_all += "\nTimeout" exitcode = 124 break except asyncio.CancelledError: - logs_all += "\n Cancelled" - # TODO: which exit code? 125 is Operation Canceled + logs_all += "\nCancelled" exitcode = 125 break - self._running_cmd_task = None - logs_all += stderr.decode() logs_all += stdout.decode() if exitcode != 0: break - code_file = str(file_names[0]) if len(file_names) > 0 else None + code_file = str(file_names[0]) if file_names else None return CommandLineCodeResult(exit_code=exitcode, output=logs_all, code_file=code_file) async def restart(self) -> None: From 04555d98674a587a3a92061dca055474e111c1d3 Mon Sep 17 00:00:00 2001 From: Leonardo Pinheiro Date: Sun, 9 Mar 2025 15:35:24 +1000 Subject: [PATCH 2/9] lint --- .../src/autogen_ext/code_executors/local/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py b/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py index 85ef3b052ef0..290b44ae399a 100644 --- a/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py +++ b/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py @@ -357,16 +357,13 @@ async def _execute_code_dont_check_setup( # Build environment env = os.environ.copy() if self._virtual_env_context: - virtual_env_exe_abs_path = os.path.abspath(self._virtual_env_context.env_exe) virtual_env_bin_abs_path = os.path.abspath(self._virtual_env_context.bin_path) env["PATH"] = f"{virtual_env_bin_abs_path}{os.pathsep}{env['PATH']}" # Decide how to invoke the script if lang == "python": program = ( - self._virtual_env_context.env_exe - if self._virtual_env_context - else sys.executable + os.path.abspath(self._virtual_env_context.env_exe) if self._virtual_env_context else sys.executable ) extra_args = [str(written_file.absolute())] elif lang in {"pwsh", "powershell", "ps1"}: From aca0218183a10f52f5c7af12651c89365c0a2856 Mon Sep 17 00:00:00 2001 From: Leonardo Pinheiro Date: Sun, 9 Mar 2025 16:01:24 +1000 Subject: [PATCH 3/9] add unit test --- .../test_commandline_code_executor.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py b/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py index 9d27d39d96fb..3186528e6295 100644 --- a/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py +++ b/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py @@ -203,3 +203,24 @@ def test_serialize_deserialize() -> None: executor_config = executor.dump_component() loaded_executor = LocalCommandLineCodeExecutor.load_component(executor_config) assert executor.work_dir == loaded_executor.work_dir + + +@pytest.mark.asyncio +@pytest.mark.skipif( + shutil.which("powershell") is None and shutil.which("pwsh") is None, + reason="No PowerShell interpreter (powershell or pwsh) found on this environment.", +) +@pytest.mark.parametrize("executor_and_temp_dir", ["local"], indirect=True) +async def test_ps1_script(executor_and_temp_dir: ExecutorFixture) -> None: + """ + Test execution of a simple PowerShell script. + This test is skipped if powershell/pwsh is not installed. + """ + executor, _ = executor_and_temp_dir + cancellation_token = CancellationToken() + code = 'Write-Host "hello from powershell!"' + code_blocks = [CodeBlock(code=code, language="powershell")] + result = await executor.execute_code_blocks(code_blocks, cancellation_token) + assert result.exit_code == 0 + assert "hello from powershell!" in result.output + assert result.code_file is not None From b53a03cdafec2f4a525c1ee93436809cafc3849f Mon Sep 17 00:00:00 2001 From: Leonardo Pinheiro Date: Sun, 9 Mar 2025 17:09:32 +1000 Subject: [PATCH 4/9] try test flag --- .../tests/code_executors/test_commandline_code_executor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py b/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py index 3186528e6295..752df5400059 100644 --- a/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py +++ b/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py @@ -18,6 +18,9 @@ from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor +HAS_POWERSHELL: bool = shutil.which("powershell") is not None or shutil.which("pwsh") is not None + + @pytest_asyncio.fixture(scope="function") # type: ignore async def executor_and_temp_dir( request: pytest.FixtureRequest, @@ -207,7 +210,7 @@ def test_serialize_deserialize() -> None: @pytest.mark.asyncio @pytest.mark.skipif( - shutil.which("powershell") is None and shutil.which("pwsh") is None, + not HAS_POWERSHELL, reason="No PowerShell interpreter (powershell or pwsh) found on this environment.", ) @pytest.mark.parametrize("executor_and_temp_dir", ["local"], indirect=True) From 9e70dfb68269d35e4c48a9e74185d32e9d556079 Mon Sep 17 00:00:00 2001 From: Leonardo Pinheiro Date: Sun, 9 Mar 2025 17:17:30 +1000 Subject: [PATCH 5/9] add platform check --- .../tests/code_executors/test_commandline_code_executor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py b/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py index 752df5400059..5cf6fb7d7973 100644 --- a/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py +++ b/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py @@ -4,6 +4,7 @@ import asyncio import os import shutil +import platform import sys import tempfile import venv @@ -18,7 +19,9 @@ from autogen_ext.code_executors.local import LocalCommandLineCodeExecutor -HAS_POWERSHELL: bool = shutil.which("powershell") is not None or shutil.which("pwsh") is not None +HAS_POWERSHELL: bool = platform.system() == "Windows" and ( + shutil.which("powershell") is not None or shutil.which("pwsh") is not None +) @pytest_asyncio.fixture(scope="function") # type: ignore From d29a7cef18348f0331448101b53f3fec6b509355 Mon Sep 17 00:00:00 2001 From: Leonardo Pinheiro Date: Mon, 10 Mar 2025 12:46:17 +1000 Subject: [PATCH 6/9] add windows ci test config --- .github/workflows/checks.yml | 38 +++++++++++++++++++ python/packages/autogen-ext/pyproject.toml | 1 + .../test_commandline_code_executor.py | 1 + python/packages/autogen-ext/tests/conftest.py | 9 +++++ 4 files changed, 49 insertions(+) create mode 100644 python/packages/autogen-ext/tests/conftest.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 3e04013cc111..8620b32498f9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -181,6 +181,44 @@ jobs: name: coverage-autogen-ext-grpc path: ./python/coverage_autogen-ext-grpc.xml + test-autogen-ext-pwsh: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + version: "0.5.18" + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install Python deps + run: | + uv sync --locked --all-extras + shell: pwsh + working-directory: ./python + + - name: Run single PowerShell-required test + # You can still invoke coverage, e.g. via pytest --cov or coverage run + run: | + poe --directory ./packages/autogen-ext test-windows + shell: pwsh + working-directory: ./python + + - name: Move coverage file + run: | + mv ./packages/autogen-ext/coverage.xml coverage_autogen_ext_windows.xml + working-directory: ./python + + - name: Upload coverage artifact + uses: actions/upload-artifact@v4 + with: + name: coverage-autogen-ext-windows + path: ./python/coverage_autogen_ext_windows.xml + codecov: runs-on: ubuntu-latest needs: [test, test-grpc] diff --git a/python/packages/autogen-ext/pyproject.toml b/python/packages/autogen-ext/pyproject.toml index 42719d25727f..e8a1ae27c95c 100644 --- a/python/packages/autogen-ext/pyproject.toml +++ b/python/packages/autogen-ext/pyproject.toml @@ -168,6 +168,7 @@ test.sequence = [ ] test.default_item_type = "cmd" test-grpc = "pytest -n 1 --cov=src --cov-report=term-missing --cov-report=xml --grpc" +test-windows = "pytest -n 1 --cov=src --cov-report=term-missing --cov-report=xml -m 'windows'" mypy = "mypy --config-file ../../pyproject.toml --exclude src/autogen_ext/runtimes/grpc/protos --exclude tests/protos src tests" [tool.mypy] diff --git a/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py b/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py index 5cf6fb7d7973..7ff87909af80 100644 --- a/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py +++ b/python/packages/autogen-ext/tests/code_executors/test_commandline_code_executor.py @@ -212,6 +212,7 @@ def test_serialize_deserialize() -> None: @pytest.mark.asyncio +@pytest.mark.windows @pytest.mark.skipif( not HAS_POWERSHELL, reason="No PowerShell interpreter (powershell or pwsh) found on this environment.", diff --git a/python/packages/autogen-ext/tests/conftest.py b/python/packages/autogen-ext/tests/conftest.py new file mode 100644 index 000000000000..1ef36bd7c816 --- /dev/null +++ b/python/packages/autogen-ext/tests/conftest.py @@ -0,0 +1,9 @@ +import pytest + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption("--windows", action="store_true", default=False, help="Run tests for Windows") + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line("markers", "windows: mark test as requiring Windows") From a0abf3d6b7d2e995bb81712586ee373df00d4891 Mon Sep 17 00:00:00 2001 From: Leonardo Pinheiro Date: Mon, 10 Mar 2025 12:54:40 +1000 Subject: [PATCH 7/9] try activating venv --- .github/workflows/checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 8620b32498f9..99714fe7ec96 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -201,9 +201,9 @@ jobs: shell: pwsh working-directory: ./python - - name: Run single PowerShell-required test - # You can still invoke coverage, e.g. via pytest --cov or coverage run + - name: Run tests for Windows run: | + .venv/Scripts/activate.ps1 poe --directory ./packages/autogen-ext test-windows shell: pwsh working-directory: ./python From 49674a4daa4cc6ec446b8b9ddab1c2ea474f8e63 Mon Sep 17 00:00:00 2001 From: Leonardo Pinheiro Date: Mon, 10 Mar 2025 13:21:34 +1000 Subject: [PATCH 8/9] use lang2cmd for powershell --- .../code_executors/local/__init__.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py b/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py index 290b44ae399a..5742d4aa47e7 100644 --- a/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py +++ b/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py @@ -366,20 +366,22 @@ async def _execute_code_dont_check_setup( os.path.abspath(self._virtual_env_context.env_exe) if self._virtual_env_context else sys.executable ) extra_args = [str(written_file.absolute())] - elif lang in {"pwsh", "powershell", "ps1"}: - # Use powershell/pwsh for .ps1 scripts with proper flags - program = "powershell" - extra_args = [ - "-NoProfile", - "-ExecutionPolicy", - "Bypass", - "-File", - str(written_file.absolute()), - ] else: - # Shell commands (bash, sh, etc.) + # Get the appropriate command for the language program = lang_to_cmd(lang) - extra_args = [str(written_file.absolute())] + + # Special handling for PowerShell + if program == "pwsh": + extra_args = [ + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(written_file.absolute()), + ] + else: + # Shell commands (bash, sh, etc.) + extra_args = [str(written_file.absolute())] # Create a subprocess and run task = asyncio.create_task( From 3423d2afbffd46d30bffad0c52fb4e4e9a40a6b9 Mon Sep 17 00:00:00 2001 From: Leonardo Pinheiro Date: Mon, 10 Mar 2025 13:27:57 +1000 Subject: [PATCH 9/9] fmt --- .../src/autogen_ext/code_executors/local/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py b/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py index 5742d4aa47e7..704c699a2665 100644 --- a/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py +++ b/python/packages/autogen-ext/src/autogen_ext/code_executors/local/__init__.py @@ -369,7 +369,7 @@ async def _execute_code_dont_check_setup( else: # Get the appropriate command for the language program = lang_to_cmd(lang) - + # Special handling for PowerShell if program == "pwsh": extra_args = [