From 5c753a70360ecce218a2cffbe8e84df06ac059ce Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Tue, 18 Jul 2023 17:13:14 -0700 Subject: [PATCH 1/5] Prevent posix paths locator from crashing --- .../lowLevel/posixKnownPathsLocator.ts | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/posixKnownPathsLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/posixKnownPathsLocator.ts index 2e4e2dc13e61..4cacbd53f5aa 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/posixKnownPathsLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/posixKnownPathsLocator.ts @@ -27,23 +27,27 @@ export class PosixKnownPathsLocator extends Locator { const iterator = async function* (kind: PythonEnvKind) { traceVerbose('Searching for interpreters in posix paths locator'); - // Filter out pyenv shims. They are not actual python binaries, they are used to launch - // the binaries specified in .python-version file in the cwd. We should not be reporting - // those binaries as environments. - const knownDirs = (await commonPosixBinPaths()).filter((dirname) => !isPyenvShimDir(dirname)); - let pythonBinaries = await getPythonBinFromPosixPaths(knownDirs); + try { + // Filter out pyenv shims. They are not actual python binaries, they are used to launch + // the binaries specified in .python-version file in the cwd. We should not be reporting + // those binaries as environments. + const knownDirs = (await commonPosixBinPaths()).filter((dirname) => !isPyenvShimDir(dirname)); + let pythonBinaries = await getPythonBinFromPosixPaths(knownDirs); - // Filter out MacOS system installs of Python 2 if necessary. - if (isMacPython2Deprecated) { - pythonBinaries = pythonBinaries.filter((binary) => !isMacDefaultPythonPath(binary)); - } + // Filter out MacOS system installs of Python 2 if necessary. + if (isMacPython2Deprecated) { + pythonBinaries = pythonBinaries.filter((binary) => !isMacDefaultPythonPath(binary)); + } - for (const bin of pythonBinaries) { - try { - yield { executablePath: bin, kind, source: [PythonEnvSource.PathEnvVar] }; - } catch (ex) { - traceError(`Failed to process environment: ${bin}`, ex); + for (const bin of pythonBinaries) { + try { + yield { executablePath: bin, kind, source: [PythonEnvSource.PathEnvVar] }; + } catch (ex) { + traceError(`Failed to process environment: ${bin}`, ex); + } } + } catch (ex) { + traceError('Failed to process posix paths', ex); } traceVerbose('Finished searching for interpreters in posix paths locator'); }; From 73a59bb65a3dc030f49dd3cd16d728d7dbba027c Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Tue, 18 Jul 2023 20:19:15 -0700 Subject: [PATCH 2/5] Add more logging --- .../base/locators/lowLevel/posixKnownPathsLocator.ts | 1 + .../pythonEnvironments/common/externalDependencies.ts | 10 +++++++--- src/client/pythonEnvironments/common/posixUtils.ts | 3 ++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/posixKnownPathsLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/posixKnownPathsLocator.ts index 4cacbd53f5aa..2d7ebc2af111 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/posixKnownPathsLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/posixKnownPathsLocator.ts @@ -33,6 +33,7 @@ export class PosixKnownPathsLocator extends Locator { // those binaries as environments. const knownDirs = (await commonPosixBinPaths()).filter((dirname) => !isPyenvShimDir(dirname)); let pythonBinaries = await getPythonBinFromPosixPaths(knownDirs); + traceVerbose(`Found ${pythonBinaries.length} python binaries in posix paths`); // Filter out MacOS system installs of Python 2 if necessary. if (isMacPython2Deprecated) { diff --git a/src/client/pythonEnvironments/common/externalDependencies.ts b/src/client/pythonEnvironments/common/externalDependencies.ts index 54f614ebdd49..a540f81ebcbe 100644 --- a/src/client/pythonEnvironments/common/externalDependencies.ts +++ b/src/client/pythonEnvironments/common/externalDependencies.ts @@ -10,7 +10,7 @@ import { IDisposable, IConfigurationService } from '../../common/types'; import { chain, iterable } from '../../common/utils/async'; import { getOSType, OSType } from '../../common/utils/platform'; import { IServiceContainer } from '../../ioc/types'; -import { traceVerbose } from '../../logging'; +import { traceError, traceVerbose } from '../../logging'; let internalServiceContainer: IServiceContainer; export function initializeExternalDependencies(serviceContainer: IServiceContainer): void { @@ -93,8 +93,12 @@ export function arePathsSame(path1: string, path2: string): boolean { return normCasePath(path1) === normCasePath(path2); } -export async function resolveSymbolicLink(absPath: string, stats?: fsapi.Stats): Promise { +export async function resolveSymbolicLink(absPath: string, stats?: fsapi.Stats, count?: number): Promise { stats = stats ?? (await fsapi.lstat(absPath)); + if (count && count > 5) { + traceError(`Too many symbolic links encountered: ${absPath}, stop resolving.`); + return absPath; + } if (stats.isSymbolicLink()) { const link = await fsapi.readlink(absPath); // Result from readlink is not guaranteed to be an absolute path. For eg. on Mac it resolves @@ -102,7 +106,7 @@ export async function resolveSymbolicLink(absPath: string, stats?: fsapi.Stats): // // The resultant path is reported relative to the symlink directory we resolve. Convert that to absolute path. const absLinkPath = path.isAbsolute(link) ? link : path.resolve(path.dirname(absPath), link); - return resolveSymbolicLink(absLinkPath); + return resolveSymbolicLink(absLinkPath, undefined, count ?? 0 + 1); } return absPath; } diff --git a/src/client/pythonEnvironments/common/posixUtils.ts b/src/client/pythonEnvironments/common/posixUtils.ts index cd8f62bf9a08..eb60fc029949 100644 --- a/src/client/pythonEnvironments/common/posixUtils.ts +++ b/src/client/pythonEnvironments/common/posixUtils.ts @@ -7,7 +7,7 @@ import * as path from 'path'; import { uniq } from 'lodash'; import { getSearchPathEntries } from '../../common/utils/exec'; import { resolveSymbolicLink } from './externalDependencies'; -import { traceError, traceInfo } from '../../logging'; +import { traceError, traceInfo, traceVerbose } from '../../logging'; /** * Determine if the given filename looks like the simplest Python executable. @@ -123,6 +123,7 @@ export async function getPythonBinFromPosixPaths(searchDirs: string[]): Promise< // Ensure that we have a collection of unique global binaries by // resolving all symlinks to the target binaries. try { + traceVerbose(`Attempting to resolve symbolic link: ${filepath}`); const resolvedBin = await resolveSymbolicLink(filepath); if (binToLinkMap.has(resolvedBin)) { binToLinkMap.get(resolvedBin)?.push(filepath); From 1aa768b2e20aa61383a0a50a8921915de85120e7 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Tue, 18 Jul 2023 22:38:56 -0700 Subject: [PATCH 3/5] Add additional logging --- .../common/externalDependencies.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/client/pythonEnvironments/common/externalDependencies.ts b/src/client/pythonEnvironments/common/externalDependencies.ts index a540f81ebcbe..a6c3c2ed58c5 100644 --- a/src/client/pythonEnvironments/common/externalDependencies.ts +++ b/src/client/pythonEnvironments/common/externalDependencies.ts @@ -95,18 +95,21 @@ export function arePathsSame(path1: string, path2: string): boolean { export async function resolveSymbolicLink(absPath: string, stats?: fsapi.Stats, count?: number): Promise { stats = stats ?? (await fsapi.lstat(absPath)); - if (count && count > 5) { - traceError(`Too many symbolic links encountered: ${absPath}, stop resolving.`); - return absPath; - } if (stats.isSymbolicLink()) { + if (count && count > 5) { + traceError(`Too many symbolic links encountered: ${absPath}, stop resolving.`); + return absPath; + } + traceVerbose(`Attempt to resolve symbolic link ${absPath}`); const link = await fsapi.readlink(absPath); + count = count ? count + 1 : 1; + traceVerbose(`Resolved symbolic link to ${absPath} -> ${link}`); // Result from readlink is not guaranteed to be an absolute path. For eg. on Mac it resolves // /usr/local/bin/python3.9 -> ../../../Library/Frameworks/Python.framework/Versions/3.9/bin/python3.9 // // The resultant path is reported relative to the symlink directory we resolve. Convert that to absolute path. const absLinkPath = path.isAbsolute(link) ? link : path.resolve(path.dirname(absPath), link); - return resolveSymbolicLink(absLinkPath, undefined, count ?? 0 + 1); + return resolveSymbolicLink(absLinkPath, undefined, count); } return absPath; } From 81ca76ad7c13edfb8dd120962f5aeefaa1b24ab7 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Tue, 18 Jul 2023 22:46:03 -0700 Subject: [PATCH 4/5] Build VSIX --- .github/workflows/pr-check.yml | 464 --------------------------------- 1 file changed, 464 deletions(-) diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index aa9cae2aa474..5c4d5502ef6a 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -33,467 +33,3 @@ jobs: node_version: ${{ env.NODE_VERSION}} vsix_name: ${{ env.VSIX_NAME }} artifact_name: ${{ env.ARTIFACT_NAME_VSIX }} - - lint: - name: Lint - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Lint - uses: ./.github/actions/lint - with: - node_version: ${{ env.NODE_VERSION }} - - check-types: - name: Check Python types - runs-on: ubuntu-latest - steps: - - name: Use Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Checkout - uses: actions/checkout@v3 - - - name: Install base Python requirements - uses: brettcannon/pip-secure-install@v1 - with: - options: '-t ./pythonFiles/lib/python --no-cache-dir --implementation py' - - - name: Install Jedi requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: './pythonFiles/jedilsp_requirements/requirements.txt' - options: '-t ./pythonFiles/lib/jedilsp --no-cache-dir --implementation py' - - - name: Install other Python requirements - run: | - python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy - python -m pip install --upgrade -r build/test-requirements.txt - - - name: Run Pyright - uses: jakebailey/pyright-action@v1 - with: - version: 1.1.308 - working-directory: 'pythonFiles' - - ### Non-smoke tests - tests: - name: Tests - # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. - runs-on: ${{ matrix.os }} - defaults: - run: - working-directory: ${{ env.special-working-directory }} - strategy: - fail-fast: false - matrix: - # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, - # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. - os: [ubuntu-latest, windows-latest] - # Run the tests on the oldest and most recent versions of Python. - python: ['3.x'] - test-suite: [ts-unit, python-unit, venv, single-workspace, debugger, functional] - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - path: ${{ env.special-working-directory-relative }} - - - name: Install Node - uses: actions/setup-node@v3 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - cache-dependency-path: ${{ env.special-working-directory-relative }}/package-lock.json - - - name: Install dependencies (npm ci) - run: npm ci - - - name: Compile - run: npx gulp prePublishNonBundle - - - name: Use Python ${{ matrix.python }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Install debugpy - run: | - # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase. - python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy - - - name: Download get-pip.py - run: | - python -m pip install wheel - python -m pip install -r build/build-install-requirements.txt - python ./pythonFiles/download_get_pip.py - shell: bash - - - name: Install base Python requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: '"${{ env.special-working-directory-relative }}/requirements.txt"' - options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/python" --no-cache-dir --implementation py' - if: startsWith(matrix.python, 3.) - - - name: Install Jedi requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: '"${{ env.special-working-directory-relative }}/pythonFiles/jedilsp_requirements/requirements.txt"' - options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/jedilsp" --no-cache-dir --implementation py' - if: startsWith(matrix.python, 3.) - - - name: Install test requirements - run: python -m pip install --upgrade -r build/test-requirements.txt - - - name: Install debugpy wheels (Python ${{ matrix.python }}) - run: | - python -m pip install wheel - python -m pip --disable-pip-version-check install -r build/build-install-requirements.txt - python ./pythonFiles/install_debugpy.py - shell: bash - if: matrix.test-suite == 'debugger' - - - name: Install functional test requirements - run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt - if: matrix.test-suite == 'functional' - - - name: Prepare pipenv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - if: matrix.test-suite == 'venv' - run: | - python -m pip install pipenv - python -m pipenv run python ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} pipenvPath - - - name: Prepare poetry for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - shell: pwsh - if: matrix.test-suite == 'venv' - run: | - python -m pip install poetry - Move-Item -Path ".\build\ci\pyproject.toml" -Destination . - poetry env use python - - - name: Prepare virtualenv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - if: matrix.test-suite == 'venv' - run: | - python -m pip install virtualenv - python -m virtualenv .virtualenv/ - if ('${{ matrix.os }}' -match 'windows-latest') { - & ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath - } else { - & ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath - } - - - name: Prepare venv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - if: matrix.test-suite == 'venv' && startsWith(matrix.python, 3.) - run: | - python -m venv .venv - if ('${{ matrix.os }}' -match 'windows-latest') { - & ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath - } else { - & ".venv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath - } - - - name: Prepare conda for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - if: matrix.test-suite == 'venv' - run: | - # 1. For `terminalActivation.testvirtualenvs.test.ts` - if ('${{ matrix.os }}' -match 'windows-latest') { - $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe - $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda - } else{ - $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python - $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda - } - & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaExecPath $condaExecPath - & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaPath - & $condaExecPath init --all - - - name: Set CI_PYTHON_PATH and CI_DISABLE_AUTO_SELECTION - run: | - echo "CI_PYTHON_PATH=python" >> $GITHUB_ENV - echo "CI_DISABLE_AUTO_SELECTION=1" >> $GITHUB_ENV - shell: bash - if: matrix.test-suite != 'ts-unit' - - # Run TypeScript unit tests only for Python 3.X. - - name: Run TypeScript unit tests - run: npm run test:unittests - if: matrix.test-suite == 'ts-unit' && startsWith(matrix.python, 3.) - - # Run the Python tests in our codebase. - - name: Run Python unit tests - run: | - python pythonFiles/tests/run_all.py - if: matrix.test-suite == 'python-unit' - - # The virtual environment based tests use the `testSingleWorkspace` set of tests - # with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`, - # which is set in the "Prepare environment for venv tests" step. - # We also use a third-party GitHub Action to install xvfb on Linux, - # run tests and then clean up the process once the tests ran. - # See https://github.com/GabrielBB/xvfb-action - - name: Run venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - CI_PYTHON_VERSION: ${{ matrix.python }} - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testSingleWorkspace - working-directory: ${{ env.special-working-directory }} - if: matrix.test-suite == 'venv' && matrix.os == 'ubuntu-latest' - - - name: Run single-workspace tests - env: - CI_PYTHON_VERSION: ${{ matrix.python }} - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testSingleWorkspace - working-directory: ${{ env.special-working-directory }} - if: matrix.test-suite == 'single-workspace' - - - name: Run debugger tests - env: - CI_PYTHON_VERSION: ${{ matrix.python }} - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testDebugger - working-directory: ${{ env.special-working-directory }} - if: matrix.test-suite == 'debugger' - - # Run TypeScript functional tests - - name: Run TypeScript functional tests - run: npm run test:functional - if: matrix.test-suite == 'functional' - - smoke-tests: - name: Smoke tests - # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. - runs-on: ${{ matrix.os }} - needs: [build-vsix] - strategy: - fail-fast: false - matrix: - # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, - # macOS runners are expensive, and we assume that Ubuntu is enough to cover the UNIX case. - os: [ubuntu-latest, windows-latest] - steps: - # Need the source to have the tests available. - - name: Checkout - uses: actions/checkout@v3 - - - name: Smoke tests - uses: ./.github/actions/smoke-tests - with: - node_version: ${{ env.NODE_VERSION }} - artifact_name: ${{ env.ARTIFACT_NAME_VSIX }} - - ### Coverage run - coverage: - name: Coverage - # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - # Only run coverage on linux for PRs - os: [ubuntu-latest] - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Install Node - uses: actions/setup-node@v3 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - - - name: Install dependencies (npm ci) - run: npm ci - - - name: Compile - run: npx gulp prePublishNonBundle - - - name: Use Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - cache-dependency-path: | - requirements.txt - pythonFiles/jedilsp_requirements/requirements.txt - build/test-requirements.txt - build/functional-test-requirements.txt - - - name: Install base Python requirements - uses: brettcannon/pip-secure-install@v1 - with: - options: '-t ./pythonFiles/lib/python --implementation py' - - - name: Install Jedi requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: './pythonFiles/jedilsp_requirements/requirements.txt' - options: '-t ./pythonFiles/lib/jedilsp --implementation py' - - - name: Install debugpy - run: | - # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase. - python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --implementation py --no-deps --upgrade --pre debugpy - - - name: Install test requirements - run: python -m pip install --upgrade -r build/test-requirements.txt - - - name: Install functional test requirements - run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt - - - name: Prepare pipenv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - run: | - python -m pip install pipenv - python -m pipenv run python ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} pipenvPath - - - name: Prepare poetry for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - shell: pwsh - run: | - python -m pip install poetry - Move-Item -Path ".\build\ci\pyproject.toml" -Destination . - poetry env use python - - - name: Prepare virtualenv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - run: | - python -m pip install virtualenv - python -m virtualenv .virtualenv/ - if ('${{ matrix.os }}' -match 'windows-latest') { - & ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath - } else { - & ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath - } - - - name: Prepare venv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - run: | - python -m venv .venv - if ('${{ matrix.os }}' -match 'windows-latest') { - & ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath - } else { - & ".venv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath - } - - - name: Prepare conda for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - run: | - # 1. For `terminalActivation.testvirtualenvs.test.ts` - if ('${{ matrix.os }}' -match 'windows-latest') { - $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe - $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda - } else{ - $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python - $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda - } - & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaExecPath $condaExecPath - & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaPath - & $condaExecPath init --all - - - name: Run TypeScript unit tests - run: npm run test:unittests:cover - - - name: Run Python unit tests - run: | - python pythonFiles/tests/run_all.py - - # The virtual environment based tests use the `testSingleWorkspace` set of tests - # with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`, - # which is set in the "Prepare environment for venv tests" step. - # We also use a third-party GitHub Action to install xvfb on Linux, - # run tests and then clean up the process once the tests ran. - # See https://github.com/GabrielBB/xvfb-action - - name: Run venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - CI_DISABLE_AUTO_SELECTION: 1 - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testSingleWorkspace:cover - - - name: Run single-workspace tests - env: - CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - CI_DISABLE_AUTO_SELECTION: 1 - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testSingleWorkspace:cover - - # Enable these tests when coverage is setup for multiroot workspace tests - # - name: Run multi-workspace tests - # env: - # CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - # CI_DISABLE_AUTO_SELECTION: 1 - # uses: GabrielBB/xvfb-action@v1.6 - # with: - # run: npm run testMultiWorkspace:cover - - # Enable these tests when coverage is setup for debugger tests - # - name: Run debugger tests - # env: - # CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - # CI_DISABLE_AUTO_SELECTION: 1 - # uses: GabrielBB/xvfb-action@v1.6 - # with: - # run: npm run testDebugger:cover - - # Run TypeScript functional tests - - name: Run TypeScript functional tests - env: - CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - CI_DISABLE_AUTO_SELECTION: 1 - run: npm run test:functional:cover - - - name: Generate coverage reports - run: npm run test:cover:report - - - name: Upload HTML report - uses: actions/upload-artifact@v3 - with: - name: ${{ runner.os }}-coverage-report-html - path: ./coverage - retention-days: 1 From 849cc409bb91a07e73a9ecfb6ee06a57b7938622 Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Wed, 19 Jul 2023 13:22:10 -0700 Subject: [PATCH 5/5] Improve logging --- .github/workflows/pr-check.yml | 464 ++++++++++++++++++ .../common/externalDependencies.ts | 6 +- 2 files changed, 466 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index 5c4d5502ef6a..aa9cae2aa474 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -33,3 +33,467 @@ jobs: node_version: ${{ env.NODE_VERSION}} vsix_name: ${{ env.VSIX_NAME }} artifact_name: ${{ env.ARTIFACT_NAME_VSIX }} + + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Lint + uses: ./.github/actions/lint + with: + node_version: ${{ env.NODE_VERSION }} + + check-types: + name: Check Python types + runs-on: ubuntu-latest + steps: + - name: Use Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Checkout + uses: actions/checkout@v3 + + - name: Install base Python requirements + uses: brettcannon/pip-secure-install@v1 + with: + options: '-t ./pythonFiles/lib/python --no-cache-dir --implementation py' + + - name: Install Jedi requirements + uses: brettcannon/pip-secure-install@v1 + with: + requirements-file: './pythonFiles/jedilsp_requirements/requirements.txt' + options: '-t ./pythonFiles/lib/jedilsp --no-cache-dir --implementation py' + + - name: Install other Python requirements + run: | + python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy + python -m pip install --upgrade -r build/test-requirements.txt + + - name: Run Pyright + uses: jakebailey/pyright-action@v1 + with: + version: 1.1.308 + working-directory: 'pythonFiles' + + ### Non-smoke tests + tests: + name: Tests + # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: ${{ env.special-working-directory }} + strategy: + fail-fast: false + matrix: + # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, + # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. + os: [ubuntu-latest, windows-latest] + # Run the tests on the oldest and most recent versions of Python. + python: ['3.x'] + test-suite: [ts-unit, python-unit, venv, single-workspace, debugger, functional] + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + path: ${{ env.special-working-directory-relative }} + + - name: Install Node + uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + cache-dependency-path: ${{ env.special-working-directory-relative }}/package-lock.json + + - name: Install dependencies (npm ci) + run: npm ci + + - name: Compile + run: npx gulp prePublishNonBundle + + - name: Use Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Install debugpy + run: | + # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase. + python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy + + - name: Download get-pip.py + run: | + python -m pip install wheel + python -m pip install -r build/build-install-requirements.txt + python ./pythonFiles/download_get_pip.py + shell: bash + + - name: Install base Python requirements + uses: brettcannon/pip-secure-install@v1 + with: + requirements-file: '"${{ env.special-working-directory-relative }}/requirements.txt"' + options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/python" --no-cache-dir --implementation py' + if: startsWith(matrix.python, 3.) + + - name: Install Jedi requirements + uses: brettcannon/pip-secure-install@v1 + with: + requirements-file: '"${{ env.special-working-directory-relative }}/pythonFiles/jedilsp_requirements/requirements.txt"' + options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/jedilsp" --no-cache-dir --implementation py' + if: startsWith(matrix.python, 3.) + + - name: Install test requirements + run: python -m pip install --upgrade -r build/test-requirements.txt + + - name: Install debugpy wheels (Python ${{ matrix.python }}) + run: | + python -m pip install wheel + python -m pip --disable-pip-version-check install -r build/build-install-requirements.txt + python ./pythonFiles/install_debugpy.py + shell: bash + if: matrix.test-suite == 'debugger' + + - name: Install functional test requirements + run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt + if: matrix.test-suite == 'functional' + + - name: Prepare pipenv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + if: matrix.test-suite == 'venv' + run: | + python -m pip install pipenv + python -m pipenv run python ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} pipenvPath + + - name: Prepare poetry for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + shell: pwsh + if: matrix.test-suite == 'venv' + run: | + python -m pip install poetry + Move-Item -Path ".\build\ci\pyproject.toml" -Destination . + poetry env use python + + - name: Prepare virtualenv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + if: matrix.test-suite == 'venv' + run: | + python -m pip install virtualenv + python -m virtualenv .virtualenv/ + if ('${{ matrix.os }}' -match 'windows-latest') { + & ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath + } else { + & ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath + } + + - name: Prepare venv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + if: matrix.test-suite == 'venv' && startsWith(matrix.python, 3.) + run: | + python -m venv .venv + if ('${{ matrix.os }}' -match 'windows-latest') { + & ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath + } else { + & ".venv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath + } + + - name: Prepare conda for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + if: matrix.test-suite == 'venv' + run: | + # 1. For `terminalActivation.testvirtualenvs.test.ts` + if ('${{ matrix.os }}' -match 'windows-latest') { + $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe + $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda + } else{ + $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python + $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda + } + & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaExecPath $condaExecPath + & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaPath + & $condaExecPath init --all + + - name: Set CI_PYTHON_PATH and CI_DISABLE_AUTO_SELECTION + run: | + echo "CI_PYTHON_PATH=python" >> $GITHUB_ENV + echo "CI_DISABLE_AUTO_SELECTION=1" >> $GITHUB_ENV + shell: bash + if: matrix.test-suite != 'ts-unit' + + # Run TypeScript unit tests only for Python 3.X. + - name: Run TypeScript unit tests + run: npm run test:unittests + if: matrix.test-suite == 'ts-unit' && startsWith(matrix.python, 3.) + + # Run the Python tests in our codebase. + - name: Run Python unit tests + run: | + python pythonFiles/tests/run_all.py + if: matrix.test-suite == 'python-unit' + + # The virtual environment based tests use the `testSingleWorkspace` set of tests + # with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`, + # which is set in the "Prepare environment for venv tests" step. + # We also use a third-party GitHub Action to install xvfb on Linux, + # run tests and then clean up the process once the tests ran. + # See https://github.com/GabrielBB/xvfb-action + - name: Run venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + CI_PYTHON_VERSION: ${{ matrix.python }} + uses: GabrielBB/xvfb-action@v1.6 + with: + run: npm run testSingleWorkspace + working-directory: ${{ env.special-working-directory }} + if: matrix.test-suite == 'venv' && matrix.os == 'ubuntu-latest' + + - name: Run single-workspace tests + env: + CI_PYTHON_VERSION: ${{ matrix.python }} + uses: GabrielBB/xvfb-action@v1.6 + with: + run: npm run testSingleWorkspace + working-directory: ${{ env.special-working-directory }} + if: matrix.test-suite == 'single-workspace' + + - name: Run debugger tests + env: + CI_PYTHON_VERSION: ${{ matrix.python }} + uses: GabrielBB/xvfb-action@v1.6 + with: + run: npm run testDebugger + working-directory: ${{ env.special-working-directory }} + if: matrix.test-suite == 'debugger' + + # Run TypeScript functional tests + - name: Run TypeScript functional tests + run: npm run test:functional + if: matrix.test-suite == 'functional' + + smoke-tests: + name: Smoke tests + # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. + runs-on: ${{ matrix.os }} + needs: [build-vsix] + strategy: + fail-fast: false + matrix: + # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, + # macOS runners are expensive, and we assume that Ubuntu is enough to cover the UNIX case. + os: [ubuntu-latest, windows-latest] + steps: + # Need the source to have the tests available. + - name: Checkout + uses: actions/checkout@v3 + + - name: Smoke tests + uses: ./.github/actions/smoke-tests + with: + node_version: ${{ env.NODE_VERSION }} + artifact_name: ${{ env.ARTIFACT_NAME_VSIX }} + + ### Coverage run + coverage: + name: Coverage + # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # Only run coverage on linux for PRs + os: [ubuntu-latest] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install Node + uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + + - name: Install dependencies (npm ci) + run: npm ci + + - name: Compile + run: npx gulp prePublishNonBundle + + - name: Use Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + cache-dependency-path: | + requirements.txt + pythonFiles/jedilsp_requirements/requirements.txt + build/test-requirements.txt + build/functional-test-requirements.txt + + - name: Install base Python requirements + uses: brettcannon/pip-secure-install@v1 + with: + options: '-t ./pythonFiles/lib/python --implementation py' + + - name: Install Jedi requirements + uses: brettcannon/pip-secure-install@v1 + with: + requirements-file: './pythonFiles/jedilsp_requirements/requirements.txt' + options: '-t ./pythonFiles/lib/jedilsp --implementation py' + + - name: Install debugpy + run: | + # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase. + python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --implementation py --no-deps --upgrade --pre debugpy + + - name: Install test requirements + run: python -m pip install --upgrade -r build/test-requirements.txt + + - name: Install functional test requirements + run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt + + - name: Prepare pipenv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + run: | + python -m pip install pipenv + python -m pipenv run python ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} pipenvPath + + - name: Prepare poetry for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + shell: pwsh + run: | + python -m pip install poetry + Move-Item -Path ".\build\ci\pyproject.toml" -Destination . + poetry env use python + + - name: Prepare virtualenv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + run: | + python -m pip install virtualenv + python -m virtualenv .virtualenv/ + if ('${{ matrix.os }}' -match 'windows-latest') { + & ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath + } else { + & ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath + } + + - name: Prepare venv for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + run: | + python -m venv .venv + if ('${{ matrix.os }}' -match 'windows-latest') { + & ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath + } else { + & ".venv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath + } + + - name: Prepare conda for venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' + shell: pwsh + run: | + # 1. For `terminalActivation.testvirtualenvs.test.ts` + if ('${{ matrix.os }}' -match 'windows-latest') { + $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe + $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda + } else{ + $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python + $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda + } + & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaExecPath $condaExecPath + & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaPath + & $condaExecPath init --all + + - name: Run TypeScript unit tests + run: npm run test:unittests:cover + + - name: Run Python unit tests + run: | + python pythonFiles/tests/run_all.py + + # The virtual environment based tests use the `testSingleWorkspace` set of tests + # with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`, + # which is set in the "Prepare environment for venv tests" step. + # We also use a third-party GitHub Action to install xvfb on Linux, + # run tests and then clean up the process once the tests ran. + # See https://github.com/GabrielBB/xvfb-action + - name: Run venv tests + env: + TEST_FILES_SUFFIX: testvirtualenvs + CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + CI_DISABLE_AUTO_SELECTION: 1 + uses: GabrielBB/xvfb-action@v1.6 + with: + run: npm run testSingleWorkspace:cover + + - name: Run single-workspace tests + env: + CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + CI_DISABLE_AUTO_SELECTION: 1 + uses: GabrielBB/xvfb-action@v1.6 + with: + run: npm run testSingleWorkspace:cover + + # Enable these tests when coverage is setup for multiroot workspace tests + # - name: Run multi-workspace tests + # env: + # CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + # CI_DISABLE_AUTO_SELECTION: 1 + # uses: GabrielBB/xvfb-action@v1.6 + # with: + # run: npm run testMultiWorkspace:cover + + # Enable these tests when coverage is setup for debugger tests + # - name: Run debugger tests + # env: + # CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + # CI_DISABLE_AUTO_SELECTION: 1 + # uses: GabrielBB/xvfb-action@v1.6 + # with: + # run: npm run testDebugger:cover + + # Run TypeScript functional tests + - name: Run TypeScript functional tests + env: + CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} + CI_DISABLE_AUTO_SELECTION: 1 + run: npm run test:functional:cover + + - name: Generate coverage reports + run: npm run test:cover:report + + - name: Upload HTML report + uses: actions/upload-artifact@v3 + with: + name: ${{ runner.os }}-coverage-report-html + path: ./coverage + retention-days: 1 diff --git a/src/client/pythonEnvironments/common/externalDependencies.ts b/src/client/pythonEnvironments/common/externalDependencies.ts index a6c3c2ed58c5..ecb6f2212aba 100644 --- a/src/client/pythonEnvironments/common/externalDependencies.ts +++ b/src/client/pythonEnvironments/common/externalDependencies.ts @@ -97,18 +97,16 @@ export async function resolveSymbolicLink(absPath: string, stats?: fsapi.Stats, stats = stats ?? (await fsapi.lstat(absPath)); if (stats.isSymbolicLink()) { if (count && count > 5) { - traceError(`Too many symbolic links encountered: ${absPath}, stop resolving.`); + traceError(`Detected a potential symbolic link loop at ${absPath}, terminating resolution.`); return absPath; } - traceVerbose(`Attempt to resolve symbolic link ${absPath}`); const link = await fsapi.readlink(absPath); - count = count ? count + 1 : 1; - traceVerbose(`Resolved symbolic link to ${absPath} -> ${link}`); // Result from readlink is not guaranteed to be an absolute path. For eg. on Mac it resolves // /usr/local/bin/python3.9 -> ../../../Library/Frameworks/Python.framework/Versions/3.9/bin/python3.9 // // The resultant path is reported relative to the symlink directory we resolve. Convert that to absolute path. const absLinkPath = path.isAbsolute(link) ? link : path.resolve(path.dirname(absPath), link); + count = count ? count + 1 : 1; return resolveSymbolicLink(absLinkPath, undefined, count); } return absPath;