diff --git a/README.md b/README.md index cccd769..64e9c4c 100644 --- a/README.md +++ b/README.md @@ -41,9 +41,8 @@ A set of PowerShell scripts to automate deployment of python applications. The a ## Developer Usage ### Deploy Application(s) -1. generate read-only deploy key for your application repo -2. generate yourself or request deploy key for gitpyup from Milo -3. (optional) add gitpyup.yml to your application repo(s) to generate shortcuts and or set the environment file +1. if application repos is private, generate read-only deploy key for your application repo +2. (optional) add gitpyup.yml to your application repo(s) to generate shortcuts and or set the environment file ```yml # example contents of optional gitpyup.yml in root of application repo @@ -57,22 +56,29 @@ shortcuts: # generate shortcuts script: myscript.ps1 ``` -4. create config file: yourAppName.yml +3. create config file: yourAppName.yml * In this example plotme is the only application. Application order doesn't matter. ```yml # .yml - name is up to you applications: -# gitpyup can be placed here to override the default that is in Deploy-gitpyup.ps1 +# optional, gitpyup can be placed here to override the default that is in Deploy-gitpyup.ps1 - name: gitpyup clone_uri: git@github.com:3Mcloud/gitpyup.git deploy_key: "replace with deploy key if needed" - name: plotme clone_uri: git@github.com:3mcloud/plotme.git - deploy_key: "replace with deploy key if needed" + # example key format + deploy_key: |- + -----BEGIN OPENSSH PRIVATE KEY----- + ********************************************** + ********************************************** + -----END OPENSSH PRIVATE KEY----- +# optional, needed for HTTPS inspection compatability (usually corporate environments) +tls_bundle: "https://raw.githubusercontent.com///main/tls-ca-bundle.txt" ``` -5. upload 3 files somewhere your users can access +4. upload 3 files somewhere your users can access 1. Deploy-gitpyup.ps1 2. yourAppName.yml 3. run-elevated-first.bat diff --git a/gitpyup/Deploy-gitpyup.ps1 b/gitpyup/Deploy-gitpyup.ps1 index 20629c2..11a6dfd 100644 --- a/gitpyup/Deploy-gitpyup.ps1 +++ b/gitpyup/Deploy-gitpyup.ps1 @@ -4,7 +4,7 @@ This script installs or updates gitpyup and its applications. #> param( [string]$YamlFile = "", - # '-debugMode' switch to enable debug mode + # '-DebugMode' switch to enable debug mode [switch]$DebugMode = $false, # -UseDev switch to checkout dev branch instead of main [switch]$UseDev = $false, @@ -16,12 +16,12 @@ param( # variables $scriptVersion = "v1" $gpun = "gitpyup" -$ENV:GITPYUPUTILSNAME = "Utility-Functions.ps1" +$Env:GITPYUPUTILSNAME = "Utility-Functions.ps1" $installConfigFile = "installConfig.yaml" $defaultGitpyup = @{ # default gitpyup config - name= $gpun - clone_uri= "https://github.com/3mcloud/gitpyup.git" + 'name' = $gpun + 'clone_uri' = "https://github.com/3mcloud/gitpyup.git" } # Test working directory and move to the script directory if needed @@ -30,18 +30,18 @@ if (Test-Path "gitpyup") { } # shortcut parent depends on installation type -$env:GITPYUP_SHORTCUT_PARENT_USER = "$env:APPDATA\Microsoft\Windows\Start Menu\Programs\$gpun" -$env:GITPYUP_SHORTCUT_PARENT_ALL = "$env:ProgramData\Microsoft\Windows\Start Menu\Programs\$gpun" +$Env:GITPYUP_SHORTCUT_PARENT_USER = "$Env:APPDATA\Microsoft\Windows\Start Menu\Programs\$gpun" +$Env:GITPYUP_SHORTCUT_PARENT_ALL = "$Env:ProgramData\Microsoft\Windows\Start Menu\Programs\$gpun" $toRemove = @( - "$env:GITPYUP_SHORTCUT_PARENT_USER\$gpun-update.lnk" - "$env:GITPYUP_SHORTCUT_PARENT_USER\$gpun-uninstall.lnk" - "$env:GITPYUP_SHORTCUT_PARENT_ALL\$gpun-update.lnk" - "$env:GITPYUP_SHORTCUT_PARENT_ALL\$gpun-uninstall.lnk" + "$Env:GITPYUP_SHORTCUT_PARENT_USER\$gpun-update.lnk" + "$Env:GITPYUP_SHORTCUT_PARENT_USER\$gpun-uninstall.lnk" + "$Env:GITPYUP_SHORTCUT_PARENT_ALL\$gpun-update.lnk" + "$Env:GITPYUP_SHORTCUT_PARENT_ALL\$gpun-uninstall.lnk" ) # setup logging -$sharepath = $env:USERPROFILE + "\Downloads" -$username = $env:USERNAME +$sharepath = $Env:USERPROFILE + "\Downloads" +$username = $Env:USERNAME $hostname = hostname $datetime = Get-Date -f 'yyyyMMddHHmmss' $filename = "Log-${username}-${hostname}-${datetime}.txt" @@ -54,7 +54,7 @@ $shortcutScript = { [array]$toAdd ) - . "./$ENV:GITPYUPUTILSNAME" + . "./$Env:GITPYUPUTILSNAME" Start-Logging # TODO: check if have permissions to remove, if needed get permissions, ideally from existing elevated process @@ -147,7 +147,7 @@ function Start-Logging { function Reset-Path { # reload path from machine and user - $env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + + $Env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User") } @@ -170,7 +170,7 @@ function Reset-Path { # save the Start-Logging function to a file # define the path to the file -$utilityFunctionsPath = Join-Path -Path (Get-Location).Path -ChildPath $ENV:GITPYUPUTILSNAME +$utilityFunctionsPath = Join-Path -Path (Get-Location).Path -ChildPath $Env:GITPYUPUTILSNAME # Write the function to the file Set-Content -Force -Path $utilityFunctionsPath -Value $utilityString @@ -268,7 +268,7 @@ $installSupportSoftware = { } } - . "./$ENV:GITPYUPUTILSNAME" + . "./$Env:GITPYUPUTILSNAME" Start-Logging -PrintVersion function Install-ViaWinget { @@ -343,13 +343,13 @@ if (!(Get-SupportStatus)) { # install type hash tables $installAll = @{ 'type' = "AllUsers" - 'path' = "$env:ProgramData\$gpun" - 'shortcutParent' = $env:GITPYUP_SHORTCUT_PARENT_ALL + 'path' = "$Env:ProgramData\$gpun" + 'shortcutParent' = $Env:GITPYUP_SHORTCUT_PARENT_ALL } $installUser = @{ 'type' = "SingleUser" - 'path' = "$env:APPDATA\$gpun" - 'shortcutParent' = $env:GITPYUP_SHORTCUT_PARENT_USER + 'path' = "$Env:APPDATA\$gpun" + 'shortcutParent' = $Env:GITPYUP_SHORTCUT_PARENT_USER } # determine install type @@ -406,7 +406,7 @@ prompt can be confusing because it doesn't show the user's input when user types #> # create empty known_hosts file if it does not exist -$knownHosts = "$env:USERPROFILE\.ssh\known_hosts" +$knownHosts = "$Env:USERPROFILE\.ssh\known_hosts" if (!(Test-Path $knownHosts)) { Write-Log "creating known_hosts file..." New-Item -Path $knownHosts -ItemType File -Force @@ -433,9 +433,9 @@ function Set-DeployKeyPermissions { # Remove Inheritance: Icacls $Key /c /t /Inheritance:d # Set Ownership to Owner: - # Key's within $env:UserProfile: + # Key's within $Env:UserProfile: Icacls $Key /c /t /Grant ${env:UserName}:F - # Key's outside of $env:UserProfile: + # Key's outside of $Env:UserProfile: TakeOwn /F $Key Icacls $Key /c /t /Grant:r ${env:UserName}:F # Remove All Users, except for Owner: @@ -626,6 +626,7 @@ if ($installConfigPathObject.Exists) { $toAdd = @() # initialize shortcuts to add array $appNames = @() # used to check for duplicate $appConfigs = @() # used to accumulate all the configs +$tlsBundleURL = $null # used to store the tls bundle path # load the yml files foreach ($file in $yamlFiles) { @@ -634,6 +635,11 @@ foreach ($file in $yamlFiles) { $configRoot = ConvertFrom-Yaml $fileContent $apps = $configRoot.applications + # update tls-bundle URL if it exists and is not already set + if (($null -eq $tlsBundleURL) -and $configRoot.ContainsKey("tls_bundle")) { + $tlsBundleURL = $configRoot.tls_bundle + } + # loop through each application foreach ($application in $apps) { $name = $application.name @@ -725,13 +731,14 @@ $installConfig = @{ "applications" = $appConfigs "created_shortcuts" = $toAdd "install" = $install + "tls_bundle" = $tlsBundleURL } # save the config to a file $response = ConvertTo-Yaml $installConfig -OutFile $installConfigPath Write-Log "The config yml has been written to $installConfigPath" -if (-not (Test-Path $ENV:GITPYUPUTILSNAME)){ +if (-not (Test-Path $Env:GITPYUPUTILSNAME)){ Copy-Item $utilityFunctionsPath . Write-Log "The application yml has been copied to $gitpyupScriptDir" } @@ -753,7 +760,8 @@ if ($Install.type -eq "AllUsers") { ) } -$env:GITPYUP_INSTALL_PARENT = Split-Path -Path $install.path -Parent +$Env:GITPYUP_BUNDLE_URL = $tlsBundleURL +$Env:GITPYUP_INSTALL_PARENT = Split-Path -Path $install.path -Parent $confirm = "" while(($confirm -ne "y") -and ($confirm -ne "n")) @@ -799,7 +807,8 @@ foreach ($application in $appConfigs) { { $confirm = Read-Host -Prompt "Do you want to install or update $name ? (y/n)" if ($confirm -ceq "y") { - $proc = Start-Process -FilePath "powershell" -PassThru -ArgumentList "-Command & '.\Setup-Application.ps1' -AppConfig $application -InstallType $install.type" + $proc = Start-Process -FilePath "powershell" -PassThru ` + -ArgumentList "-Command & '.\Setup-Application.ps1' -AppConfig $application -InstallType $($install.type)" $handle = $proc.Handle $proc.WaitForExit(); if ($proc.ExitCode -ne 0) { diff --git a/gitpyup/Setup-Python.ps1 b/gitpyup/Setup-Python.ps1 index fc41150..fda66f9 100644 --- a/gitpyup/Setup-Python.ps1 +++ b/gitpyup/Setup-Python.ps1 @@ -1,4 +1,5 @@ <# +Copyright (c) 2024 3M Company This script installs Miniforge3 for the 3M corporate environment. It can be run as part of gitpyup for now. #> @@ -111,7 +112,7 @@ if (!($CondaVersion)) { $EnvSetupScript = { param( - [string]$Repo, + [string]$EnvName, [string]$MiniforgeInstallPath, [string]$InstallType ) @@ -120,45 +121,54 @@ $EnvSetupScript = { . "./$ENV:GITPYUPUTILSNAME" Start-Logging - $BundlePath = "$env:ProgramData\tls-ca-bundle.pem" - if (Test-Path $BundlePath) { - Remove-Item -Force $BundlePath - } + # this sections is needed if the device is subject to SSL inspection + # currently Minforge/conda does not have a way to use the system certs + # The bundle URL is set by an environment variable + if ($Env:GITPYUP_BUNDLE_URL) { - # this is needed if corporation uses a SSL inspection aka MitM attack - # this bundle is for our corporation - $URL = "https://raw.githubusercontent.com/nikolarobottesla/bacon/main/bits.txt" - Write-Log "downloading tls bundle..." - Invoke-WebRequest $URL -OutFile $BundlePath - # are these redundant because setting the .condarc file? - conda config --set ssl_verify True - conda config --set ssl_verify $BundlePath - - # Check for existing conda environment - # TODO get any existing conda environment directories - - # configure path to save environments depending on installation type - if ($InstallType -eq "AllUsers") { - $EnvDir = "$env:ProgramData\.conda\envs" - } else { - $EnvDir = "$env:UserProfile\.conda\envs" - } + # remove existing gitpyup-tls-ca-bundle.pem if it exists + $BundlePath = "$env:ProgramData\gitpyup-tls-ca-bundle.pem" + if (Test-Path $BundlePath) { + Remove-Item -Force $BundlePath + } - # Create a .condarc file in the root dir of the MiniForge installation - $CondarcPath = "$MiniforgeInstallPath\.condarc" - $CondarcContent = + # download the tls bundle + $URL = $Env:GITPYUP_BUNDLE_URL + Write-Log "downloading tls bundle from $URL" + Invoke-WebRequest $URL -OutFile $BundlePath + # are these redundant because setting the .condarc file? + conda config --set ssl_verify True + conda config --set ssl_verify $BundlePath + + # Check for existing conda environment + # TODO get any existing conda environment directories + + # configure path to save environments depending on installation type + if ($InstallType -eq "AllUsers") { + $EnvDir = "$env:ProgramData\.conda\envs" + } else { + $EnvDir = "$env:UserProfile\.conda\envs" + } + + # Create a .condarc file in the root dir of the MiniForge installation + $CondarcPath = "$MiniforgeInstallPath\.condarc" + $CondarcContent = "channels: - conda-forge ssl_verify: $BundlePath envs_dirs: - - $EnvDir -" - Set-Content -Force -Path $CondarcPath -Value $CondarcContent + - $EnvDir + " + Set-Content -Force -Path $CondarcPath -Value $CondarcContent + + } else { + Write-Log "No bundle URL provided" + } # function to check if pip has SSL errors, return true if error detected function Test-PipTlsError { # Define the command - $Command = "conda run -n $Repo python -m pip install --dry-run tiny" + $Command = "conda run -n $EnvName python -m pip install --dry-run tiny" Write-Log "Running SSL test command: $Command" # Create a temporary file for output $TempFile = [System.IO.Path]::GetTempFileName() @@ -169,7 +179,7 @@ envs_dirs: # Clean up the temporary file Remove-Item -Path $TempFile - # $TlsTest = conda run -n $Repo python -m pip install --dry-run tiny + # $TlsTest = conda run -n $EnvName python -m pip install --dry-run tiny if ($TlsTest | Select-String -Pattern "SSL: CERTIFICATE_VERIFY_FAILED") { Write-Log "pip SSL error detected" return $true @@ -182,17 +192,17 @@ envs_dirs: # check if pip has SSL errors, install or uninstall pip-system-certs if (Test-PipTlsError) { # check if pip-system-certs is installed - if (!(conda run -n $Repo python -m pip list | Select-String -Pattern pip-system-certs)) { + if (!(conda run -n $EnvName python -m pip list | Select-String -Pattern pip-system-certs)) { # patch pip and requests to use system certs Write-Log "installing pip-system-certs..." - conda install -n $Repo pip-system-certs -y - # conda run -n $Repo python -m pip install --trusted-host pypi.org --trusted-host pypi.python.org --trusted-host files.pythonhosted.org pip-system-certs + conda install -n $EnvName pip-system-certs -y + # conda run -n $EnvName python -m pip install --trusted-host pypi.org --trusted-host pypi.python.org --trusted-host files.pythonhosted.org pip-system-certs } # check if pip still has SSL errors, set pip to use the tls-ca-bundle.pem if (Test-PipTlsError) { Write-Log "pip still has SSL errors, setting pip to use tls-ca-bundle.pem" - conda run -n $Repo python -m pip config set global.cert $BundlePath + conda run -n $EnvName python -m pip config set global.cert $BundlePath } }