🎉 SpecFile Detection and LockFIles

ModuleFast.psm1

@@ -28,10 +28,14 @@ $SCRIPT:DefaultSource = 'https://pwsh.gallery/index.json'
 #region Public
 <#
 .SYNOPSIS
-High Performance Powershell Module Installation
-.NOTES
-THIS IS NOT FOR PRODUCTION, it should be considered "Fragile" and has very little error handling and type safety
-It also doesn't generate the PowershellGet XML files currently, so PSGet v2 will see them as "External" modules (PSGetv3 doesn't care)
+High performance, declarative Powershell Module Installer
+.DESCRIPTION
+ModuleFast is a high performance, declarative PowerShell module installer. It is designed with no external dependencies and can be bootstrapped in a single line of code. It is ideal for Continuous Integration/Deployment and serverless scenarios where you want to install modules quickly and without any user interaction. It is inspired by pnpm and other high performance declarative package managers.
+
+ModuleFast accepts a variety of familiar PowerShell syntaxes and objects for module specification as well as a custom shorthand syntax allowing complex version requirements to be defined in a single string.
+
+ModuleFast can also install the required modules specified in the #Requires line of a script, or in the RequiredModules section of a module manifest, by simplying providing the path to that file in the -Path parameter (which also accepts remote UNC, http, and https URLs).
+
 .EXAMPLE
 Install-ModuleFast 'Az'
 .EXAMPLE
@@ -50,11 +54,11 @@ function Install-ModuleFast {
     [Alias('ModulesToInstall')]
     [AllowNull()]
     [AllowEmptyCollection()]
-    [Parameter(Mandatory, Position = 0, ValueFromPipeline, ParameterSetName = 'Specification')][ModuleFastSpec[]]$Specification,
+    [Parameter(Position = 0, ValueFromPipeline, ParameterSetName = 'Specification')][ModuleFastSpec[]]$Specification,
 
     #Provide a required module specification path to install from. This can be a local psd1/json file, or a remote URL with a psd1/json file in supported manifest formats, or a .ps1/.psm1 file with a #Requires statement.
     [Parameter(Mandatory, ParameterSetName = 'Path')][string]$Path,
-    #Where to install the modules. This defaults to the builtin module path on non-windows and a custom LOCALAPPDATA location on Windows.
+    #Where to install the modules. This defaults to the builtin module path on non-windows and a custom LOCALAPPDATA location on Windows. You can also specify 'CurrentUser' to install to the Documents folder on Windows Only (this is not recommended)
     [string]$Destination,
     #The repository to scan for modules. TODO: Multi-repo support
     [string]$Source = $SCRIPT:DefaultSource,
@@ -68,13 +72,20 @@ function Install-ModuleFast {
     [Switch]$Update,
     #Consider prerelease packages in the evaluation. Note that if a non-prerelease package has a prerelease dependency, that dependency will be included regardless of this setting.
     [Switch]$Prerelease,
+    #Using the CI switch will write a lockfile to the current folder. If this file is present and -CI is specified in the future, ModuleFast will only install the versions specified in the lockfile, which is useful for reproducing CI builds even if newer versions of software come out.
+    [Switch]$CI,
+    #The path to the lockfile. By default it is requires.lock.json in the current folder. This is ignored if CI is not present. It is generally not recommended to change this setting.
+    [string]$CILockFilePath = $(Join-Path $PWD 'requires.lock.json'),
     [Parameter(Mandatory, ValueFromPipeline, ParameterSetName = 'ModuleFastInfo')][ModuleFastInfo]$ModuleFastInfo
   )
   begin {
     # Setup the Destination repository
-    $defaultRepoPath = $(Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'powershell/Modules')
     if (-not $Destination) {
+      $defaultRepoPath = $(Join-Path ([Environment]::GetFolderPath('LocalApplicationData')) 'powershell/Modules')
       $Destination = $defaultRepoPath
+    } elseif ($IsWindows -and $Destination -eq 'CurrentUser') {
+      $windowsDefaultDocumentsPath = Join-Path [environment]::GetFolderPath('MyDocuments') 'PowerShell/Modules'
+      $Destination = $windowsDefaultDocumentsPath
     }
 
     # Autocreate the default as a convenience, otherwise require the path to be present to avoid mistakes
@@ -143,13 +154,22 @@ function Install-ModuleFast {
   }
 
   end {
-    if (-not $ModulesToInstall) {
-      if ($WhatIfPreference) {
-        Write-Host -fore DarkGreen "`u{2705} All module specifications have been satisfied by locally installed modules. If you want to look for newer versions, try -Update."
+
+    if ($ModulesToInstall.Count -eq 0 -and $PSCmdlet.ParameterSetName -eq 'Specification') {
+      Write-Verbose 'No modules specified to install. Beginning SpecFile detection...'
+      $modulesToInstall = if ($CI -and (Test-Path $CILockFilePath)) {
+        Write-Debug "Found lockfile at $CILockFilePath. Using for specification evaluation and ignoring all others."
+        ConvertFrom-RequiredSpec -RequiredSpecPath $CILockFilePath
+      } else {
+        $specFiles = Find-RequiredSpecFile $PWD -CILockFileHint $CILockFilePath
+        foreach ($specfile in $specFiles) {
+          ConvertFrom-RequiredSpec -RequiredSpecPath $Path
+        }
       }
-      #TODO: Deduplicate this with the end into its own function
-      Write-Verbose "`u{2705} All required modules installed! Exiting."
-      return
+    }
+
+    if (-not $ModulesToInstall) {
+      throw [InvalidDataException]'No modules specifications found to evaluate.'
     }
 
     #If we do not have an explicit implementation plan, fetch it
@@ -164,11 +184,12 @@ function Install-ModuleFast {
     $WhatIfPreference = $currentWhatIfPreference
 
     if ($plan.Count -eq 0) {
+      $planAlreadySatisfiedMessage = "`u{2705} $($ModulesToInstall.count) Module Specifications have all been satisfied by installed modules. If you would like to check for newer versions remotely, specify -Update"
       if ($WhatIfPreference) {
-        Write-Host -fore DarkGreen "`u{2705} No modules found to install or all modules are already installed."
+        Write-Host -fore DarkGreen $planAlreadySatisfiedMessage
       }
       #TODO: Deduplicate this with the end into its own function
-      Write-Verbose "`u{2705} All required modules installed! Exiting."
+      Write-Verbose $planAlreadySatisfiedMessage
       return
     }
 
@@ -194,6 +215,19 @@ function Install-ModuleFast {
     Install-ModuleFastHelper @installHelperParams
     Write-Progress -Id 1 -Activity 'Install-ModuleFast' -Completed
     Write-Verbose "`u{2705} All required modules installed! Exiting."
+    if ($CI) {
+      #FIXME: If a package was already installed, it doesn't show up in this lockfile.
+      Write-Verbose "Writing lockfile to $CILockFilePath"
+      [Dictionary[string, string]]$lockFile = @{}
+      $plan
+      | ForEach-Object {
+        $lockFile.Add($PSItem.Name, $PSItem.ModuleVersion)
+      }
+
+      $lockFile
+      | ConvertTo-Json -Depth 2
+      | Out-File -FilePath $CILockFilePath -Encoding UTF8
+    }
   }
 
 }
@@ -377,6 +411,7 @@ function Get-ModuleFastPlan {
             }
 
             if ($currentModuleSpec.SatisfiedBy($candidate)) {
+              #TODO: If the found version still matches an installed version, we should skip it
               Write-Debug "$currentModuleSpec`: Found satisfying version $candidate in the inlined index."
               $matchingEntry = $entries | Where-Object version -EQ $candidate
               if ($matchingEntry.count -gt 1) { throw 'Multiple matching Entries found for a specific version. This is a bug and should not happen' }
@@ -1415,6 +1450,22 @@ filter ConvertFrom-RequiredSpec {
   throw [InvalidDataException]'Could not evaluate the Required Specification to a known format.'
 }
 
+function Find-RequiredSpecFile ([string]$Path) {
+  Write-Debug "Attempting to find a Required Spec file at $Path"
+
+  $resolvedPath = Resolve-Path $Path
+
+  $requireFiles = Get-Item $resolvedPath/*.requires.* -ErrorAction SilentlyContinue
+  | Where-Object {
+    $extension = [Path]::GetExtension($_.FullName)
+    $extension -in '.psd1', '.ps1', '.psm1', '.json', '.jsonc'
+  }
+
+  if (-not $requireFiles) {
+    throw [NotSupportedException]"Could not find any required spec files in $Path. Verify the path is correct or specify Module Specifications either via -Path or -Specification"
+  }
+}
+
 function Read-RequiredSpecFile ($RequiredSpecPath) {
   if ($uri.scheme -in 'http', 'https') {
     [string]$content = (Invoke-WebRequest -Uri $uri).Content
@@ -1485,3 +1536,4 @@ filter Resolve-FolderVersion([NuGetVersion]$version) {
 # FIXME: DBops dependency version issue
 
 Export-ModuleMember -Function Get-ModuleFastPlan, Install-ModuleFast
+Set-Alias imf -Value Install-ModuleFast

ModuleFast.tests.ps1

@@ -572,4 +572,27 @@ Describe 'Install-ModuleFast' -Tag 'E2E' {
     $modules = Install-ModuleFast @imfParams -Path $scriptPath -WhatIf
     $modules.count | Should -Be 2
   }
+
+  It 'Writes a CI File' {
+    Set-Location $testDrive
+    Install-ModuleFast @imfParams -CI -Specification 'PreReleaseTest'
+    Get-Item 'requires.lock.json' | Should -Not -BeNullOrEmpty
+    #TODO: CI Content
+  }
+
+  It 'Installs from CI File and Installs CI Pinned Version' {
+    Set-Location $testDrive
+    Install-ModuleFast @imfParams -CI -Specification 'PreReleaseTest@0.0.1-prerelease'
+    Get-Item 'requires.lock.json' | Should -Not -BeNullOrEmpty
+
+    Remove-Item $imfParams.Destination -Recurse -Force
+    New-Item -ItemType Directory -Path $imfParams.Destination -ErrorAction stop
+    Install-ModuleFast @imfParams -CI
+    $PreReleaseManifest = "$($imfParams.Destination)\PreReleaseTest\0.0.1\PreReleaseTest.psd1"
+    Resolve-Path $PreReleaseManifest
+
+    (Import-PowerShellDataFile $PreReleaseManifest).PrivateData.PSData.Prerelease
+    | Should -Be 'prerelease' -Because 'CI lock file should have 0.1 prerelease even if 0.2 is available'
+    #TODO: CI Content
+  }
 }

README.MD

@@ -41,6 +41,26 @@ you can have only a very specific version to be installed, or only versions belo
 & ([scriptblock]::Create((iwr 'bit.ly/modulefast'))) @{ModuleName='ImportExcel';MaximumVersion='7.7.0'}
 ```
 
+### Requires Spec Installation
+
+ModuleFast can read a variety of files for module specifications:
+
+1. .ps1 PowerShell Scripts with #Requires comment
+1. .psm1 PowerShell Modules with #Requires comment
+1. .psd1 Module Manifest with a RequiredModules property
+1. .psd1 file with module/version pairs
+1. .json file with module/version pairs
+1. .json file with array of Module Specification strings
+
+You can specify one of these files by using the `-Path parameter`. If you run `Install-ModuleFast` with no arguments, it will search for *.required.psd1|psm1|json|jsonc files in the current directory to install.
+
+### CI Lockfile
+
+You can specify the `-CI` parameter to create a lockfile in the current directory that will pin specific version specifications.
+If you commit this to a repository, others who run `Install-ModuleFast -CI` will get exactly the modules that were installed
+by your installation process, even if newer ones are available that meet the spec. This helps ensure proper
+reproducible builds, but ideally you will specify your versions appropriately instead.
+
 ### NOTE: Recommended PSModulePath Prerequisite
 
 ModuleFast installs modules to your `LocalAppData/powershell/Modules` folder.
@@ -87,7 +107,7 @@ It makes a lot of very bad assumptions, most of which are safe for PowerShell Ga
 
 1. Multipart $batch queries suck, they still only execute server-side in parallel. I wrote a full multipart implementation only to throw it away
 1. Fiddler and some proxies only supports HTTP/1.1 and can give false positives about how many connections are actually being made. Use wireshark to be sure when testing HTTP/2 and HTTP/3
-1. [Dictionary].keys is not a stable target at all for task iteration, best to maintain a separate list
+1. `[Dictionary].keys`` is not a stable target at all for task iteration, best to maintain a separate list
 1. PSGetv3 doesn't follow the nuget server v3 spec of optional params (stuff marked optional will cause psgetv3 to throw if not present)
 1. Initially the logic would fetch the main page and resolve dependencies by fetching individual versions. Turns out its only barely slower to return all versions of a module in single call, probably because the PSGallery server-side filtering doesnt cache, so we instead fetch all versions of a module in a reduced way from our Cloudflare worker and use that as a cache.
 1. Parallel dependency lookups (lots of Az dependencies require Az.Account) resulted in lots of duplicate calls for Az.Account. Since all task calls bottleneck through our main async loop logic, we can safely inspect existing dependency calls just before execution to determine if the existing call will satisfy it, and then add the context on to the existing object to prevent the duplicate calls.