diff --git a/dsc/tests/dsc_discovery.tests.ps1 b/dsc/tests/dsc_discovery.tests.ps1 index e941540f0..d394ca8b4 100644 --- a/dsc/tests/dsc_discovery.tests.ps1 +++ b/dsc/tests/dsc_discovery.tests.ps1 @@ -4,6 +4,12 @@ Describe 'tests for resource discovery' { BeforeAll { $env:DSC_RESOURCE_PATH = $testdrive + + $script:lookupTableFilePath = if ($IsWindows) { + Join-Path $env:LocalAppData "dsc\AdaptedResourcesLookupTable.json" + } else { + Join-Path $env:HOME ".dsc" "AdaptedResourcesLookupTable.json" + } } AfterEach { @@ -96,4 +102,52 @@ Describe 'tests for resource discovery' { $env:DSC_RESOURCE_PATH = $oldPath } } + + It 'Ensure List operation populates adapter lookup table' { + # remove adapter lookup table file + Remove-Item -Force -Path $script:lookupTableFilePath -ErrorAction SilentlyContinue + Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeFalse + + # perform List on an adapter - this should create adapter lookup table file + $oldPSModulePath = $env:PSModulePath + $TestClassResourcePath = Resolve-Path "$PSScriptRoot/../../powershell-adapter/Tests" + $env:DSC_RESOURCE_PATH = $null + $env:PSModulePath += [System.IO.Path]::PathSeparator + $TestClassResourcePath + dsc resource list -a Microsoft.DSC/PowerShell | Out-Null + gc -raw $script:lookupTableFilePath + $script:lookupTableFilePath | Should -FileContentMatchExactly 'Microsoft.DSC/PowerShell' + Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeTrue + $env:PSModulePath = $oldPSModulePath + } + + It 'Ensure non-List operation populates adapter lookup table' { + + # remove adapter lookup table file + Remove-Item -Force -Path $script:lookupTableFilePath -ErrorAction SilentlyContinue + Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeFalse + + # perform Get on an adapter - this should create adapter lookup table file + $oldPSModulePath = $env:PSModulePath + $TestClassResourcePath = Resolve-Path "$PSScriptRoot/../../powershell-adapter/Tests" + $env:DSC_RESOURCE_PATH = $null + $env:PSModulePath += [System.IO.Path]::PathSeparator + $TestClassResourcePath + "{'Name':'TestClassResource1'}" | dsc resource get -r 'TestClassResource/TestClassResource' | Out-Null + + Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeTrue + $script:lookupTableFilePath | Should -FileContentMatchExactly 'testclassresource/testclassresource' + $env:PSModulePath = $oldPSModulePath + } + + It 'Verify adapter lookup table is used on repeat invocations' { + + $oldPSModulePath = $env:PSModulePath + $TestClassResourcePath = Resolve-Path "$PSScriptRoot/../../powershell-adapter/Tests" + $env:DSC_RESOURCE_PATH = $null + $env:PSModulePath += [System.IO.Path]::PathSeparator + $TestClassResourcePath + dsc resource list -a Microsoft.DSC/PowerShell | Out-Null + "{'Name':'TestClassResource1'}" | dsc -l trace resource get -r 'TestClassResource/TestClassResource' 2> $TestDrive/tracing.txt + + "$TestDrive/tracing.txt" | Should -FileContentMatchExactly "Lookup table found resource 'testclassresource/testclassresource' in adapter 'Microsoft.DSC/PowerShell'" + $env:PSModulePath = $oldPSModulePath + } } diff --git a/dsc_lib/Cargo.toml b/dsc_lib/Cargo.toml index cd8244f4d..d382c4547 100644 --- a/dsc_lib/Cargo.toml +++ b/dsc_lib/Cargo.toml @@ -10,6 +10,7 @@ clap = { version = "4.4", features = ["derive"] } derive_builder ="0.20.0" indicatif = "0.17.0" jsonschema = "0.18.0" +linked-hash-map = "0.5.6" num-traits = "0.2.14" regex = "1.7.0" reqwest = { version = "0.12.0", features = ["rustls-tls"], default-features = false } diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index c3410dc9a..04772ee75 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -8,11 +8,13 @@ use crate::dscresources::resource_manifest::{import_manifest, validate_semver, K use crate::dscresources::command_resource::invoke_command; use crate::dscerror::DscError; use indicatif::ProgressStyle; +use linked_hash_map::LinkedHashMap; use regex::RegexBuilder; use semver::Version; -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, HashSet, HashMap}; use std::env; use std::ffi::OsStr; +use std::fs; use std::fs::File; use std::io::BufReader; use std::path::{Path, PathBuf}; @@ -279,6 +281,7 @@ impl ResourceDiscovery for CommandDiscovery { } self.adapted_resources = adapted_resources; + Ok(()) } @@ -294,6 +297,11 @@ impl ResourceDiscovery for CommandDiscovery { } else { self.discover_resources("*")?; self.discover_adapted_resources(type_name_filter, adapter_name_filter)?; + + // add/update found adapted resources to the lookup_table + add_resources_to_lookup_table(&self.adapted_resources); + + // note: in next line 'BTreeMap::append' will leave self.adapted_resources empty resources.append(&mut self.adapted_resources); } @@ -334,7 +342,8 @@ impl ResourceDiscovery for CommandDiscovery { debug!("Found {} matching non-adapter-based resources", found_resources.len()); // now go through the adapters - for (adapter_name, adapters) in self.adapters.clone() { + let sorted_adapters = sort_adapters_based_on_lookup_table(&self.adapters, &remaining_required_resource_types); + for (adapter_name, adapters) in sorted_adapters { // TODO: handle version requirements let Some(adapter) = adapters.first() else { // skip if no adapters @@ -353,6 +362,8 @@ impl ResourceDiscovery for CommandDiscovery { } self.discover_adapted_resources("*", &adapter_name)?; + // add/update found adapted resources to the lookup_table + add_resources_to_lookup_table(&self.adapted_resources); // now go through the adapter resources and add them to the list of resources for (adapted_name, adapted_resource) in &self.adapted_resources { @@ -496,3 +507,95 @@ fn load_manifest(path: &Path) -> Result { Ok(resource) } + +fn sort_adapters_based_on_lookup_table(unsorted_adapters: &BTreeMap>, needed_resource_types: &Vec) -> LinkedHashMap> +{ + let mut result = LinkedHashMap::>::new(); + let lookup_table = load_adapted_resources_lookup_table(); + // first add adapters (for needed types) that can be found in the lookup table + for needed_resource in needed_resource_types { + if let Some(adapter_name) = lookup_table.get(needed_resource) { + if let Some(resource_vec) = unsorted_adapters.get(adapter_name) { + debug!("Lookup table found resource '{}' in adapter '{}'", needed_resource, adapter_name); + result.insert(adapter_name.to_string(), resource_vec.clone()); + } + } + } + + // now add remaining adapters + for (adapter_name, adapters) in unsorted_adapters { + if !result.contains_key(adapter_name) { + result.insert(adapter_name.to_string(), adapters.clone()); + } + } + + result +} + +fn add_resources_to_lookup_table(adapted_resources: &BTreeMap>) +{ + let mut lookup_table = load_adapted_resources_lookup_table(); + + for (resource_name, res_vec) in adapted_resources { + if let Some(adapter_name) = &res_vec[0].require_adapter { + lookup_table.insert(resource_name.to_string().to_lowercase(), adapter_name.to_string()); + } else { + info!("Resource '{resource_name}' in 'adapted_resources' is missing 'require_adapter' field."); + } + }; + + save_adapted_resources_lookup_table(&lookup_table); +} + +fn save_adapted_resources_lookup_table(lookup_table: &HashMap) +{ + if let Ok(lookup_table_json) = serde_json::to_string(&lookup_table) { + let file_path = get_lookup_table_file_path(); + debug!("Saving lookup table with {} items to {:?}", lookup_table.len(), file_path); + + let path = std::path::Path::new(&file_path); + if let Some(prefix) = path.parent() { + if fs::create_dir_all(prefix).is_ok() { + if fs::write(file_path.clone(), lookup_table_json).is_err() { + info!("Unable to write lookup_table file {file_path:?}"); + } + } else { + info!("Unable to create parent directories of the lookup_table file {file_path:?}"); + } + } else { + info!("Unable to get directory of the lookup_table file {file_path:?}"); + } + } else { + info!("Unable to serialize lookup_table to json"); + } +} + +fn load_adapted_resources_lookup_table() -> HashMap +{ + let file_path = get_lookup_table_file_path(); + + let lookup_table: HashMap = match fs::read(file_path.clone()){ + Ok(data) => { serde_json::from_slice(&data).unwrap_or_default() }, + Err(_) => { HashMap::new() } + }; + + debug!("Read {} items into lookup table from {:?}", lookup_table.len(), file_path); + lookup_table +} + +#[cfg(target_os = "windows")] +fn get_lookup_table_file_path() -> String +{ + // $env:LocalAppData+"dsc\AdaptedResourcesLookupTable.json" + let Ok(local_app_data_path) = std::env::var("LocalAppData") else { return String::new(); }; + + Path::new(&local_app_data_path).join("dsc").join("AdaptedResourcesLookupTable.json").display().to_string() +} + +#[cfg(not(target_os = "windows"))] +fn get_lookup_table_file_path() -> String +{ + // $env:HOME+".dsc/AdaptedResourcesLookupTable.json" + let Ok(home_path) = std::env::var("HOME") else { return String::new(); }; + Path::new(&home_path).join(".dsc").join("AdaptedResourcesLookupTable.json").display().to_string() +} diff --git a/powershell-adapter/Tests/powershellgroup.resource.tests.ps1 b/powershell-adapter/Tests/powershellgroup.resource.tests.ps1 index 4e07f0390..a9052ef7b 100644 --- a/powershell-adapter/Tests/powershellgroup.resource.tests.ps1 +++ b/powershell-adapter/Tests/powershellgroup.resource.tests.ps1 @@ -270,9 +270,10 @@ Describe 'PowerShell adapter resource tests' { } } - It 'Dsc can process large resource output' -Tag z1{ + It 'Dsc can process large resource output' { $env:TestClassResourceResultCount = 5000 # with sync resource invocations this was not possible + dsc resource list -a Microsoft.DSC/PowerShell | Out-Null $r = dsc resource export -r TestClassResource/TestClassResource $LASTEXITCODE | Should -Be 0 $res = $r | ConvertFrom-Json