From 3493ca536638b0eb0a57728b83584363fa838531 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 21 Aug 2024 18:01:38 -0700 Subject: [PATCH 01/30] Stage 1 --- dsc_lib/src/discovery/command_discovery.rs | 41 +++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index c3410dc9a..6a313b704 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -10,9 +10,10 @@ use crate::dscerror::DscError; use indicatif::ProgressStyle; use regex::RegexBuilder; use semver::Version; -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, HashSet, HashMap}; use std::env; use std::ffi::OsStr; +use std::fs; use std::fs::File; use std::io::BufReader; use std::path::{Path, PathBuf}; @@ -279,6 +280,8 @@ impl ResourceDiscovery for CommandDiscovery { } self.adapted_resources = adapted_resources; + save_adapted_resources_lookup_table(&self.adapted_resources); + Ok(()) } @@ -496,3 +499,39 @@ fn load_manifest(path: &Path) -> Result { Ok(resource) } + +fn save_adapted_resources_lookup_table(adapted_resources: &BTreeMap>) +{ + let mut lookup_table: HashMap = HashMap::new(); + for (resource_name, res_vec) in adapted_resources { + let adapter_name = &res_vec[0].require_adapter.as_ref().unwrap(); + lookup_table.insert(resource_name.to_string(), adapter_name.to_string()); + } + + debug!("============ABC=========="); + + match serde_json::to_string_pretty(&lookup_table) { + Ok(lookup_table_json) => { + debug!("{:?}", lookup_table_json); + + let file_path = get_lookup_table_file_path(); + debug!("{:?}", file_path); + + fs::write(file_path, lookup_table_json).expect("Unable to write lookup_table file"); + }, + Err(_) => {} + } + +} + +#[cfg(target_os = "windows")] +fn get_lookup_table_file_path() -> String +{ + // $env:LocalAppData+"dsc\AdaptedResourcesLookupTable.json" + let local_app_data_path = match std::env::var("LocalAppData") { + Ok(path) => path, + Err(_) => { return "".to_string(); } + }; + + Path::new(&local_app_data_path).join("dsc").join("AdaptedResourcesLookupTable.json").display().to_string() +} From 057452ec8c4c1fd2066abcfb9a486ae68c8baae6 Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 23 Aug 2024 15:53:23 -0700 Subject: [PATCH 02/30] Stage 2 --- dsc_lib/src/discovery/command_discovery.rs | 41 +++++++++++++++++----- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 6a313b704..807f742d0 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -280,7 +280,6 @@ impl ResourceDiscovery for CommandDiscovery { } self.adapted_resources = adapted_resources; - save_adapted_resources_lookup_table(&self.adapted_resources); Ok(()) } @@ -297,6 +296,11 @@ impl ResourceDiscovery for CommandDiscovery { } else { self.discover_resources("*")?; self.discover_adapted_resources(type_name_filter, adapter_name_filter)?; + + // add found adapted resources to the lookup_table + add_resources_to_lookup_table(&self.adapted_resources); + + // note: in next line 'BTreeMap::append' will leave self.adapted_resources empty resources.append(&mut self.adapted_resources); } @@ -356,6 +360,8 @@ impl ResourceDiscovery for CommandDiscovery { } self.discover_adapted_resources("*", &adapter_name)?; + // add found adapted resources to the lookup_table + add_resources_to_lookup_table(&self.adapted_resources); // now go through the adapter resources and add them to the list of resources for (adapted_name, adapted_resource) in &self.adapted_resources { @@ -500,28 +506,47 @@ fn load_manifest(path: &Path) -> Result { Ok(resource) } -fn save_adapted_resources_lookup_table(adapted_resources: &BTreeMap>) +fn add_resources_to_lookup_table(adapted_resources: &BTreeMap>) { - let mut lookup_table: HashMap = HashMap::new(); + let mut lookup_table:HashMap = load_adapted_resources_lookup_table(); + for (resource_name, res_vec) in adapted_resources { let adapter_name = &res_vec[0].require_adapter.as_ref().unwrap(); - lookup_table.insert(resource_name.to_string(), adapter_name.to_string()); - } + lookup_table.insert(resource_name.to_string().to_lowercase(), adapter_name.to_string()); + }; - debug!("============ABC=========="); + save_adapted_resources_lookup_table(&lookup_table); +} +fn save_adapted_resources_lookup_table(lookup_table: &HashMap) +{ match serde_json::to_string_pretty(&lookup_table) { Ok(lookup_table_json) => { - debug!("{:?}", lookup_table_json); let file_path = get_lookup_table_file_path(); - debug!("{:?}", file_path); + debug!("Saving lookup table with {} items to {:?}", lookup_table.len(), file_path); fs::write(file_path, lookup_table_json).expect("Unable to write lookup_table file"); }, Err(_) => {} } +} + +fn load_adapted_resources_lookup_table() -> HashMap +{ + let file_path = get_lookup_table_file_path(); + let lookup_table: HashMap = match fs::read(file_path.clone()){ + Ok(data) => { match serde_json::from_slice(&data) { + Ok(lt) => { lt }, + Err(_) => { HashMap::new() } + } + }, + Err(_) => { HashMap::new() } + }; + + debug!("Read {} items into lookup table from {:?}", lookup_table.len(), file_path); + lookup_table } #[cfg(target_os = "windows")] From b091bbfd0f7f420a59f539d7cfb9060828221482 Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 23 Aug 2024 23:26:36 -0700 Subject: [PATCH 03/30] Stage 3 --- dsc_lib/Cargo.toml | 1 + dsc_lib/src/discovery/command_discovery.rs | 52 +++++++++++++++++++--- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/dsc_lib/Cargo.toml b/dsc_lib/Cargo.toml index 1cf0610af..a61426a58 100644 --- a/dsc_lib/Cargo.toml +++ b/dsc_lib/Cargo.toml @@ -9,6 +9,7 @@ chrono = "0.4.26" derive_builder ="0.20.0" indicatif = "0.17.0" jsonschema = "0.18.0" +linked-hash-map = "0.5.6" num-traits = "0.2.14" regex = "1.7.0" reqwest = { version = "0.12.0", features = ["rustls-tls"], default-features = false } diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 807f742d0..d69c2c626 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -8,6 +8,7 @@ use crate::dscresources::resource_manifest::{import_manifest, validate_semver, K use crate::dscresources::command_resource::invoke_command; use crate::dscerror::DscError; use indicatif::ProgressStyle; +use linked_hash_map::LinkedHashMap; use regex::RegexBuilder; use semver::Version; use std::collections::{BTreeMap, HashSet, HashMap}; @@ -297,7 +298,7 @@ impl ResourceDiscovery for CommandDiscovery { self.discover_resources("*")?; self.discover_adapted_resources(type_name_filter, adapter_name_filter)?; - // add found adapted resources to the lookup_table + // add/update found adapted resources to the lookup_table add_resources_to_lookup_table(&self.adapted_resources); // note: in next line 'BTreeMap::append' will leave self.adapted_resources empty @@ -341,7 +342,8 @@ impl ResourceDiscovery for CommandDiscovery { debug!("Found {} matching non-adapter-based resources", found_resources.len()); // now go through the adapters - for (adapter_name, adapters) in self.adapters.clone() { + let sorted_adapters = sort_adapters_based_on_lookup_table(&self.adapters, &remaining_required_resource_types); + for (adapter_name, adapters) in sorted_adapters { // TODO: handle version requirements let Some(adapter) = adapters.first() else { // skip if no adapters @@ -360,7 +362,7 @@ impl ResourceDiscovery for CommandDiscovery { } self.discover_adapted_resources("*", &adapter_name)?; - // add found adapted resources to the lookup_table + // add/update found adapted resources to the lookup_table add_resources_to_lookup_table(&self.adapted_resources); // now go through the adapter resources and add them to the list of resources @@ -506,6 +508,36 @@ fn load_manifest(path: &Path) -> Result { Ok(resource) } +fn sort_adapters_based_on_lookup_table(unsorted_adapters: &BTreeMap>, needed_resource_types: &Vec) -> LinkedHashMap> +{ + let mut result:LinkedHashMap> = LinkedHashMap::new(); + let lookup_table:HashMap = load_adapted_resources_lookup_table(); + // first add adapters (for needed types) that can be found in the lookup table + for needed_resource in needed_resource_types { + match lookup_table.get(needed_resource) { + Some(adapter_name) => { + match unsorted_adapters.get(adapter_name) { + Some(resource_vec) => { + trace!("Lookup table found resource '{}' in adapter '{}'", needed_resource, adapter_name); + result.insert(adapter_name.to_string(), resource_vec.to_vec()); + } + None => {} + } + } + None => {} + } + } + + // now add remaining adapters + for (adapter_name, adapters) in unsorted_adapters { + if !result.contains_key(adapter_name) { + result.insert(adapter_name.to_string(), adapters.to_vec()); + } + } + + result +} + fn add_resources_to_lookup_table(adapted_resources: &BTreeMap>) { let mut lookup_table:HashMap = load_adapted_resources_lookup_table(); @@ -522,10 +554,8 @@ fn save_adapted_resources_lookup_table(lookup_table: &HashMap) { match serde_json::to_string_pretty(&lookup_table) { Ok(lookup_table_json) => { - let file_path = get_lookup_table_file_path(); debug!("Saving lookup table with {} items to {:?}", lookup_table.len(), file_path); - fs::write(file_path, lookup_table_json).expect("Unable to write lookup_table file"); }, Err(_) => {} @@ -560,3 +590,15 @@ fn get_lookup_table_file_path() -> String Path::new(&local_app_data_path).join("dsc").join("AdaptedResourcesLookupTable.json").display().to_string() } + +#[cfg(not(target_os = "windows"))] +fn get_lookup_table_file_path() -> String +{ + // $env:HOME+".dsc/AdaptedResourcesLookupTable.json" + let home_path = match std::env::var("HOME") { + Ok(path) => path, + Err(_) => { return "".to_string(); } + }; + + Path::new(&home_path).join(".dsc").join("AdaptedResourcesLookupTable.json").display().to_string() +} From 6e1339b8148a0a9b02f151c790710e29a34c295d Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 23 Aug 2024 23:38:18 -0700 Subject: [PATCH 04/30] Revert "Blocked non-list operations on adapters" This reverts commit 6505b11122c523d1496cb70fd98b56f478c64be7. --- dsc/src/resource_command.rs | 34 +--------------------------------- dsc/tests/dsc_args.tests.ps1 | 36 ------------------------------------ 2 files changed, 1 insertion(+), 69 deletions(-) diff --git a/dsc/src/resource_command.rs b/dsc/src/resource_command.rs index b0c4eba86..41c1890df 100644 --- a/dsc/src/resource_command.rs +++ b/dsc/src/resource_command.rs @@ -5,7 +5,7 @@ use crate::args::OutputFormat; use crate::util::{EXIT_DSC_ERROR, EXIT_INVALID_ARGS, EXIT_JSON_ERROR, add_type_name_to_json, write_output}; use dsc_lib::configure::config_doc::{Configuration, ExecutionKind}; use dsc_lib::configure::add_resource_export_results_to_configuration; -use dsc_lib::dscresources::{resource_manifest::Kind, invoke_result::{GetResult, ResourceGetResponse}}; +use dsc_lib::dscresources::invoke_result::{GetResult, ResourceGetResponse}; use dsc_lib::dscerror::DscError; use tracing::{error, debug}; @@ -22,11 +22,6 @@ pub fn get(dsc: &DscManager, resource_type: &str, mut input: String, format: &Op }; debug!("resource.type_name - {} implemented_as - {:?}", resource.type_name, resource.implemented_as); - if resource.kind == Kind::Adapter { - error!("Can not perform this operation on the adapter {} itself", resource.type_name); - exit(EXIT_DSC_ERROR); - } - if let Some(requires) = &resource.require_adapter { input = add_type_name_to_json(input, resource.type_name.clone()); if let Some(pr) = get_resource(dsc, requires) { @@ -64,11 +59,6 @@ pub fn get_all(dsc: &DscManager, resource_type: &str, format: &Option { // verify is json @@ -263,11 +236,6 @@ pub fn export(dsc: &mut DscManager, resource_type: &str, format: &Option = None; if let Some(requires) = &dsc_resource.require_adapter { input = add_type_name_to_json(input, dsc_resource.type_name.clone()); diff --git a/dsc/tests/dsc_args.tests.ps1 b/dsc/tests/dsc_args.tests.ps1 index d1158aa9d..e4ed2c510 100644 --- a/dsc/tests/dsc_args.tests.ps1 +++ b/dsc/tests/dsc_args.tests.ps1 @@ -272,40 +272,4 @@ resources: $stderr = dsc config get -d $configFile 2>&1 $stderr | Should -Match '.*?--path.*?' } - - It 'Get operation on the adapter itself should fail' -Tag 'z1' { - dsc resource get -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt - $LASTEXITCODE | Should -Be 2 - "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' - } - - It 'Get-all operation on the adapter itself should fail' -Tag 'z1' { - dsc resource get --all -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt - $LASTEXITCODE | Should -Be 2 - "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' - } - - It 'Set operation on the adapter itself should fail' -Tag 'z1' { - 'abc' | dsc resource set -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt - $LASTEXITCODE | Should -Be 2 - "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' - } - - It 'Test operation on the adapter itself should fail' -Tag 'z1' { - dsc resource test -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt - $LASTEXITCODE | Should -Be 2 - "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' - } - - It 'Export operation on the adapter itself should fail' -Tag 'z1' { - dsc resource export -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt - $LASTEXITCODE | Should -Be 2 - "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' - } - - It 'Delete operation on the adapter itself should fail' -Tag 'z1' { - dsc resource delete -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt - $LASTEXITCODE | Should -Be 2 - "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' - } } From 86c55a09f4252ecfbead02126507bd7614a89926 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 26 Aug 2024 23:23:09 -0700 Subject: [PATCH 05/30] Stage 4 --- dsc_lib/src/discovery/command_discovery.rs | 35 +++++++--------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index d69c2c626..b275661a4 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -514,24 +514,18 @@ fn sort_adapters_based_on_lookup_table(unsorted_adapters: &BTreeMap = load_adapted_resources_lookup_table(); // first add adapters (for needed types) that can be found in the lookup table for needed_resource in needed_resource_types { - match lookup_table.get(needed_resource) { - Some(adapter_name) => { - match unsorted_adapters.get(adapter_name) { - Some(resource_vec) => { - trace!("Lookup table found resource '{}' in adapter '{}'", needed_resource, adapter_name); - result.insert(adapter_name.to_string(), resource_vec.to_vec()); - } - None => {} - } + if let Some(adapter_name) = lookup_table.get(needed_resource) { + if let Some(resource_vec) = unsorted_adapters.get(adapter_name) { + trace!("Lookup table found resource '{}' in adapter '{}'", needed_resource, adapter_name); + result.insert(adapter_name.to_string(), resource_vec.clone()); } - None => {} } } // now add remaining adapters for (adapter_name, adapters) in unsorted_adapters { if !result.contains_key(adapter_name) { - result.insert(adapter_name.to_string(), adapters.to_vec()); + result.insert(adapter_name.to_string(), adapters.clone()); } } @@ -544,7 +538,7 @@ fn add_resources_to_lookup_table(adapted_resources: &BTreeMap) { - match serde_json::to_string_pretty(&lookup_table) { - Ok(lookup_table_json) => { - let file_path = get_lookup_table_file_path(); - debug!("Saving lookup table with {} items to {:?}", lookup_table.len(), file_path); - fs::write(file_path, lookup_table_json).expect("Unable to write lookup_table file"); - }, - Err(_) => {} + if let Ok(lookup_table_json) = serde_json::to_string_pretty(&lookup_table) { + let file_path = get_lookup_table_file_path(); + debug!("Saving lookup table with {} items to {:?}", lookup_table.len(), file_path); + fs::write(file_path, lookup_table_json).expect("Unable to write lookup_table file"); } } @@ -595,10 +586,6 @@ fn get_lookup_table_file_path() -> String fn get_lookup_table_file_path() -> String { // $env:HOME+".dsc/AdaptedResourcesLookupTable.json" - let home_path = match std::env::var("HOME") { - Ok(path) => path, - Err(_) => { return "".to_string(); } - }; - + let Ok(home_path) = std::env::var("HOME") else { return String::new(); }; Path::new(&home_path).join(".dsc").join("AdaptedResourcesLookupTable.json").display().to_string() } From aea50e202f071438d896a627b5bbdca1ebf5179b Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 26 Aug 2024 23:28:52 -0700 Subject: [PATCH 06/30] Stage 5 --- dsc_lib/src/discovery/command_discovery.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index b275661a4..49b263fea 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -574,10 +574,7 @@ fn load_adapted_resources_lookup_table() -> HashMap fn get_lookup_table_file_path() -> String { // $env:LocalAppData+"dsc\AdaptedResourcesLookupTable.json" - let local_app_data_path = match std::env::var("LocalAppData") { - Ok(path) => path, - Err(_) => { return "".to_string(); } - }; + let Ok(local_app_data_path) = std::env::var("LocalAppData") else { return String::new(); }; Path::new(&local_app_data_path).join("dsc").join("AdaptedResourcesLookupTable.json").display().to_string() } From 5004d371ef39ed13da49f69e1ff64145e90ed178 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 21 Aug 2024 14:02:20 -0700 Subject: [PATCH 07/30] Blocked non-list operations on adapters --- dsc/src/resource_command.rs | 34 +++++++++++++++++++++++++++++++++- dsc/tests/dsc_args.tests.ps1 | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/dsc/src/resource_command.rs b/dsc/src/resource_command.rs index 41c1890df..b0c4eba86 100644 --- a/dsc/src/resource_command.rs +++ b/dsc/src/resource_command.rs @@ -5,7 +5,7 @@ use crate::args::OutputFormat; use crate::util::{EXIT_DSC_ERROR, EXIT_INVALID_ARGS, EXIT_JSON_ERROR, add_type_name_to_json, write_output}; use dsc_lib::configure::config_doc::{Configuration, ExecutionKind}; use dsc_lib::configure::add_resource_export_results_to_configuration; -use dsc_lib::dscresources::invoke_result::{GetResult, ResourceGetResponse}; +use dsc_lib::dscresources::{resource_manifest::Kind, invoke_result::{GetResult, ResourceGetResponse}}; use dsc_lib::dscerror::DscError; use tracing::{error, debug}; @@ -22,6 +22,11 @@ pub fn get(dsc: &DscManager, resource_type: &str, mut input: String, format: &Op }; debug!("resource.type_name - {} implemented_as - {:?}", resource.type_name, resource.implemented_as); + if resource.kind == Kind::Adapter { + error!("Can not perform this operation on the adapter {} itself", resource.type_name); + exit(EXIT_DSC_ERROR); + } + if let Some(requires) = &resource.require_adapter { input = add_type_name_to_json(input, resource.type_name.clone()); if let Some(pr) = get_resource(dsc, requires) { @@ -59,6 +64,11 @@ pub fn get_all(dsc: &DscManager, resource_type: &str, format: &Option { // verify is json @@ -236,6 +263,11 @@ pub fn export(dsc: &mut DscManager, resource_type: &str, format: &Option = None; if let Some(requires) = &dsc_resource.require_adapter { input = add_type_name_to_json(input, dsc_resource.type_name.clone()); diff --git a/dsc/tests/dsc_args.tests.ps1 b/dsc/tests/dsc_args.tests.ps1 index e4ed2c510..d1158aa9d 100644 --- a/dsc/tests/dsc_args.tests.ps1 +++ b/dsc/tests/dsc_args.tests.ps1 @@ -272,4 +272,40 @@ resources: $stderr = dsc config get -d $configFile 2>&1 $stderr | Should -Match '.*?--path.*?' } + + It 'Get operation on the adapter itself should fail' -Tag 'z1' { + dsc resource get -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt + $LASTEXITCODE | Should -Be 2 + "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' + } + + It 'Get-all operation on the adapter itself should fail' -Tag 'z1' { + dsc resource get --all -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt + $LASTEXITCODE | Should -Be 2 + "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' + } + + It 'Set operation on the adapter itself should fail' -Tag 'z1' { + 'abc' | dsc resource set -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt + $LASTEXITCODE | Should -Be 2 + "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' + } + + It 'Test operation on the adapter itself should fail' -Tag 'z1' { + dsc resource test -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt + $LASTEXITCODE | Should -Be 2 + "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' + } + + It 'Export operation on the adapter itself should fail' -Tag 'z1' { + dsc resource export -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt + $LASTEXITCODE | Should -Be 2 + "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' + } + + It 'Delete operation on the adapter itself should fail' -Tag 'z1' { + dsc resource delete -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt + $LASTEXITCODE | Should -Be 2 + "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' + } } From e8e48d158fabc1f252294b32e2d7f75540447fdf Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 21 Aug 2024 18:01:38 -0700 Subject: [PATCH 08/30] Stage 1 --- dsc_lib/src/discovery/command_discovery.rs | 41 +++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index c3410dc9a..6a313b704 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -10,9 +10,10 @@ use crate::dscerror::DscError; use indicatif::ProgressStyle; use regex::RegexBuilder; use semver::Version; -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, HashSet, HashMap}; use std::env; use std::ffi::OsStr; +use std::fs; use std::fs::File; use std::io::BufReader; use std::path::{Path, PathBuf}; @@ -279,6 +280,8 @@ impl ResourceDiscovery for CommandDiscovery { } self.adapted_resources = adapted_resources; + save_adapted_resources_lookup_table(&self.adapted_resources); + Ok(()) } @@ -496,3 +499,39 @@ fn load_manifest(path: &Path) -> Result { Ok(resource) } + +fn save_adapted_resources_lookup_table(adapted_resources: &BTreeMap>) +{ + let mut lookup_table: HashMap = HashMap::new(); + for (resource_name, res_vec) in adapted_resources { + let adapter_name = &res_vec[0].require_adapter.as_ref().unwrap(); + lookup_table.insert(resource_name.to_string(), adapter_name.to_string()); + } + + debug!("============ABC=========="); + + match serde_json::to_string_pretty(&lookup_table) { + Ok(lookup_table_json) => { + debug!("{:?}", lookup_table_json); + + let file_path = get_lookup_table_file_path(); + debug!("{:?}", file_path); + + fs::write(file_path, lookup_table_json).expect("Unable to write lookup_table file"); + }, + Err(_) => {} + } + +} + +#[cfg(target_os = "windows")] +fn get_lookup_table_file_path() -> String +{ + // $env:LocalAppData+"dsc\AdaptedResourcesLookupTable.json" + let local_app_data_path = match std::env::var("LocalAppData") { + Ok(path) => path, + Err(_) => { return "".to_string(); } + }; + + Path::new(&local_app_data_path).join("dsc").join("AdaptedResourcesLookupTable.json").display().to_string() +} From 2efd5d110d0ca88b46a3485adcd03f8a7aa1e500 Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 23 Aug 2024 15:53:23 -0700 Subject: [PATCH 09/30] Stage 2 --- dsc_lib/src/discovery/command_discovery.rs | 41 +++++++++++++++++----- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 6a313b704..807f742d0 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -280,7 +280,6 @@ impl ResourceDiscovery for CommandDiscovery { } self.adapted_resources = adapted_resources; - save_adapted_resources_lookup_table(&self.adapted_resources); Ok(()) } @@ -297,6 +296,11 @@ impl ResourceDiscovery for CommandDiscovery { } else { self.discover_resources("*")?; self.discover_adapted_resources(type_name_filter, adapter_name_filter)?; + + // add found adapted resources to the lookup_table + add_resources_to_lookup_table(&self.adapted_resources); + + // note: in next line 'BTreeMap::append' will leave self.adapted_resources empty resources.append(&mut self.adapted_resources); } @@ -356,6 +360,8 @@ impl ResourceDiscovery for CommandDiscovery { } self.discover_adapted_resources("*", &adapter_name)?; + // add found adapted resources to the lookup_table + add_resources_to_lookup_table(&self.adapted_resources); // now go through the adapter resources and add them to the list of resources for (adapted_name, adapted_resource) in &self.adapted_resources { @@ -500,28 +506,47 @@ fn load_manifest(path: &Path) -> Result { Ok(resource) } -fn save_adapted_resources_lookup_table(adapted_resources: &BTreeMap>) +fn add_resources_to_lookup_table(adapted_resources: &BTreeMap>) { - let mut lookup_table: HashMap = HashMap::new(); + let mut lookup_table:HashMap = load_adapted_resources_lookup_table(); + for (resource_name, res_vec) in adapted_resources { let adapter_name = &res_vec[0].require_adapter.as_ref().unwrap(); - lookup_table.insert(resource_name.to_string(), adapter_name.to_string()); - } + lookup_table.insert(resource_name.to_string().to_lowercase(), adapter_name.to_string()); + }; - debug!("============ABC=========="); + save_adapted_resources_lookup_table(&lookup_table); +} +fn save_adapted_resources_lookup_table(lookup_table: &HashMap) +{ match serde_json::to_string_pretty(&lookup_table) { Ok(lookup_table_json) => { - debug!("{:?}", lookup_table_json); let file_path = get_lookup_table_file_path(); - debug!("{:?}", file_path); + debug!("Saving lookup table with {} items to {:?}", lookup_table.len(), file_path); fs::write(file_path, lookup_table_json).expect("Unable to write lookup_table file"); }, Err(_) => {} } +} + +fn load_adapted_resources_lookup_table() -> HashMap +{ + let file_path = get_lookup_table_file_path(); + let lookup_table: HashMap = match fs::read(file_path.clone()){ + Ok(data) => { match serde_json::from_slice(&data) { + Ok(lt) => { lt }, + Err(_) => { HashMap::new() } + } + }, + Err(_) => { HashMap::new() } + }; + + debug!("Read {} items into lookup table from {:?}", lookup_table.len(), file_path); + lookup_table } #[cfg(target_os = "windows")] From 78a3c355d59300b476fa01f921678f932c6c221e Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 23 Aug 2024 23:26:36 -0700 Subject: [PATCH 10/30] Stage 3 --- dsc_lib/Cargo.toml | 1 + dsc_lib/src/discovery/command_discovery.rs | 52 +++++++++++++++++++--- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/dsc_lib/Cargo.toml b/dsc_lib/Cargo.toml index 1cf0610af..a61426a58 100644 --- a/dsc_lib/Cargo.toml +++ b/dsc_lib/Cargo.toml @@ -9,6 +9,7 @@ chrono = "0.4.26" derive_builder ="0.20.0" indicatif = "0.17.0" jsonschema = "0.18.0" +linked-hash-map = "0.5.6" num-traits = "0.2.14" regex = "1.7.0" reqwest = { version = "0.12.0", features = ["rustls-tls"], default-features = false } diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 807f742d0..d69c2c626 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -8,6 +8,7 @@ use crate::dscresources::resource_manifest::{import_manifest, validate_semver, K use crate::dscresources::command_resource::invoke_command; use crate::dscerror::DscError; use indicatif::ProgressStyle; +use linked_hash_map::LinkedHashMap; use regex::RegexBuilder; use semver::Version; use std::collections::{BTreeMap, HashSet, HashMap}; @@ -297,7 +298,7 @@ impl ResourceDiscovery for CommandDiscovery { self.discover_resources("*")?; self.discover_adapted_resources(type_name_filter, adapter_name_filter)?; - // add found adapted resources to the lookup_table + // add/update found adapted resources to the lookup_table add_resources_to_lookup_table(&self.adapted_resources); // note: in next line 'BTreeMap::append' will leave self.adapted_resources empty @@ -341,7 +342,8 @@ impl ResourceDiscovery for CommandDiscovery { debug!("Found {} matching non-adapter-based resources", found_resources.len()); // now go through the adapters - for (adapter_name, adapters) in self.adapters.clone() { + let sorted_adapters = sort_adapters_based_on_lookup_table(&self.adapters, &remaining_required_resource_types); + for (adapter_name, adapters) in sorted_adapters { // TODO: handle version requirements let Some(adapter) = adapters.first() else { // skip if no adapters @@ -360,7 +362,7 @@ impl ResourceDiscovery for CommandDiscovery { } self.discover_adapted_resources("*", &adapter_name)?; - // add found adapted resources to the lookup_table + // add/update found adapted resources to the lookup_table add_resources_to_lookup_table(&self.adapted_resources); // now go through the adapter resources and add them to the list of resources @@ -506,6 +508,36 @@ fn load_manifest(path: &Path) -> Result { Ok(resource) } +fn sort_adapters_based_on_lookup_table(unsorted_adapters: &BTreeMap>, needed_resource_types: &Vec) -> LinkedHashMap> +{ + let mut result:LinkedHashMap> = LinkedHashMap::new(); + let lookup_table:HashMap = load_adapted_resources_lookup_table(); + // first add adapters (for needed types) that can be found in the lookup table + for needed_resource in needed_resource_types { + match lookup_table.get(needed_resource) { + Some(adapter_name) => { + match unsorted_adapters.get(adapter_name) { + Some(resource_vec) => { + trace!("Lookup table found resource '{}' in adapter '{}'", needed_resource, adapter_name); + result.insert(adapter_name.to_string(), resource_vec.to_vec()); + } + None => {} + } + } + None => {} + } + } + + // now add remaining adapters + for (adapter_name, adapters) in unsorted_adapters { + if !result.contains_key(adapter_name) { + result.insert(adapter_name.to_string(), adapters.to_vec()); + } + } + + result +} + fn add_resources_to_lookup_table(adapted_resources: &BTreeMap>) { let mut lookup_table:HashMap = load_adapted_resources_lookup_table(); @@ -522,10 +554,8 @@ fn save_adapted_resources_lookup_table(lookup_table: &HashMap) { match serde_json::to_string_pretty(&lookup_table) { Ok(lookup_table_json) => { - let file_path = get_lookup_table_file_path(); debug!("Saving lookup table with {} items to {:?}", lookup_table.len(), file_path); - fs::write(file_path, lookup_table_json).expect("Unable to write lookup_table file"); }, Err(_) => {} @@ -560,3 +590,15 @@ fn get_lookup_table_file_path() -> String Path::new(&local_app_data_path).join("dsc").join("AdaptedResourcesLookupTable.json").display().to_string() } + +#[cfg(not(target_os = "windows"))] +fn get_lookup_table_file_path() -> String +{ + // $env:HOME+".dsc/AdaptedResourcesLookupTable.json" + let home_path = match std::env::var("HOME") { + Ok(path) => path, + Err(_) => { return "".to_string(); } + }; + + Path::new(&home_path).join(".dsc").join("AdaptedResourcesLookupTable.json").display().to_string() +} From 22804261df48c0134ecfde0b9c933ddea0b09d0b Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 23 Aug 2024 23:38:18 -0700 Subject: [PATCH 11/30] Revert "Blocked non-list operations on adapters" This reverts commit 6505b11122c523d1496cb70fd98b56f478c64be7. --- dsc/src/resource_command.rs | 34 +--------------------------------- dsc/tests/dsc_args.tests.ps1 | 36 ------------------------------------ 2 files changed, 1 insertion(+), 69 deletions(-) diff --git a/dsc/src/resource_command.rs b/dsc/src/resource_command.rs index b0c4eba86..41c1890df 100644 --- a/dsc/src/resource_command.rs +++ b/dsc/src/resource_command.rs @@ -5,7 +5,7 @@ use crate::args::OutputFormat; use crate::util::{EXIT_DSC_ERROR, EXIT_INVALID_ARGS, EXIT_JSON_ERROR, add_type_name_to_json, write_output}; use dsc_lib::configure::config_doc::{Configuration, ExecutionKind}; use dsc_lib::configure::add_resource_export_results_to_configuration; -use dsc_lib::dscresources::{resource_manifest::Kind, invoke_result::{GetResult, ResourceGetResponse}}; +use dsc_lib::dscresources::invoke_result::{GetResult, ResourceGetResponse}; use dsc_lib::dscerror::DscError; use tracing::{error, debug}; @@ -22,11 +22,6 @@ pub fn get(dsc: &DscManager, resource_type: &str, mut input: String, format: &Op }; debug!("resource.type_name - {} implemented_as - {:?}", resource.type_name, resource.implemented_as); - if resource.kind == Kind::Adapter { - error!("Can not perform this operation on the adapter {} itself", resource.type_name); - exit(EXIT_DSC_ERROR); - } - if let Some(requires) = &resource.require_adapter { input = add_type_name_to_json(input, resource.type_name.clone()); if let Some(pr) = get_resource(dsc, requires) { @@ -64,11 +59,6 @@ pub fn get_all(dsc: &DscManager, resource_type: &str, format: &Option { // verify is json @@ -263,11 +236,6 @@ pub fn export(dsc: &mut DscManager, resource_type: &str, format: &Option = None; if let Some(requires) = &dsc_resource.require_adapter { input = add_type_name_to_json(input, dsc_resource.type_name.clone()); diff --git a/dsc/tests/dsc_args.tests.ps1 b/dsc/tests/dsc_args.tests.ps1 index d1158aa9d..e4ed2c510 100644 --- a/dsc/tests/dsc_args.tests.ps1 +++ b/dsc/tests/dsc_args.tests.ps1 @@ -272,40 +272,4 @@ resources: $stderr = dsc config get -d $configFile 2>&1 $stderr | Should -Match '.*?--path.*?' } - - It 'Get operation on the adapter itself should fail' -Tag 'z1' { - dsc resource get -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt - $LASTEXITCODE | Should -Be 2 - "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' - } - - It 'Get-all operation on the adapter itself should fail' -Tag 'z1' { - dsc resource get --all -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt - $LASTEXITCODE | Should -Be 2 - "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' - } - - It 'Set operation on the adapter itself should fail' -Tag 'z1' { - 'abc' | dsc resource set -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt - $LASTEXITCODE | Should -Be 2 - "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' - } - - It 'Test operation on the adapter itself should fail' -Tag 'z1' { - dsc resource test -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt - $LASTEXITCODE | Should -Be 2 - "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' - } - - It 'Export operation on the adapter itself should fail' -Tag 'z1' { - dsc resource export -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt - $LASTEXITCODE | Should -Be 2 - "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' - } - - It 'Delete operation on the adapter itself should fail' -Tag 'z1' { - dsc resource delete -r Microsoft.DSC/PowerShell 2> $TestDrive/tracing.txt - $LASTEXITCODE | Should -Be 2 - "$TestDrive/tracing.txt" | Should -FileContentMatchExactly 'Can not perform this operation on the adapter' - } } From 1e1daecd64975539bb49580d7f5213a45686395c Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 26 Aug 2024 23:23:09 -0700 Subject: [PATCH 12/30] Stage 4 --- dsc_lib/src/discovery/command_discovery.rs | 35 +++++++--------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index d69c2c626..b275661a4 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -514,24 +514,18 @@ fn sort_adapters_based_on_lookup_table(unsorted_adapters: &BTreeMap = load_adapted_resources_lookup_table(); // first add adapters (for needed types) that can be found in the lookup table for needed_resource in needed_resource_types { - match lookup_table.get(needed_resource) { - Some(adapter_name) => { - match unsorted_adapters.get(adapter_name) { - Some(resource_vec) => { - trace!("Lookup table found resource '{}' in adapter '{}'", needed_resource, adapter_name); - result.insert(adapter_name.to_string(), resource_vec.to_vec()); - } - None => {} - } + if let Some(adapter_name) = lookup_table.get(needed_resource) { + if let Some(resource_vec) = unsorted_adapters.get(adapter_name) { + trace!("Lookup table found resource '{}' in adapter '{}'", needed_resource, adapter_name); + result.insert(adapter_name.to_string(), resource_vec.clone()); } - None => {} } } // now add remaining adapters for (adapter_name, adapters) in unsorted_adapters { if !result.contains_key(adapter_name) { - result.insert(adapter_name.to_string(), adapters.to_vec()); + result.insert(adapter_name.to_string(), adapters.clone()); } } @@ -544,7 +538,7 @@ fn add_resources_to_lookup_table(adapted_resources: &BTreeMap) { - match serde_json::to_string_pretty(&lookup_table) { - Ok(lookup_table_json) => { - let file_path = get_lookup_table_file_path(); - debug!("Saving lookup table with {} items to {:?}", lookup_table.len(), file_path); - fs::write(file_path, lookup_table_json).expect("Unable to write lookup_table file"); - }, - Err(_) => {} + if let Ok(lookup_table_json) = serde_json::to_string_pretty(&lookup_table) { + let file_path = get_lookup_table_file_path(); + debug!("Saving lookup table with {} items to {:?}", lookup_table.len(), file_path); + fs::write(file_path, lookup_table_json).expect("Unable to write lookup_table file"); } } @@ -595,10 +586,6 @@ fn get_lookup_table_file_path() -> String fn get_lookup_table_file_path() -> String { // $env:HOME+".dsc/AdaptedResourcesLookupTable.json" - let home_path = match std::env::var("HOME") { - Ok(path) => path, - Err(_) => { return "".to_string(); } - }; - + let Ok(home_path) = std::env::var("HOME") else { return String::new(); }; Path::new(&home_path).join(".dsc").join("AdaptedResourcesLookupTable.json").display().to_string() } From 2872f0385675c3468194ae54e9ba106030ee77aa Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 26 Aug 2024 23:28:52 -0700 Subject: [PATCH 13/30] Stage 5 --- dsc_lib/src/discovery/command_discovery.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index b275661a4..49b263fea 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -574,10 +574,7 @@ fn load_adapted_resources_lookup_table() -> HashMap fn get_lookup_table_file_path() -> String { // $env:LocalAppData+"dsc\AdaptedResourcesLookupTable.json" - let local_app_data_path = match std::env::var("LocalAppData") { - Ok(path) => path, - Err(_) => { return "".to_string(); } - }; + let Ok(local_app_data_path) = std::env::var("LocalAppData") else { return String::new(); }; Path::new(&local_app_data_path).join("dsc").join("AdaptedResourcesLookupTable.json").display().to_string() } From 9cff4708dc2a2e31151792e98ea4ffc99d75ade0 Mon Sep 17 00:00:00 2001 From: Andrew Date: Tue, 27 Aug 2024 14:28:24 -0700 Subject: [PATCH 14/30] Stage 6 --- dsc_lib/src/discovery/command_discovery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 49b263fea..809365fef 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -549,7 +549,7 @@ fn save_adapted_resources_lookup_table(lookup_table: &HashMap) if let Ok(lookup_table_json) = serde_json::to_string_pretty(&lookup_table) { let file_path = get_lookup_table_file_path(); debug!("Saving lookup table with {} items to {:?}", lookup_table.len(), file_path); - fs::write(file_path, lookup_table_json).expect("Unable to write lookup_table file"); + fs::write(file_path.clone(), lookup_table_json).expect(&format!("Unable to write lookup_table file {:?}", file_path).to_string()); } } From d828c516fd5af03c9a9dc57c66b683d857ca78f5 Mon Sep 17 00:00:00 2001 From: Andrew Date: Tue, 27 Aug 2024 14:32:22 -0700 Subject: [PATCH 15/30] Stage 7 --- dsc_lib/src/discovery/command_discovery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 809365fef..a8c93d73a 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -549,7 +549,7 @@ fn save_adapted_resources_lookup_table(lookup_table: &HashMap) if let Ok(lookup_table_json) = serde_json::to_string_pretty(&lookup_table) { let file_path = get_lookup_table_file_path(); debug!("Saving lookup table with {} items to {:?}", lookup_table.len(), file_path); - fs::write(file_path.clone(), lookup_table_json).expect(&format!("Unable to write lookup_table file {:?}", file_path).to_string()); + fs::write(file_path.clone(), lookup_table_json)unwrap_or_else(|_| { panic!("{}", format!("Unable to write lookup_table file {:?}", file_path).to_string()) }); } } From 36b362e286d345d6cdb4c6cdc917fbae38d57d39 Mon Sep 17 00:00:00 2001 From: Andrew Date: Tue, 27 Aug 2024 14:34:00 -0700 Subject: [PATCH 16/30] Stage 8 --- dsc_lib/src/discovery/command_discovery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index a8c93d73a..1361371a3 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -549,7 +549,7 @@ fn save_adapted_resources_lookup_table(lookup_table: &HashMap) if let Ok(lookup_table_json) = serde_json::to_string_pretty(&lookup_table) { let file_path = get_lookup_table_file_path(); debug!("Saving lookup table with {} items to {:?}", lookup_table.len(), file_path); - fs::write(file_path.clone(), lookup_table_json)unwrap_or_else(|_| { panic!("{}", format!("Unable to write lookup_table file {:?}", file_path).to_string()) }); + fs::write(file_path.clone(), lookup_table_json).unwrap_or_else(|_| { panic!("{}", format!("Unable to write lookup_table file {:?}", file_path).to_string()) }); } } From db8daa93885f80591e9d7faf151f534964dc8064 Mon Sep 17 00:00:00 2001 From: Andrew Date: Tue, 27 Aug 2024 14:43:35 -0700 Subject: [PATCH 17/30] Stage 9 --- dsc_lib/src/discovery/command_discovery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 1361371a3..ccd6e0bd3 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -549,7 +549,7 @@ fn save_adapted_resources_lookup_table(lookup_table: &HashMap) if let Ok(lookup_table_json) = serde_json::to_string_pretty(&lookup_table) { let file_path = get_lookup_table_file_path(); debug!("Saving lookup table with {} items to {:?}", lookup_table.len(), file_path); - fs::write(file_path.clone(), lookup_table_json).unwrap_or_else(|_| { panic!("{}", format!("Unable to write lookup_table file {:?}", file_path).to_string()) }); + fs::write(file_path.clone(), lookup_table_json).unwrap_or_else(|_| { panic!("{}", format!("Unable to write lookup_table file {file_path:?}").to_string()) }); } } From 85dbd62ecda06f5447f0f2e1515b1556af1f68d1 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 28 Aug 2024 21:30:24 -0700 Subject: [PATCH 18/30] Updated tests --- powershell-adapter/Tests/powershellgroup.resource.tests.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/powershell-adapter/Tests/powershellgroup.resource.tests.ps1 b/powershell-adapter/Tests/powershellgroup.resource.tests.ps1 index 4d635133b..968fc2e3e 100644 --- a/powershell-adapter/Tests/powershellgroup.resource.tests.ps1 +++ b/powershell-adapter/Tests/powershellgroup.resource.tests.ps1 @@ -270,9 +270,10 @@ Describe 'PowerShell adapter resource tests' { } } - It 'Dsc can process large resource output' -Tag z1{ + It 'Dsc can process large resource output' { $env:TestClassResourceResultCount = 5000 # with sync resource invocations this was not possible + dsc resource list -a Microsoft.DSC/PowerShell | Out-Null $r = dsc resource export -r TestClassResource/TestClassResource $LASTEXITCODE | Should -Be 0 $res = $r | ConvertFrom-Json From a7ae36fcbe082537e34ff83a41288c4b14254c7d Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 28 Aug 2024 21:49:13 -0700 Subject: [PATCH 19/30] create directory before saving lookup_table file --- dsc_lib/src/discovery/command_discovery.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index ccd6e0bd3..275d1bad1 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -549,7 +549,21 @@ fn save_adapted_resources_lookup_table(lookup_table: &HashMap) if let Ok(lookup_table_json) = serde_json::to_string_pretty(&lookup_table) { let file_path = get_lookup_table_file_path(); debug!("Saving lookup table with {} items to {:?}", lookup_table.len(), file_path); - fs::write(file_path.clone(), lookup_table_json).unwrap_or_else(|_| { panic!("{}", format!("Unable to write lookup_table file {file_path:?}").to_string()) }); + + let path = std::path::Path::new(&file_path); + if let Some(prefix) = path.parent() { + if fs::create_dir_all(prefix).is_ok() { + if !fs::write(file_path.clone(), lookup_table_json).is_ok() { + debug!("Unable to write lookup_table file {file_path:?}"); + } + } else { + debug!("Unable to create parent directories of the lookup_table file {file_path:?}"); + } + } else { + debug!("Unable to get directory of the lookup_table file {file_path:?}"); + } + } else { + debug!("Unable to serialize lookup_table to json"); } } From 17c38bae5a95440fdbfb30b2f734db95a3d4b0e0 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 28 Aug 2024 21:51:32 -0700 Subject: [PATCH 20/30] clippy fix --- dsc_lib/src/discovery/command_discovery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 275d1bad1..87faf231b 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -553,7 +553,7 @@ fn save_adapted_resources_lookup_table(lookup_table: &HashMap) let path = std::path::Path::new(&file_path); if let Some(prefix) = path.parent() { if fs::create_dir_all(prefix).is_ok() { - if !fs::write(file_path.clone(), lookup_table_json).is_ok() { + if fs::write(file_path.clone(), lookup_table_json).is_err() { debug!("Unable to write lookup_table file {file_path:?}"); } } else { From 650c34c978b1fe43bd315200d05281a1b86333cf Mon Sep 17 00:00:00 2001 From: Andrew Date: Thu, 29 Aug 2024 22:45:42 -0700 Subject: [PATCH 21/30] added tests for adapter lookup table --- dsc/tests/dsc_discovery.tests.ps1 | 52 +++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/dsc/tests/dsc_discovery.tests.ps1 b/dsc/tests/dsc_discovery.tests.ps1 index e941540f0..052321589 100644 --- a/dsc/tests/dsc_discovery.tests.ps1 +++ b/dsc/tests/dsc_discovery.tests.ps1 @@ -96,4 +96,56 @@ Describe 'tests for resource discovery' { $env:DSC_RESOURCE_PATH = $oldPath } } + + It 'Ensure List operation populates adapter lookup table' { + $lookupTableFilePath = if ($IsWindows) { + Join-Path $env:LocalAppData "dsc\AdaptedResourcesLookupTable.json" + } else { + Join-Path $env:HOME ".dsc" "AdaptedResourcesLookupTable.json" + } + + # remove adapter lookup table file + Remove-Item -Force -Path $lookupTableFilePath -ErrorAction SilentlyContinue + Test-Path $lookupTableFilePath -PathType Leaf | Should -BeFalse + + # perform List on an adapter - this should create adapter lookup table file + dsc resource list -a Microsoft.DSC/PowerShell | Out-Null + + Test-Path $lookupTableFilePath -PathType Leaf | Should -BeTrue + } + + It 'Ensure non-List operation populates adapter lookup table' { + $lookupTableFilePath = if ($IsWindows) { + Join-Path $env:LocalAppData "dsc\AdaptedResourcesLookupTable.json" + } else { + Join-Path $env:HOME ".dsc" "AdaptedResourcesLookupTable.json" + } + + # remove adapter lookup table file + Remove-Item -Force -Path $lookupTableFilePath -ErrorAction SilentlyContinue + Test-Path $lookupTableFilePath -PathType Leaf | Should -BeFalse + + # perform Get on an adapter - this should create adapter lookup table file + $oldPSModulePath = $env:PSModulePath + $TestClassResourcePath = Resolve-Path "$PSScriptRoot/../../powershell-adapter/Tests" + $env:DSC_RESOURCE_PATH = $null + $env:PSModulePath += [System.IO.Path]::PathSeparator + $TestClassResourcePath + "{'Name':'TestClassResource1'}" | dsc resource get -r 'TestClassResource/TestClassResource' | Out-Null + + Test-Path $lookupTableFilePath -PathType Leaf | Should -BeTrue + $env:PSModulePath = $oldPSModulePath + } + + It 'Verify adapter lookup table is used on repeat invocations' { + + $oldPSModulePath = $env:PSModulePath + $TestClassResourcePath = Resolve-Path "$PSScriptRoot/../../powershell-adapter/Tests" + $env:DSC_RESOURCE_PATH = $null + $env:PSModulePath += [System.IO.Path]::PathSeparator + $TestClassResourcePath + dsc resource list -a Microsoft.DSC/PowerShell | Out-Null + "{'Name':'TestClassResource1'}" | dsc -l trace resource get -r 'TestClassResource/TestClassResource' 2> $TestDrive/tracing.txt + + "$TestDrive/tracing.txt" | Should -FileContentMatchExactly "Lookup table found resource 'testclassresource/testclassresource' in adapter 'Microsoft.DSC/PowerShell'" + $env:PSModulePath = $oldPSModulePath + } } From 28dd171f034680de4016c8fa36ce61b20c6acbe4 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 4 Sep 2024 16:04:30 -0700 Subject: [PATCH 22/30] test update 1 --- dsc/tests/dsc_discovery.tests.ps1 | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/dsc/tests/dsc_discovery.tests.ps1 b/dsc/tests/dsc_discovery.tests.ps1 index 052321589..7b3ad399d 100644 --- a/dsc/tests/dsc_discovery.tests.ps1 +++ b/dsc/tests/dsc_discovery.tests.ps1 @@ -4,6 +4,12 @@ Describe 'tests for resource discovery' { BeforeAll { $env:DSC_RESOURCE_PATH = $testdrive + + $script:lookupTableFilePath = if ($IsWindows) { + Join-Path $env:LocalAppData "dsc\AdaptedResourcesLookupTable.json" + } else { + Join-Path $env:HOME ".dsc" "AdaptedResourcesLookupTable.json" + } } AfterEach { @@ -98,32 +104,23 @@ Describe 'tests for resource discovery' { } It 'Ensure List operation populates adapter lookup table' { - $lookupTableFilePath = if ($IsWindows) { - Join-Path $env:LocalAppData "dsc\AdaptedResourcesLookupTable.json" - } else { - Join-Path $env:HOME ".dsc" "AdaptedResourcesLookupTable.json" - } # remove adapter lookup table file - Remove-Item -Force -Path $lookupTableFilePath -ErrorAction SilentlyContinue - Test-Path $lookupTableFilePath -PathType Leaf | Should -BeFalse + Remove-Item -Force -Path $script:lookupTableFilePath -ErrorAction SilentlyContinue + Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeFalse # perform List on an adapter - this should create adapter lookup table file dsc resource list -a Microsoft.DSC/PowerShell | Out-Null - Test-Path $lookupTableFilePath -PathType Leaf | Should -BeTrue + Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeTrue + $script:lookupTableFilePath | Should -FileContentMatchExactly 'Microsoft.DSC/PowerShell' } It 'Ensure non-List operation populates adapter lookup table' { - $lookupTableFilePath = if ($IsWindows) { - Join-Path $env:LocalAppData "dsc\AdaptedResourcesLookupTable.json" - } else { - Join-Path $env:HOME ".dsc" "AdaptedResourcesLookupTable.json" - } # remove adapter lookup table file - Remove-Item -Force -Path $lookupTableFilePath -ErrorAction SilentlyContinue - Test-Path $lookupTableFilePath -PathType Leaf | Should -BeFalse + Remove-Item -Force -Path $script:lookupTableFilePath -ErrorAction SilentlyContinue + Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeFalse # perform Get on an adapter - this should create adapter lookup table file $oldPSModulePath = $env:PSModulePath @@ -132,7 +129,8 @@ Describe 'tests for resource discovery' { $env:PSModulePath += [System.IO.Path]::PathSeparator + $TestClassResourcePath "{'Name':'TestClassResource1'}" | dsc resource get -r 'TestClassResource/TestClassResource' | Out-Null - Test-Path $lookupTableFilePath -PathType Leaf | Should -BeTrue + Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeTrue + $script:lookupTableFilePath | Should -FileContentMatchExactly "TestClassResource/TestClassResource" $env:PSModulePath = $oldPSModulePath } From 3b27828666794e1309944a3dd5eda832e78bd060 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 4 Sep 2024 16:11:56 -0700 Subject: [PATCH 23/30] test update 2 --- dsc_lib/src/discovery/command_discovery.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 87faf231b..e3341e677 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -516,7 +516,7 @@ fn sort_adapters_based_on_lookup_table(unsorted_adapters: &BTreeMap) { - if let Ok(lookup_table_json) = serde_json::to_string_pretty(&lookup_table) { + if let Ok(lookup_table_json) = serde_json::to_string(&lookup_table) { let file_path = get_lookup_table_file_path(); debug!("Saving lookup table with {} items to {:?}", lookup_table.len(), file_path); From fbc50a168acd70d6980df7c46f7ac59233d4756f Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 4 Sep 2024 16:36:24 -0700 Subject: [PATCH 24/30] test update 3 --- dsc/tests/dsc_discovery.tests.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dsc/tests/dsc_discovery.tests.ps1 b/dsc/tests/dsc_discovery.tests.ps1 index 7b3ad399d..9f9e74bec 100644 --- a/dsc/tests/dsc_discovery.tests.ps1 +++ b/dsc/tests/dsc_discovery.tests.ps1 @@ -104,7 +104,7 @@ Describe 'tests for resource discovery' { } It 'Ensure List operation populates adapter lookup table' { - + $env:DSC_RESOURCE_PATH = $null # remove adapter lookup table file Remove-Item -Force -Path $script:lookupTableFilePath -ErrorAction SilentlyContinue Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeFalse @@ -112,8 +112,8 @@ Describe 'tests for resource discovery' { # perform List on an adapter - this should create adapter lookup table file dsc resource list -a Microsoft.DSC/PowerShell | Out-Null - Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeTrue $script:lookupTableFilePath | Should -FileContentMatchExactly 'Microsoft.DSC/PowerShell' + Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeTrue } It 'Ensure non-List operation populates adapter lookup table' { @@ -127,10 +127,10 @@ Describe 'tests for resource discovery' { $TestClassResourcePath = Resolve-Path "$PSScriptRoot/../../powershell-adapter/Tests" $env:DSC_RESOURCE_PATH = $null $env:PSModulePath += [System.IO.Path]::PathSeparator + $TestClassResourcePath - "{'Name':'TestClassResource1'}" | dsc resource get -r 'TestClassResource/TestClassResource' | Out-Null + "{'Name':'TestClassResource1'}" | dsc -l trace resource get -r 'TestClassResource/TestClassResource' | Out-Null Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeTrue - $script:lookupTableFilePath | Should -FileContentMatchExactly "TestClassResource/TestClassResource" + $script:lookupTableFilePath | Should -FileContentMatchExactly 'testclassresource/testclassresource' $env:PSModulePath = $oldPSModulePath } From d1a3d0df9267b56b1fe140d5d4826cb9e3e00c99 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 4 Sep 2024 17:01:00 -0700 Subject: [PATCH 25/30] test update 4 --- dsc/tests/dsc_discovery.tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsc/tests/dsc_discovery.tests.ps1 b/dsc/tests/dsc_discovery.tests.ps1 index 9f9e74bec..313f413f0 100644 --- a/dsc/tests/dsc_discovery.tests.ps1 +++ b/dsc/tests/dsc_discovery.tests.ps1 @@ -127,7 +127,7 @@ Describe 'tests for resource discovery' { $TestClassResourcePath = Resolve-Path "$PSScriptRoot/../../powershell-adapter/Tests" $env:DSC_RESOURCE_PATH = $null $env:PSModulePath += [System.IO.Path]::PathSeparator + $TestClassResourcePath - "{'Name':'TestClassResource1'}" | dsc -l trace resource get -r 'TestClassResource/TestClassResource' | Out-Null + "{'Name':'TestClassResource1'}" | dsc resource get -r 'TestClassResource/TestClassResource' | Out-Null Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeTrue $script:lookupTableFilePath | Should -FileContentMatchExactly 'testclassresource/testclassresource' From 79c0a0adf37afcbf5d2815b669a7d354f5fa84cd Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 4 Sep 2024 19:07:11 -0700 Subject: [PATCH 26/30] test update 5 --- dsc/tests/dsc_discovery.tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dsc/tests/dsc_discovery.tests.ps1 b/dsc/tests/dsc_discovery.tests.ps1 index 313f413f0..e12586862 100644 --- a/dsc/tests/dsc_discovery.tests.ps1 +++ b/dsc/tests/dsc_discovery.tests.ps1 @@ -111,7 +111,7 @@ Describe 'tests for resource discovery' { # perform List on an adapter - this should create adapter lookup table file dsc resource list -a Microsoft.DSC/PowerShell | Out-Null - + gc -raw $script:lookupTableFilePath $script:lookupTableFilePath | Should -FileContentMatchExactly 'Microsoft.DSC/PowerShell' Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeTrue } From 1bdcf36778bd2e63e04ab639660f770ea74bb017 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 4 Sep 2024 19:36:18 -0700 Subject: [PATCH 27/30] test update 6 --- dsc/tests/dsc_discovery.tests.ps1 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dsc/tests/dsc_discovery.tests.ps1 b/dsc/tests/dsc_discovery.tests.ps1 index e12586862..d394ca8b4 100644 --- a/dsc/tests/dsc_discovery.tests.ps1 +++ b/dsc/tests/dsc_discovery.tests.ps1 @@ -104,16 +104,20 @@ Describe 'tests for resource discovery' { } It 'Ensure List operation populates adapter lookup table' { - $env:DSC_RESOURCE_PATH = $null # remove adapter lookup table file Remove-Item -Force -Path $script:lookupTableFilePath -ErrorAction SilentlyContinue Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeFalse # perform List on an adapter - this should create adapter lookup table file + $oldPSModulePath = $env:PSModulePath + $TestClassResourcePath = Resolve-Path "$PSScriptRoot/../../powershell-adapter/Tests" + $env:DSC_RESOURCE_PATH = $null + $env:PSModulePath += [System.IO.Path]::PathSeparator + $TestClassResourcePath dsc resource list -a Microsoft.DSC/PowerShell | Out-Null gc -raw $script:lookupTableFilePath $script:lookupTableFilePath | Should -FileContentMatchExactly 'Microsoft.DSC/PowerShell' Test-Path $script:lookupTableFilePath -PathType Leaf | Should -BeTrue + $env:PSModulePath = $oldPSModulePath } It 'Ensure non-List operation populates adapter lookup table' { From 3aaee8022ba937cd9ca245b7d8911c47642c7a14 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 16 Sep 2024 12:24:07 -0700 Subject: [PATCH 28/30] test update 7 --- dsc_lib/src/discovery/command_discovery.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index e3341e677..77a58a836 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -510,8 +510,8 @@ fn load_manifest(path: &Path) -> Result { fn sort_adapters_based_on_lookup_table(unsorted_adapters: &BTreeMap>, needed_resource_types: &Vec) -> LinkedHashMap> { - let mut result:LinkedHashMap> = LinkedHashMap::new(); - let lookup_table:HashMap = load_adapted_resources_lookup_table(); + let mut result = LinkedHashMap::>::new(); + let lookup_table = load_adapted_resources_lookup_table(); // first add adapters (for needed types) that can be found in the lookup table for needed_resource in needed_resource_types { if let Some(adapter_name) = lookup_table.get(needed_resource) { @@ -534,11 +534,14 @@ fn sort_adapters_based_on_lookup_table(unsorted_adapters: &BTreeMap>) { - let mut lookup_table:HashMap = load_adapted_resources_lookup_table(); + let mut lookup_table = load_adapted_resources_lookup_table(); for (resource_name, res_vec) in adapted_resources { - let adapter_name = &res_vec[0].require_adapter.as_ref().unwrap(); - lookup_table.insert(resource_name.to_string().to_lowercase(), (*adapter_name).to_string()); + if let Some(adapter_name) = &res_vec[0].require_adapter { + lookup_table.insert(resource_name.to_string().to_lowercase(), adapter_name.to_string()); + } else { + debug!("Resource '{resource_name}' in 'adapted_resources' is missing 'require_adapter' field."); + } }; save_adapted_resources_lookup_table(&lookup_table); From cfb01320c75d13798bca24c3c3e705e89368aa68 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 16 Sep 2024 12:33:29 -0700 Subject: [PATCH 29/30] clippy fix --- dsc_lib/src/discovery/command_discovery.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 77a58a836..2c3bc7986 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -575,11 +575,7 @@ fn load_adapted_resources_lookup_table() -> HashMap let file_path = get_lookup_table_file_path(); let lookup_table: HashMap = match fs::read(file_path.clone()){ - Ok(data) => { match serde_json::from_slice(&data) { - Ok(lt) => { lt }, - Err(_) => { HashMap::new() } - } - }, + Ok(data) => { serde_json::from_slice(&data).unwrap_or_default() }, Err(_) => { HashMap::new() } }; From 72ca7e2953b46c1c746a0b09874f8425b1feba13 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 16 Sep 2024 14:51:37 -0700 Subject: [PATCH 30/30] debug-info --- dsc_lib/src/discovery/command_discovery.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dsc_lib/src/discovery/command_discovery.rs b/dsc_lib/src/discovery/command_discovery.rs index 2c3bc7986..04772ee75 100644 --- a/dsc_lib/src/discovery/command_discovery.rs +++ b/dsc_lib/src/discovery/command_discovery.rs @@ -540,7 +540,7 @@ fn add_resources_to_lookup_table(adapted_resources: &BTreeMap) if let Some(prefix) = path.parent() { if fs::create_dir_all(prefix).is_ok() { if fs::write(file_path.clone(), lookup_table_json).is_err() { - debug!("Unable to write lookup_table file {file_path:?}"); + info!("Unable to write lookup_table file {file_path:?}"); } } else { - debug!("Unable to create parent directories of the lookup_table file {file_path:?}"); + info!("Unable to create parent directories of the lookup_table file {file_path:?}"); } } else { - debug!("Unable to get directory of the lookup_table file {file_path:?}"); + info!("Unable to get directory of the lookup_table file {file_path:?}"); } } else { - debug!("Unable to serialize lookup_table to json"); + info!("Unable to serialize lookup_table to json"); } }