From 520de207e351fb5bdd359e555e56121905a772ba Mon Sep 17 00:00:00 2001 From: nnikolopoulos Date: Thu, 9 Feb 2023 15:29:10 +0100 Subject: [PATCH 1/5] Replace DataSource with options.TargetDataSource for A and CF and adjust SequenceImportTest --- ifrs17-template/Test/SequenceImportTest.ipynb | 20 ------------------- ifrs17/Import/Importers.ipynb | 4 ++-- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/ifrs17-template/Test/SequenceImportTest.ipynb b/ifrs17-template/Test/SequenceImportTest.ipynb index fec21ac1..542c57bc 100644 --- a/ifrs17-template/Test/SequenceImportTest.ipynb +++ b/ifrs17-template/Test/SequenceImportTest.ipynb @@ -78,16 +78,6 @@ "execution_count": 0, "outputs": [] }, - { - "cell_type": "code", - "source": [ - "ws1.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\nws1.Initialize(x => x.FromSource(DataSource));" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, { "cell_type": "code", "source": [ @@ -152,16 +142,6 @@ "execution_count": 0, "outputs": [] }, - { - "cell_type": "code", - "source": [ - "ws2.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\nws2.Initialize(x => x.FromSource(DataSource));" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, { "cell_type": "code", "source": [ diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index fe8f34ec..5c7aece9 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -1303,7 +1303,7 @@ "\n log = log.Merge(await ComputeAsync(args, workspace, workspaceToCompute, args == primaryArgs));", "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", "\n }", - "\n await workspaceToCompute.CommitToTargetAsync(DataSource);", + "\n await workspaceToCompute.CommitToTargetAsync(options.TargetDataSource);", "\n return Activity.Finish().Merge(log); ", "\n});" ], @@ -1404,7 +1404,7 @@ "\n log = log.Merge(await ComputeAsync(args, workspace, workspaceToCompute, false)); ", "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", "\n }", - "\n await workspaceToCompute.CommitToTargetAsync(DataSource);", + "\n await workspaceToCompute.CommitToTargetAsync(options.TargetDataSource);", "\n return Activity.Finish().Merge(log);", "\n});" ], From 544ffe0da7cde9ba6397f4a40f166e9b94e7b56a Mon Sep 17 00:00:00 2001 From: nnikolopoulos Date: Wed, 15 Feb 2023 15:15:49 +0100 Subject: [PATCH 2/5] Fix .WithActivityLog() --- ifrs17-template/Test/MapTemplateAndImportTest.ipynb | 12 ++++++------ .../Test/ReimportWithDifferentScopeTest.ipynb | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ifrs17-template/Test/MapTemplateAndImportTest.ipynb b/ifrs17-template/Test/MapTemplateAndImportTest.ipynb index dbcf479d..4b071121 100644 --- a/ifrs17-template/Test/MapTemplateAndImportTest.ipynb +++ b/ifrs17-template/Test/MapTemplateAndImportTest.ipynb @@ -165,7 +165,7 @@ "\n .GroupofContractConfiguration(typeof(ReinsurancePortfolio))", "\n .GroupofContractConfiguration(typeof(InsurancePortfolio))", "\n .MainTabConfiguration(partition)", - "\n .WithActivityLog(Session, DataSource, DataSetReader)", + "\n .WithActivityLog()", "\n .ExecuteAsync();", "\n", "\nexportResult.ActivityLog.Status.Should().Be(ActivityLogStatus.Succeeded);" @@ -184,7 +184,7 @@ "\n .GroupofContractConfiguration(typeof(ReinsurancePortfolio))", "\n .GroupofContractConfiguration(typeof(InsurancePortfolio))", "\n .MainTabConfiguration(partition)", - "\n .WithActivityLog(Session, DataSource, DataSetReader)", + "\n .WithActivityLog()", "\n .ExecuteAsync()" ], "metadata": {}, @@ -266,7 +266,7 @@ "\n .StateEnumConfiguration() ", "\n .DataNodeStateConfiguration(dataNodeStates)", "\n .MainTabConfiguration(partition)", - "\n .WithActivityLog(Session, DataSource, DataSetReader)", + "\n .WithActivityLog()", "\n .ExecuteAsync();", "\n", "\nexportResult.ActivityLog.Status.Should().Be(ActivityLogStatus.Succeeded);" @@ -283,7 +283,7 @@ "\n .StateEnumConfiguration() ", "\n .DataNodeStateConfiguration(dataNodeStates)", "\n .MainTabConfiguration(partition)", - "\n .WithActivityLog(Session, DataSource, DataSetReader)", + "\n .WithActivityLog()", "\n .ExecuteAsync()" ], "metadata": {}, @@ -360,7 +360,7 @@ "\n .WithSource(Workspace)", "\n .DataNodeParameterConfiguration(dataNodeParameters)", "\n .MainTabConfiguration(partition)", - "\n .WithActivityLog(Session, DataSource, DataSetReader)", + "\n .WithActivityLog()", "\n .ExecuteAsync();", "\n", "\nexportResult.ActivityLog.Status.Should().Be(ActivityLogStatus.Succeeded);" @@ -376,7 +376,7 @@ "\n .WithSource(Workspace)", "\n .DataNodeParameterConfiguration(dataNodeParameters)", "\n .MainTabConfiguration(partition)", - "\n .WithActivityLog(Session, DataSource, DataSetReader)", + "\n .WithActivityLog()", "\n .ExecuteAsync()" ], "metadata": {}, diff --git a/ifrs17-template/Test/ReimportWithDifferentScopeTest.ipynb b/ifrs17-template/Test/ReimportWithDifferentScopeTest.ipynb index 84609e9c..37e4280a 100644 --- a/ifrs17-template/Test/ReimportWithDifferentScopeTest.ipynb +++ b/ifrs17-template/Test/ReimportWithDifferentScopeTest.ipynb @@ -51,7 +51,7 @@ "await Import.FromFile(\"../Files/TransactionalData/NominalCashflows_CH_2020_12.csv\")", "\n .WithFormat(ImportFormats.Cashflow)", "\n .WithTarget(DataSource)", - "\n .WithActivityLog(Session, DataSource, DataSetReader)", + "\n .WithActivityLog()", "\n .ExecuteAsync()" ], "metadata": {}, @@ -86,7 +86,7 @@ "await Import.FromFile(\"Data/NominalCashflows_CH_2020_12_DT1.1NoPrem.csv\")", "\n .WithFormat(ImportFormats.Cashflow)", "\n .WithTarget(DataSource)", - "\n .WithActivityLog(Session, DataSource, DataSetReader)", + "\n .WithActivityLog()", "\n .ExecuteAsync()" ], "metadata": {}, From dd206f3de7739b2acb3db84aec1facea4194dfff Mon Sep 17 00:00:00 2001 From: nnikolopoulos Date: Wed, 15 Feb 2023 16:34:46 +0100 Subject: [PATCH 3/5] Replace DataSource with options.TargetDataSource except in CommitPartitionAsync (remove empty cell and fix markdown typo) --- ifrs17/Import/Importers.ipynb | 135 ++++++++++++++++------------------ 1 file changed, 63 insertions(+), 72 deletions(-) diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index 76128e63..3dc162d6 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -422,12 +422,12 @@ { "cell_type": "code", "source": [ - "public async Task GetArgsAndCommitPartitionAsync(IDataSet dataSet)", + "public async Task GetArgsAndCommitPartitionAsync(IDataSet dataSet, IDataSource targetDataSource)", "\n{", "\n var args = GetArgsFromMain(dataSet);", "\n if(ApplicationMessage.HasErrors()) return null;", "\n if(args.ReportingNode == default(string)) { ApplicationMessage.Log(Error.ReportingNodeInMainNotFound); return null; }", - "\n await CommitPartitionAsync(args, DataSource);", + "\n await CommitPartitionAsync(args, targetDataSource);", "\n return args;", "\n}" ], @@ -447,7 +447,7 @@ { "cell_type": "code", "source": [ - "public async Task DataNodeFactoryAsync(IDataSet dataSet, string tableName, ImportArgs args)", + "public async Task DataNodeFactoryAsync(IDataSet dataSet, string tableName, ImportArgs args, IDataSource targetDataSource)", "\n{", "\n var partition = (await DataSource.Query().Where(p => p.ReportingNode == args.ReportingNode).ToArrayAsync()).FirstOrDefault();", "\n if(partition == null) { ApplicationMessage.Log(Error.ParsedPartitionNotFound); return; }", @@ -455,15 +455,15 @@ "\n var table = dataSet.Tables[tableName];", "\n ", "\n var dataNodesImported = table.Rows.Select(x => x.Field(nameof(RawVariable.DataNode))).ToHashSet();", - "\n var dataNodesDefined = await DataSource.Query().Where(x => dataNodesImported.Contains(x.SystemName)).ToArrayAsync();", - "\n var dataNodeStatesDefined = await DataSource.Query().Select(x => x.DataNode).ToArrayAsync();", - "\n var dataNodeParametersDefined = await DataSource.Query().Select(x => x.DataNode).ToArrayAsync(); ", + "\n var dataNodesDefined = await targetDataSource.Query().Where(x => dataNodesImported.Contains(x.SystemName)).ToArrayAsync();", + "\n var dataNodeStatesDefined = await targetDataSource.Query().Select(x => x.DataNode).ToArrayAsync();", + "\n var dataNodeParametersDefined = await targetDataSource.Query().Select(x => x.DataNode).ToArrayAsync(); ", "\n var dataNodeStatesUndefined = dataNodesImported.Where(x => x != null && !dataNodeStatesDefined.Contains(x)).ToHashSet();", "\n var dataNodeSingleParametersUndefined = dataNodesImported.Where(x => x != null &&", "\n !dataNodeParametersDefined.Contains(x) && ", "\n dataNodesDefined.SingleOrDefault(y => y.SystemName == x) is GroupOfInsuranceContract).ToHashSet();", "\n if ((dataNodeStatesUndefined?.Any() ?? false))", - "\n await DataSource.UpdateAsync( dataNodeStatesUndefined.Select(x => ", + "\n await targetDataSource.UpdateAsync( dataNodeStatesUndefined.Select(x => ", "\n new DataNodeState {DataNode = x, ", "\n Year = args.Year, ", "\n Month = DefaultDataNodeActivationMonth, ", @@ -471,14 +471,14 @@ "\n Partition = partition.Id})", "\n .ToArray() );", "\n if ((dataNodeSingleParametersUndefined?.Any() ?? false))", - "\n await DataSource.UpdateAsync( dataNodeSingleParametersUndefined.Select(x => ", + "\n await targetDataSource.UpdateAsync( dataNodeSingleParametersUndefined.Select(x => ", "\n new SingleDataNodeParameter {DataNode = x, ", "\n Year = args.Year, ", "\n Month = DefaultDataNodeActivationMonth, ", "\n PremiumAllocation = DefaultPremiumExperienceAdjustmentFactor, ", "\n Partition = partition.Id})", "\n .ToArray() );", - "\n await DataSource.CommitAsync();", + "\n await targetDataSource.CommitAsync();", "\n}" ], "metadata": {}, @@ -503,7 +503,7 @@ { "cell_type": "code", "source": [ - "public async Task GetAllArgsAsync(ImportArgs args, IDataSource dataSource, string format)", + "public async Task GetAllArgsAsync(ImportArgs args, IDataSource dataSource, string format, IDataSource targetDataSource)", "\n{", "\n ImportArgs[] allArgs;", "\n switch(format)", @@ -529,7 +529,7 @@ "\n default : ", "\n {", "\n if(args.Scenario != null) return args.RepeatOnce().ToArray();", - "\n var secondaryArgs = await DataSource.Query() ", + "\n var secondaryArgs = await targetDataSource.Query() ", "\n .Where(x => x.ReportingNode == args.ReportingNode && x.Year == args.Year && x.Month == args.Month && x.Scenario != null)", "\n .Select(x => new ImportArgs(x.ReportingNode, x.Year, x.Month, default(Periodicity), x.Scenario, ImportFormats.Cashflow)).ToArrayAsync();", "\n ", @@ -797,7 +797,7 @@ "source": [ "# Yield Curves ", "\n", - "\n[Yield curves](../DataModel/DataStructure#yield-curve) are imported for a specific year, month, and scenario (not required if best estimate). This informations are collected in the main section of the file. ", + "\n[Yield curves](../DataModel/DataStructure#yield-curve) are imported for a specific year, month, and scenario (not required if best estimate). This information is collected in the main section of the file. ", "\n", "\nYield curves to be imported are compared against those already present in the DataSource and only the new ones are stored." ], @@ -839,7 +839,7 @@ "\n return Activity.Finish().Merge(importLog);", "\n }", "\n", - "\n var allArgs = await GetAllArgsAsync(primaryArgs, options.TargetDataSource, ImportFormats.YieldCurve);", + "\n var allArgs = await GetAllArgsAsync(primaryArgs, options.TargetDataSource, ImportFormats.YieldCurve, options.TargetDataSource);", "\n var updatedCurrencies = toCommitYieldCurves.Select(x => x.Currency).Distinct();", "\n var dataNodesToUpdate = await workspace.Query().Where(x => updatedCurrencies.Contains(x.ContractualCurrency)).Select(x => x.SystemName).ToArrayAsync();", "\n var workspaceToCompute = Workspace.CreateNew();", @@ -900,20 +900,20 @@ { "cell_type": "code", "source": [ - "public async Task UploadDataNodesToWorkspaceAsync(IDataSet dataSet, IWorkspace workspace)", + "public async Task UploadDataNodesToWorkspaceAsync(IDataSet dataSet, IWorkspace workspace, IDataSource targetDataSource)", "\n{", "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(DataSource)", + "\n workspace.Initialize(x => x.FromSource(targetDataSource)", "\n .DisableInitialization()", "\n .DisableInitialization()", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n ", "\n Activity.Start();", - "\n var args = await GetArgsAndCommitPartitionAsync(dataSet);", + "\n var args = await GetArgsAndCommitPartitionAsync(dataSet, targetDataSource);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", - "\n var storage = new ParsingStorage(args, DataSource, workspace);", + "\n var storage = new ParsingStorage(args, targetDataSource, workspace);", "\n await storage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", @@ -1009,12 +1009,12 @@ "source": [ "Import.DefineFormat(ImportFormats.DataNode, async (options, dataSet) => {", "\n var workspace = Workspace.CreateNew();", - "\n var log = await UploadDataNodesToWorkspaceAsync(dataSet, workspace);", + "\n var log = await UploadDataNodesToWorkspaceAsync(dataSet, workspace, options.TargetDataSource);", "\n var partition = (Guid)workspace.Partition.GetCurrent(nameof(PartitionByReportingNode));", - "\n await workspace.CommitToAsync(DataSource, partition);", - "\n await workspace.CommitToAsync(DataSource, partition);", - "\n await workspace.CommitToAsync(DataSource, partition);", - "\n await workspace.CommitToAsync(DataSource, partition);", + "\n await workspace.CommitToAsync(options.TargetDataSource, partition);", + "\n await workspace.CommitToAsync(options.TargetDataSource, partition);", + "\n await workspace.CommitToAsync(options.TargetDataSource, partition);", + "\n await workspace.CommitToAsync(options.TargetDataSource, partition);", "\n return log;", "\n});" ], @@ -1034,27 +1034,18 @@ { "cell_type": "code", "source": [ - "" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public async Task UploadDataNodeStateToWorkspaceAsync(IDataSet dataSet, IWorkspace workspace)", + "public async Task UploadDataNodeStateToWorkspaceAsync(IDataSet dataSet, IWorkspace workspace, IDataSource targetDataSource)", "\n{", "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(DataSource)", + "\n workspace.Initialize(x => x.FromSource(targetDataSource)", "\n .DisableInitialization()", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n Activity.Start();", - "\n var args = await GetArgsAndCommitPartitionAsync(dataSet) with {ImportFormat = ImportFormats.DataNodeState};", + "\n var args = await GetArgsAndCommitPartitionAsync(dataSet, targetDataSource) with {ImportFormat = ImportFormats.DataNodeState};", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", - "\n var storage = new ParsingStorage(args, DataSource, workspace);", + "\n var storage = new ParsingStorage(args, targetDataSource, workspace);", "\n await storage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", @@ -1081,8 +1072,8 @@ "source": [ "Import.DefineFormat(ImportFormats.DataNodeState, async (options, dataSet) => {", "\n var workspace = Workspace.CreateNew();", - "\n var log = await UploadDataNodeStateToWorkspaceAsync(dataSet, workspace);", - "\n await workspace.CommitToAsync(DataSource, (Guid)workspace.Partition.GetCurrent(nameof(PartitionByReportingNode)), snapshot: false); ", + "\n var log = await UploadDataNodeStateToWorkspaceAsync(dataSet, workspace, options.TargetDataSource);", + "\n await workspace.CommitToAsync(options.TargetDataSource, (Guid)workspace.Partition.GetCurrent(nameof(PartitionByReportingNode)), snapshot: false); ", "\n return log;", "\n});" ], @@ -1102,18 +1093,18 @@ { "cell_type": "code", "source": [ - "public async Task UploadDataNodeParameterToWorkspaceAsync(IDataSet dataSet, Guid targetPartitionByReportingNodeId, IWorkspace workspace)", + "public async Task UploadDataNodeParameterToWorkspaceAsync(IDataSet dataSet, Guid targetPartitionByReportingNodeId, IWorkspace workspace, IDataSource targetDataSource)", "\n{", "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(DataSource)", + "\n workspace.Initialize(x => x.FromSource(targetDataSource)", "\n .DisableInitialization()", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n Activity.Start();", - "\n var args = await GetArgsAndCommitPartitionAsync(dataSet) with {ImportFormat = ImportFormats.DataNodeParameter};", + "\n var args = await GetArgsAndCommitPartitionAsync(dataSet, targetDataSource) with {ImportFormat = ImportFormats.DataNodeParameter};", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", - "\n var storage = new ParsingStorage(args, DataSource, workspace);", + "\n var storage = new ParsingStorage(args, targetDataSource, workspace);", "\n await storage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", @@ -1193,9 +1184,9 @@ "Import.DefineFormat(ImportFormats.DataNodeParameter, async (options, dataSet) => {", "\n Guid partitionId = new Guid();", "\n var workspace = Workspace.CreateNew();", - "\n var log = await UploadDataNodeParameterToWorkspaceAsync(dataSet, partitionId, workspace);", - "\n await workspace.CommitToAsync(DataSource, partitionId, snapshot: false);", - "\n await workspace.CommitToAsync(DataSource, partitionId, snapshot: false); ", + "\n var log = await UploadDataNodeParameterToWorkspaceAsync(dataSet, partitionId, workspace, options.TargetDataSource);", + "\n await workspace.CommitToAsync(options.TargetDataSource, partitionId, snapshot: false);", + "\n await workspace.CommitToAsync(options.TargetDataSource, partitionId, snapshot: false); ", "\n ", "\n return log;", "\n});" @@ -1222,13 +1213,13 @@ { "cell_type": "code", "source": [ - "public async Task ParseCashflowsToWorkspaceAsync(IDataSet dataSet, ImportArgs args, IWorkspace workspace)", + "public async Task ParseCashflowsToWorkspaceAsync(IDataSet dataSet, ImportArgs args, IWorkspace workspace, IDataSource targetDataSource)", "\n{", "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(DataSource).DisableInitialization().DisableInitialization());", + "\n workspace.Initialize(x => x.FromSource(targetDataSource).DisableInitialization().DisableInitialization());", "\n ", "\n Activity.Start();", - "\n var parsingStorage = new ParsingStorage(args, DataSource, workspace);", + "\n var parsingStorage = new ParsingStorage(args, targetDataSource, workspace);", "\n await parsingStorage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", @@ -1296,19 +1287,19 @@ "source": [ "Import.DefineFormat(ImportFormats.Cashflow, async (options, dataSet) => {", "\n Activity.Start();", - "\n var primaryArgs = await GetArgsAndCommitPartitionAsync(dataSet) with {ImportFormat = ImportFormats.Cashflow};", + "\n var primaryArgs = await GetArgsAndCommitPartitionAsync(dataSet, options.TargetDataSource) with {ImportFormat = ImportFormats.Cashflow};", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", - "\n var allArgs = await GetAllArgsAsync(primaryArgs, options.TargetDataSource, ImportFormats.Cashflow);", - "\n await DataNodeFactoryAsync(dataSet, ImportFormats.Cashflow, primaryArgs);", + "\n var allArgs = await GetAllArgsAsync(primaryArgs, options.TargetDataSource, ImportFormats.Cashflow, options.TargetDataSource);", + "\n await DataNodeFactoryAsync(dataSet, ImportFormats.Cashflow, primaryArgs, options.TargetDataSource);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", "\n var workspace = Workspace.CreateNew();", - "\n var log = await ParseCashflowsToWorkspaceAsync(dataSet, primaryArgs, workspace);", + "\n var log = await ParseCashflowsToWorkspaceAsync(dataSet, primaryArgs, workspace, options.TargetDataSource);", "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", "\n", "\n var workspaceToCompute = Workspace.CreateNew();", - "\n workspaceToCompute.Initialize(x => x.FromSource(DataSource));", + "\n workspaceToCompute.Initialize(x => x.FromSource(options.TargetDataSource));", "\n foreach (var args in allArgs) {", "\n log = log.Merge(await ComputeAsync(args, workspace, workspaceToCompute, args == primaryArgs));", "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", @@ -1341,15 +1332,15 @@ { "cell_type": "code", "source": [ - "public async Task ParseActualsToWorkspaceAsync(IDataSet dataSet, ImportArgs args, IWorkspace workspace)", + "public async Task ParseActualsToWorkspaceAsync(IDataSet dataSet, ImportArgs args, IWorkspace workspace, IDataSource targetDataSource)", "\n{", "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(DataSource)", + "\n workspace.Initialize(x => x.FromSource(targetDataSource)", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n ", "\n Activity.Start();", - "\n var parsingStorage = new ParsingStorage(args, DataSource, workspace);", + "\n var parsingStorage = new ParsingStorage(args, targetDataSource, workspace);", "\n await parsingStorage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", @@ -1397,19 +1388,19 @@ "source": [ "Import.DefineFormat(ImportFormats.Actual, async (options, dataSet) => {", "\n Activity.Start();", - "\n var primaryArgs = await GetArgsAndCommitPartitionAsync(dataSet) with {ImportFormat = ImportFormats.Actual};", + "\n var primaryArgs = await GetArgsAndCommitPartitionAsync(dataSet, options.TargetDataSource) with {ImportFormat = ImportFormats.Actual};", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", - "\n var allArgs = await GetAllArgsAsync(primaryArgs, options.TargetDataSource, ImportFormats.Actual);", - "\n await DataNodeFactoryAsync(dataSet, ImportFormats.Actual, primaryArgs);", + "\n var allArgs = await GetAllArgsAsync(primaryArgs, options.TargetDataSource, ImportFormats.Actual, options.TargetDataSource);", + "\n await DataNodeFactoryAsync(dataSet, ImportFormats.Actual, primaryArgs, options.TargetDataSource);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", "\n var workspace = Workspace.CreateNew();", - "\n var log = await ParseActualsToWorkspaceAsync(dataSet, primaryArgs, workspace);", + "\n var log = await ParseActualsToWorkspaceAsync(dataSet, primaryArgs, workspace, options.TargetDataSource);", "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", "\n", "\n var workspaceToCompute = Workspace.CreateNew();", - "\n workspaceToCompute.Initialize(x => x.FromSource(DataSource));", + "\n workspaceToCompute.Initialize(x => x.FromSource(options.TargetDataSource));", "\n foreach (var args in allArgs) {", "\n log = log.Merge(await ComputeAsync(args, workspace, workspaceToCompute, false)); ", "\n if(log.Errors.Any()) return Activity.Finish().Merge(log);", @@ -1437,16 +1428,16 @@ { "cell_type": "code", "source": [ - "public async Task ParseSimpleValueToWorkspaceAsync(IDataSet dataSet, ImportArgs args, Guid targetPartitionByReportingNodeAndPeriodId, IWorkspace workspace)", + "public async Task ParseSimpleValueToWorkspaceAsync(IDataSet dataSet, ImportArgs args, Guid targetPartitionByReportingNodeAndPeriodId, IWorkspace workspace, IDataSource targetDataSource)", "\n{", "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(DataSource)", + "\n workspace.Initialize(x => x.FromSource(targetDataSource)", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n ", "\n Activity.Start();", "\n var importFormat = args.ImportFormat;", - "\n var parsingStorage = new ParsingStorage(args, DataSource, workspace);", + "\n var parsingStorage = new ParsingStorage(args, targetDataSource, workspace);", "\n await parsingStorage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish(); ", "\n", @@ -1506,20 +1497,20 @@ "source": [ "Import.DefineFormat(ImportFormats.SimpleValue, async (options, dataSet) => {", "\n Activity.Start();", - "\n var args = await GetArgsAndCommitPartitionAsync(dataSet) with {ImportFormat = ImportFormats.SimpleValue};", + "\n var args = await GetArgsAndCommitPartitionAsync(dataSet, options.TargetDataSource) with {ImportFormat = ImportFormats.SimpleValue};", "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n await DataNodeFactoryAsync(dataSet, ImportFormats.SimpleValue, args);", + "\n await DataNodeFactoryAsync(dataSet, ImportFormats.SimpleValue, args, options.TargetDataSource);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", "\n Guid partitionId = new Guid();", "\n var workspace = Workspace.CreateNew();", - "\n var parsingLog = await ParseSimpleValueToWorkspaceAsync(dataSet, args, partitionId, workspace);", + "\n var parsingLog = await ParseSimpleValueToWorkspaceAsync(dataSet, args, partitionId, workspace, options.TargetDataSource);", "\n if(parsingLog.Errors.Any()) return Activity.Finish().Merge(parsingLog);", "\n", "\n workspace.Query().Select(v => new {v.DataNode, v.AccidentYear}).Distinct();", "\n", "\n var targetDataNodes = workspace.Query().Select(v => v.DataNode).Distinct().ToArray();", - "\n await workspace.CommitToAsync(DataSource, partitionId, snapshot : true,", + "\n await workspace.CommitToAsync(options.TargetDataSource, partitionId, snapshot : true,", "\n filter : x => targetDataNodes.Contains(x.DataNode));", "\n return Activity.Finish().Merge(parsingLog);", "\n});" @@ -1544,21 +1535,21 @@ "source": [ "Import.DefineFormat(ImportFormats.Opening, async (options, dataSet) => {", "\n Activity.Start();", - "\n var args = await GetArgsAndCommitPartitionAsync(dataSet) with {ImportFormat = ImportFormats.Opening};", + "\n var args = await GetArgsAndCommitPartitionAsync(dataSet, options.TargetDataSource) with {ImportFormat = ImportFormats.Opening};", "\n if(Activity.HasErrors()) return Activity.Finish();", - "\n await DataNodeFactoryAsync(dataSet, ImportFormats.Opening, args);", + "\n await DataNodeFactoryAsync(dataSet, ImportFormats.Opening, args, options.TargetDataSource);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", "\n Guid partitionId = new Guid();", "\n var workspace = Workspace.CreateNew();", - "\n var parsingLog = await ParseSimpleValueToWorkspaceAsync(dataSet, args, partitionId, workspace);", + "\n var parsingLog = await ParseSimpleValueToWorkspaceAsync(dataSet, args, partitionId, workspace, options.TargetDataSource);", "\n if(parsingLog.Errors.Any()) return Activity.Finish().Merge(parsingLog);", "\n", "\n var workspaceToCompute = Workspace.CreateNew();", - "\n workspaceToCompute.Initialize(x => x.FromSource(DataSource));", + "\n workspaceToCompute.Initialize(x => x.FromSource(options.TargetDataSource));", "\n var calculationLog = await ComputeAsync(args, workspace, workspaceToCompute, false); ", "\n if(calculationLog.Errors.Any()) return Activity.Finish().Merge(calculationLog);", - "\n await workspaceToCompute.CommitToTargetAsync(DataSource, x => x.SnapshotMode());", + "\n await workspaceToCompute.CommitToTargetAsync(options.TargetDataSource, x => x.SnapshotMode());", "\n return Activity.Finish().Merge(parsingLog).Merge(calculationLog);", "\n})" ], From d697f1ac21c33ae6e072ea6091aafc4cd4104a90 Mon Sep 17 00:00:00 2001 From: nnikolopoulos Date: Wed, 15 Feb 2023 16:35:08 +0100 Subject: [PATCH 4/5] Fix MapTemplateAndImportTest --- ifrs17-template/Test/MapTemplateAndImportTest.ipynb | 1 + 1 file changed, 1 insertion(+) diff --git a/ifrs17-template/Test/MapTemplateAndImportTest.ipynb b/ifrs17-template/Test/MapTemplateAndImportTest.ipynb index 4b071121..180b6038 100644 --- a/ifrs17-template/Test/MapTemplateAndImportTest.ipynb +++ b/ifrs17-template/Test/MapTemplateAndImportTest.ipynb @@ -545,6 +545,7 @@ "public async Task CheckErrors(string inputFileName, List errorBms)", "\n{", "\n Activity.Start();", + "\n Workspace.InitializeFrom(DataSource);", "\n var log = await Import.FromFile(inputFileName).WithFormat(ImportFormats.DataNodeParameter).WithTarget(Workspace).ExecuteAsync();", "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", "\n errorBms.Intersect(log.Errors.Select(x => x.ToString().Substring(0,x.ToString().Length-2).Substring(40)).ToArray()).Count().Should().Be(errorBms.Count());", From 31a6e6d487feca4f341d1fc57998699a43c79af7 Mon Sep 17 00:00:00 2001 From: nnikolopoulos Date: Wed, 15 Feb 2023 17:12:16 +0100 Subject: [PATCH 5/5] Use disposable Workspaces --- .../Test/MapTemplateAndImportTest.ipynb | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/ifrs17-template/Test/MapTemplateAndImportTest.ipynb b/ifrs17-template/Test/MapTemplateAndImportTest.ipynb index 180b6038..60ac6284 100644 --- a/ifrs17-template/Test/MapTemplateAndImportTest.ipynb +++ b/ifrs17-template/Test/MapTemplateAndImportTest.ipynb @@ -542,12 +542,10 @@ { "cell_type": "code", "source": [ - "public async Task CheckErrors(string inputFileName, List errorBms)", + "public async Task CheckErrors(string inputFileName, List errorBms, IWorkspace workspace)", "\n{", "\n Activity.Start();", - "\n Workspace.InitializeFrom(DataSource);", - "\n var log = await Import.FromFile(inputFileName).WithFormat(ImportFormats.DataNodeParameter).WithTarget(Workspace).ExecuteAsync();", - "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n var log = await Import.FromFile(inputFileName).WithFormat(ImportFormats.DataNodeParameter).WithTarget(workspace).ExecuteAsync();", "\n errorBms.Intersect(log.Errors.Select(x => x.ToString().Substring(0,x.ToString().Length-2).Substring(40)).ToArray()).Count().Should().Be(errorBms.Count());", "\n return Activity.Finish();", "\n}" @@ -556,6 +554,16 @@ "execution_count": 0, "outputs": [] }, + { + "cell_type": "code", + "source": [ + "var ws1 = Workspace.CreateNew();", + "\nws1.InitializeFrom(DataSource);" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, { "cell_type": "code", "source": [ @@ -563,32 +571,52 @@ "\nvar errorsBm = new List(){Get(Error.InvalidDataNode, \"DataNodeInvalid0\"),", "\n Get(Error.InvalidDataNode, \"DataNodeInvalid1\"),", "\n Get(Error.InvalidDataNode, \"DataNodeInvalid2\")};", - "\nvar activity = await CheckErrors(inputFileName, errorsBm);", + "\nvar activity = await CheckErrors(inputFileName, errorsBm, ws1);", "\nactivity" ], "metadata": {}, "execution_count": 0, "outputs": [] }, + { + "cell_type": "code", + "source": [ + "var ws2 = Workspace.CreateNew();", + "\nws2.InitializeFrom(DataSource);" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, { "cell_type": "code", "source": [ "var inputFileName = \"Data/DataNodeParameter_Duplicate.csv\";", "\nvar errorsBm = new List(){Get(Error.DuplicateSingleDataNode, \"DT1.1\"),", "\n Get(Error.DuplicateInterDataNode, \"DT1.1\",\"DTR1.1\"),};", - "\nvar activity = await CheckErrors(inputFileName, errorsBm);", + "\nvar activity = await CheckErrors(inputFileName, errorsBm, ws2);", "\nactivity" ], "metadata": {}, "execution_count": 0, "outputs": [] }, + { + "cell_type": "code", + "source": [ + "var ws3 = Workspace.CreateNew();", + "\nws3.InitializeFrom(DataSource);" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, { "cell_type": "code", "source": [ "var inputFileName = \"Data/DataNodeParameter_InvalidReinsCov.csv\";", "\nvar errorsBm = new List(){Get(Error.ReinsuranceCoverageDataNode, \"DT1.1\",\"DT1.1\")};", - "\nvar activity = await CheckErrors(inputFileName, errorsBm);", + "\nvar activity = await CheckErrors(inputFileName, errorsBm, ws3);", "\nactivity" ], "metadata": {},