From a10ca65d39529f8e16d0d9961c7c49fd0d8b3d3c Mon Sep 17 00:00:00 2001 From: nnikolopoulos Date: Mon, 31 Oct 2022 18:40:26 +0100 Subject: [PATCH 1/7] Define new ImportFormats --- ifrs17/Constants/Consts.ipynb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ifrs17/Constants/Consts.ipynb b/ifrs17/Constants/Consts.ipynb index 8c67e92e..19c0eec8 100644 --- a/ifrs17/Constants/Consts.ipynb +++ b/ifrs17/Constants/Consts.ipynb @@ -142,6 +142,8 @@ "\n public const string SimpleValue = nameof(SimpleValue);", "\n public const string Opening = nameof(Opening);", "\n public const string AocConfiguration = nameof(AocConfiguration);", + "\n public const string DataNode = nameof(DataNode);", + "\n public const string DataNodeState = nameof(DataNodeState);", "\n}" ] }, From aadb952656e491a6c22806f88487c40e39b524d6 Mon Sep 17 00:00:00 2001 From: nnikolopoulos Date: Mon, 31 Oct 2022 18:42:45 +0100 Subject: [PATCH 2/7] Apply import w format in DN, DNS, DP, CF --- ifrs17/Import/Importers.ipynb | 126 +++++++++++++++------------------- 1 file changed, 57 insertions(+), 69 deletions(-) diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index 6be43097..db45dbf4 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -266,14 +266,12 @@ { "cell_type": "code", "source": [ - "public async Task DataNodeFactoryAsync(string file, string tab, ImportArgs args)", + "public async Task DataNodeFactoryAsync(IDataSet dataSet, string tab, ImportArgs args)", "\n{", "\n var partition = (await DataSource.Query().Where(p => p.ReportingNode == args.ReportingNode && p.Scenario == null).ToArrayAsync()).SingleOrDefault();", "\n if(partition == null) { ApplicationMessage.Log(Error.ParsedPartitionNotFound); return; }", "\n", - "\n var extension = System.IO.Path.GetExtension(file);", - "\n var stream = await Project.FileStorage.ReadAsync(file);", - "\n var target = (await DataSetReader.ReadFromStream(stream).WithContentType(extension).ExecuteAsync()).DataSet.Tables[tab];", + "\n var target = dataSet.Tables[tab];", "\n", "\n var dataNodesImported = target.Rows.Select(x => x.Field(nameof(RawVariable.DataNode))).ToHashSet();", "\n var dataNodesDefined = await DataSource.Query().Where(x => dataNodesImported.Contains(x.SystemName)).ToArrayAsync();", @@ -430,11 +428,9 @@ { "cell_type": "code", "source": [ - "public async Task GetArgsFromMainAsync(string file)", + "public async Task GetArgsFromMainAsync(IDataSet dataSet)", "\n{", - "\n var extension = System.IO.Path.GetExtension(file);", - "\n var stream = await Project.FileStorage.ReadAsync(file);", - "\n var mainTab = (await DataSetReader.ReadFromStream(stream).WithContentType(extension).ExecuteAsync()).DataSet.Tables[Main];", + "\n var mainTab = dataSet.Tables[Main];", "\n ", "\n if(mainTab == null) ApplicationMessage.Log(Error.NoMainTab);", "\n if(mainTab.Rows.Count() == 0) ApplicationMessage.Log(Error.IncompleteMainTab);", @@ -471,7 +467,8 @@ { "cell_type": "markdown", "source": [ - "### Parse and Upload: Data Nodes" + "", + "\n## Parse and Upload: Data Nodes" ] }, { @@ -483,17 +480,17 @@ { "cell_type": "code", "source": [ - "public async Task UploadDataNodesToWorkspaceAsync(string fileName)", - "\n{", - "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n Workspace.Initialize(x => x.FromSource(DataSource)", + "Import.DefineFormat(ImportFormats.DataNode, async (options, dataSet) => {", + "\n var workspace = Workspace.CreateNew();", + "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n workspace.Initialize(x => x.FromSource(DataSource)", "\n .DisableInitialization()", "\n .DisableInitialization()", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n ", "\n Activity.Start();", - "\n var args = await GetArgsFromMainAsync(fileName);", + "\n var args = await GetArgsFromMainAsync(dataSet);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", "\n var storage = new ParsingStorage(args, DataSource, Workspace);", @@ -501,7 +498,7 @@ "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", "\n var errors = new List();", - "\n var importLogPortfolios = await Import.FromFile(fileName)", + "\n var importLogPortfolios = await Import.FromDataSet(dataSet)", "\n .WithType((dataset, datarow) => new InsurancePortfolio {", "\n SystemName = datarow.Field(nameof(DataNode.SystemName)),", "\n DisplayName = datarow.Field(nameof(DataNode.DisplayName)),", @@ -522,11 +519,11 @@ "\n ValuationApproach = datarow.Field(nameof(DataNode.ValuationApproach)),", "\n OciType = datarow.Field(nameof(DataNode.OciType))", "\n })", - "\n .WithTarget(Workspace)", + "\n .WithTarget(workspace)", "\n .ExecuteAsync();", "\n ", "\n var portfolios = await Workspace.Query().ToDictionaryAsync(x => x.SystemName);", - "\n var importLogGroupOfContracts = await Import.FromFile(fileName)", + "\n var importLogGroupOfContracts = await Import.FromDataSet(dataSet)", "\n .WithType((dataset, datarow) => {", "\n var gicSystemName = datarow.Field(nameof(DataNode.SystemName));", "\n var pf = datarow.Field(nameof(InsurancePortfolio));", @@ -576,25 +573,23 @@ "\n };", "\n return ExtendGroupOfContract(gric, datarow);", "\n })", - "\n .WithTarget(Workspace)", + "\n .WithTarget(workspace)", "\n .ExecuteAsync();", "\n ", "\n return Activity.Finish().Merge(importLogPortfolios).Merge(importLogGroupOfContracts);", - "\n}" + "\n});" ] }, { "cell_type": "code", "source": [ - "public async Task UploadDataNodesAsync(string fileName)", + "public async Task UploadDataNodesAsync()", "\n{", - "\n var log = await UploadDataNodesToWorkspaceAsync(fileName);", "\n var partition = (Guid)Workspace.Partition.GetCurrent(nameof(PartitionByReportingNode));", "\n await CommitToDatabase(partition);", "\n await CommitToDatabase(partition);", "\n await CommitToDatabase(partition);", "\n await CommitToDatabase(partition);", - "\n return log;", "\n}" ] }, @@ -628,25 +623,25 @@ { "cell_type": "code", "source": [ - "public async Task UploadDataNodeStateToWorkspaceAsync(string fileName)", - "\n{", - "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n Workspace.Initialize(x => x.FromSource(DataSource)", + "Import.DefineFormat(ImportFormats.DataNodeState, async (options, dataSet) => {", + "\n var workspace = Workspace.CreateNew();", + "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n workspace.Initialize(x => x.FromSource(DataSource)", "\n .DisableInitialization()", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n ", - "\n await Workspace.DeleteAsync(await Workspace.Query().ToArrayAsync() );", + "\n await workspace.DeleteAsync(await workspace.Query().ToArrayAsync() );", "\n ", "\n Activity.Start();", - "\n var args = await GetArgsFromMainAsync(fileName);", + "\n var args = await GetArgsFromMainAsync(dataSet);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", "\n var storage = new ParsingStorage(args, DataSource, Workspace);", "\n await storage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", - "\n var importLog = await Import.FromFile(fileName).WithType(", + "\n var importLog = await Import.FromDataSet(dataSet).WithType(", "\n (dataset, datarow) => new DataNodeState {", "\n DataNode = datarow.Field(nameof(DataNodeState.DataNode)),", "\n State = (State)Enum.Parse(typeof(State), datarow.Field(nameof(DataNodeState.State))),", @@ -654,21 +649,19 @@ "\n Month = args.Month,", "\n Partition = storage.TargetPartitionByReportingNode.Id", "\n }", - "\n ).WithTarget(Workspace).ExecuteAsync();", + "\n ).WithTarget(workspace).ExecuteAsync();", "\n", "\n await ValidateDataNodeStatesAsync(storage.DataNodeDataBySystemName);", "\n return Activity.Finish().Merge(importLog);", - "\n}" + "\n});" ] }, { "cell_type": "code", "source": [ - "public async Task UploadDataNodeStateAsync(string fileName)", + "public async Task UploadDataNodeStateAsync()", "\n{", - "\n var log = await UploadDataNodeStateToWorkspaceAsync(fileName);", - "\n await CommitToDatabase((Guid)Workspace.Partition.GetCurrent(nameof(PartitionByReportingNode)), snapshot: false); ", - "\n return log;", + "\n await CommitToDatabase((Guid)Workspace.Partition.GetCurrent(nameof(PartitionByReportingNode)), snapshot: false);", "\n}" ] }, @@ -681,18 +674,18 @@ { "cell_type": "code", "source": [ - "public async Task UploadDataNodeParameterToWorkspaceAsync(string fileName, Guid targetPartitionByReportingNodeId)", - "\n{", - "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n Workspace.Initialize(x => x.FromSource(DataSource)", + "Import.DefineFormat(ImportFormats.DataNodeParameter, async (options, dataSet) => {", + "\n var workspace = Workspace.CreateNew();", + "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n workspace.Initialize(x => x.FromSource(DataSource)", "\n .DisableInitialization()", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n ", - "\n await Workspace.DeleteAsync(await Workspace.Query().ToArrayAsync() );", + "\n await workspace.DeleteAsync(await workspace.Query().ToArrayAsync() );", "\n ", "\n Activity.Start();", - "\n var args = await GetArgsFromMainAsync(fileName) with {ImportFormat = ImportFormats.DataNodeParameter};", + "\n var args = await GetArgsFromMainAsync(dataSet) with {ImportFormat = ImportFormats.DataNodeParameter};", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", "\n var storage = new ParsingStorage(args, DataSource, Workspace);", @@ -702,7 +695,7 @@ "\n var singleDataNode = new List();", "\n var interDataNode = new List<(string,string)>();", "\n ", - "\n var importLog = await Import.FromFile(fileName)", + "\n var importLog = await Import.FromDataSet(dataSet)", "\n .WithType( (dataset, datarow) => {", "\n", "\n //read and validate DataNodes", @@ -758,26 +751,23 @@ "\n ReinsuranceCoverage = reinsCov,", "\n };", "\n })", - "\n .WithTarget(Workspace)", + "\n .WithTarget(workspace)", "\n .ExecuteAsync();", "\n ", - "\n targetPartitionByReportingNodeId = storage.TargetPartitionByReportingNode.Id;", + "\n Guid targetPartitionByReportingNodeId = storage.TargetPartitionByReportingNode.Id;", "\n return Activity.Finish().Merge(importLog);", - "\n}" + "\n});" ] }, { "cell_type": "code", "source": [ - "public async Task UploadDataNodeParameterAsync(string fileName)", + "public async Task UploadDataNodeParameterAsync()", "\n{", "\n Guid partitionId = new Guid();", - "\n var log = await UploadDataNodeParameterToWorkspaceAsync(fileName, partitionId);", "\n ", "\n await CommitToDatabase(partitionId, snapshot: false);", - "\n await CommitToDatabase(partitionId, snapshot: false); ", - "\n ", - "\n return log;", + "\n await CommitToDatabase(partitionId, snapshot: false);", "\n}" ] }, @@ -813,19 +803,20 @@ { "cell_type": "code", "source": [ - "public async Task ParseCashflowsToWorkspaceAsync(string fileName, ImportArgs args)", - "\n{", - "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n Workspace.Initialize(x => x.FromSource(DataSource)", + "Import.DefineFormat(ImportFormats.Cashflow, async (options, dataSet) => {", + "\n var workspace = Workspace.CreateNew();", + "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n workspace.Initialize(x => x.FromSource(DataSource)", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n ", "\n Activity.Start();", + "\n var args = (await GetArgsFromMainAsync(dataSet)) with {ImportFormat = ImportFormats.Cashflow};", "\n var parsingStorage = new ParsingStorage(args, DataSource, Workspace);", "\n await parsingStorage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", - "\n var importLog = await Import.FromFile(fileName)", + "\n var importLog = await Import.FromDataSet(dataSet)", "\n .WithType ( (dataset, datarow) => {", "\n var aocType = datarow.Field(nameof(RawVariable.AocType));", "\n var novelty = datarow.Field(nameof(RawVariable.Novelty));", @@ -871,34 +862,31 @@ "\n };", "\n return item;", "\n }, ImportFormats.Cashflow", - "\n ).WithTarget(Workspace).ExecuteAsync();", + "\n ).WithTarget(workspace).ExecuteAsync();", "\n ", "\n await ValidateForDataNodeStateActiveAsync(parsingStorage.DataNodeDataBySystemName);", "\n return Activity.Finish().Merge(importLog);", - "\n}" + "\n});" ] }, { "cell_type": "code", "source": [ - "public async Task UploadCashflowsAsync(string fileName)", + "public async Task UploadCashflowsAsync(IDataSet dataSet)", "\n{", "\n Activity.Start();", - "\n var args = (await GetArgsFromMainAsync(fileName)) with {ImportFormat = ImportFormats.Cashflow};", - "\n await DataNodeFactoryAsync(fileName, ImportFormats.Cashflow, args);", + "\n var args = (await GetArgsFromMainAsync(dataSet)) with {ImportFormat = ImportFormats.Cashflow};", + "\n await DataNodeFactoryAsync(dataSet, ImportFormats.Cashflow, args);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", - "\n var parsingLog = await ParseCashflowsToWorkspaceAsync(fileName, args);", - "\n if(parsingLog.Errors.Any()) return Activity.Finish().Merge(parsingLog);", - "\n ", "\n var storage = new ImportStorage(args, DataSource, Workspace);", "\n await storage.InitializeAsync();", - "\n if(Activity.HasErrors()) return Activity.Finish().Merge(parsingLog);", + "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", "\n var universe = Scopes.ForStorage(storage).ToScope();", "\n var identities = universe.GetScopes(storage.DataNodesByImportScope[ImportScope.Primary]).SelectMany(s => s.Identities);", "\n var ivs = universe.GetScopes(identities).SelectMany(x => x.CalculatedIfrsVariables);", - "\n if(Activity.HasErrors()) return Activity.Finish().Merge(parsingLog);", + "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", "\n await Workspace.UpdateAsync(ivs);", "\n await CommitToDatabase(storage.TargetPartition, ", @@ -909,7 +897,7 @@ "\n snapshot : true,", "\n filter : x => storage.DataNodesByImportScope[ImportScope.Primary].Contains(x.DataNode));", "\n", - "\n return Activity.Finish().Merge(parsingLog); ", + "\n return Activity.Finish(); ", "\n}" ] }, @@ -1131,10 +1119,10 @@ { "cell_type": "code", "source": [ - "public async Task UploadOpeningAsync(string fileName)", - "\n{", + "// public async Task UploadOpeningAsync(string fileName)", + "\nImport.DefineFormat(ImportFormats.Opening, async (options, dataSet) => {", "\n Activity.Start();", - "\n var args = (await GetArgsFromMainAsync(fileName)) with {ImportFormat = ImportFormats.Opening};", + "\n var args = (await GetArgsFromMainAsync(dataSet)) with {ImportFormat = ImportFormats.Opening};", "\n await DataNodeFactoryAsync(fileName, ImportFormats.Opening, args);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", From a188d5cb74aa465a1d0c6a04164afb650043fff0 Mon Sep 17 00:00:00 2001 From: nnikolopoulos Date: Tue, 1 Nov 2022 10:14:52 +0100 Subject: [PATCH 3/7] Apply import w format in A, SV and O --- ifrs17/Import/Importers.ipynb | 78 ++++++++++++++++------------------- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index db45dbf4..3aa01054 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -910,19 +910,20 @@ { "cell_type": "code", "source": [ - "public async Task ParseActualsToWorkspaceAsync(string fileName, ImportArgs args)", - "\n{", - "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n Workspace.Initialize(x => x.FromSource(DataSource)", + "Import.DefineFormat(ImportFormats.Actual, async (options, dataSet) => {", + "\n var workspace = Workspace.CreateNew();", + "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n workspace.Initialize(x => x.FromSource(DataSource)", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n ", "\n Activity.Start();", - "\n var parsingStorage = new ParsingStorage(args, DataSource, Workspace);", + "\n var args = (await GetArgsFromMainAsync(dataSet)) with {ImportFormat = ImportFormats.Actual};", + "\n var parsingStorage = new ParsingStorage(args, DataSource, workspace);", "\n await parsingStorage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", - "\n var importLog = await Import.FromFile(fileName)", + "\n var importLog = await Import.FromDataSet(dataSet)", "\n .WithType ( (dataset, datarow) => {", "\n var dataNode = datarow.Field(nameof(DataNode));", "\n if(!parsingStorage.DataNodeDataBySystemName.TryGetValue(dataNode, out var dataNodeData)) {", @@ -963,34 +964,31 @@ "\n };", "\n return item;", "\n }, ImportFormats.Actual", - "\n ).WithTarget(Workspace).ExecuteAsync();", + "\n ).WithTarget(workspace).ExecuteAsync();", "\n ", "\n await ValidateForDataNodeStateActiveAsync(parsingStorage.DataNodeDataBySystemName);", "\n return Activity.Finish().Merge(importLog);", - "\n}" + "\n});" ] }, { "cell_type": "code", "source": [ - "public async Task UploadActualAsync(string fileName)", + "public async Task UploadActualAsync(IDataSet dataSet)", "\n{", "\n Activity.Start();", - "\n var args = (await GetArgsFromMainAsync(fileName)) with {ImportFormat = ImportFormats.Actual};", - "\n await DataNodeFactoryAsync(fileName, ImportFormats.Actual, args);", + "\n var args = (await GetArgsFromMainAsync(dataSet)) with {ImportFormat = ImportFormats.Actual};", + "\n await DataNodeFactoryAsync(dataSet, ImportFormats.Actual, args);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", - "\n var parsingLog = await ParseActualsToWorkspaceAsync(fileName, args);", - "\n if(parsingLog.Errors.Any()) return Activity.Finish().Merge(parsingLog);", - "\n", "\n var storage = new ImportStorage(args, DataSource, Workspace);", "\n await storage.InitializeAsync();", - "\n if(Activity.HasErrors()) return Activity.Finish().Merge(parsingLog);", + "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", "\n var universe = Scopes.ForStorage(storage).ToScope();", "\n var identities = universe.GetScopes(storage.DataNodesByImportScope[ImportScope.Primary]).SelectMany(s => s.Identities);", "\n var ivs = universe.GetScopes(identities).SelectMany(x => x.CalculatedIfrsVariables);", - "\n if(Activity.HasErrors()) return Activity.Finish().Merge(parsingLog);", + "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", "\n await Workspace.UpdateAsync(ivs);", "\n await CommitToDatabase(storage.TargetPartition, ", @@ -998,7 +996,7 @@ "\n filter : x => storage.EstimateTypesByImportFormat[ImportFormats.Actual].Contains(x.EstimateType) && ", "\n storage.DataNodesByImportScope[ImportScope.Primary].Contains(x.DataNode));", "\n", - "\n return Activity.Finish().Merge(parsingLog);", + "\n return Activity.Finish();", "\n}" ] }, @@ -1017,20 +1015,21 @@ { "cell_type": "code", "source": [ - "public async Task ParseSimpleValueToWorkspaceAsync(string fileName, ImportArgs args, Guid targetPartitionByReportingNodeAndPeriodId)", - "\n{", - "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n Workspace.Initialize(x => x.FromSource(DataSource)", + "Import.DefineFormat(ImportFormats.SimpleValue, async (options, dataSet) => {", + "\n var workspace = Workspace.CreateNew(); ", + "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n workspace.Initialize(x => x.FromSource(DataSource)", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n ", "\n Activity.Start();", + "\n var args = (await GetArgsFromMainAsync(dataSet)) with {ImportFormat = ImportFormats.SimpleValue};", "\n var importFormat = args.ImportFormat;", - "\n var parsingStorage = new ParsingStorage(args, DataSource, Workspace);", + "\n var parsingStorage = new ParsingStorage(args, DataSource, workspace);", "\n await parsingStorage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish(); ", "\n", - "\n var importLog = await Import.FromFile(fileName)", + "\n var importLog = await Import.FromDataSet(dataSet)", "\n .WithType ( (dataset, datarow) => {", "\n var dataNode = parsingStorage.ValidateDataNode(datarow.Field(nameof(DataNode)));", "\n var amountType = parsingStorage.ValidateAmountType(datarow.Field(nameof(IfrsVariable.AmountType)));", @@ -1060,10 +1059,10 @@ "\n };", "\n return iv;", "\n }, importFormat // This should indicate the table name, not the input format", - "\n ).WithTarget(Workspace).ExecuteAsync();", + "\n ).WithTarget(workspace).ExecuteAsync();", "\n ", "\n // Checking if there are inconsistencies in the TechnicalMarginEstimateTypes --> double entries in the steps where we expect to have unique values", - "\n var invalidVariables = await Workspace.Query()", + "\n var invalidVariables = await workspace.Query()", "\n .Where(iv => parsingStorage.TechnicalMarginEstimateTypes.Contains(iv.EstimateType))", "\n .Where(iv => iv.AocType == AocTypes.BOP || iv.AocType == AocTypes.EOP || iv.AocType == AocTypes.AM || iv.AocType == AocTypes.EA)", "\n .GroupBy(iv => new {iv.DataNode, iv.AocType, iv.Novelty})", @@ -1075,9 +1074,10 @@ "\n ApplicationMessage.Log(Error.MultipleTechnicalMarginOpening, $\"{iv.DataNode},{iv.AocType},{iv.Novelty}\");", "\n ", "\n await ValidateForDataNodeStateActiveAsync(parsingStorage.DataNodeDataBySystemName);", + "\n Guid targetPartitionByReportingNodeAndPeriodId = new Guid();", "\n targetPartitionByReportingNodeAndPeriodId = parsingStorage.TargetPartitionByReportingNodeAndPeriod.Id;", "\n return Activity.Finish().Merge(importLog);", - "\n}" + "\n});" ] }, { @@ -1089,16 +1089,14 @@ { "cell_type": "code", "source": [ - "public async Task UploadSimpleValueAsync(string fileName)", + "public async Task UploadSimpleValueAsync(IDataSet dataSet)", "\n{", "\n Activity.Start();", - "\n var args = (await GetArgsFromMainAsync(fileName)) with {ImportFormat = ImportFormats.SimpleValue};", - "\n await DataNodeFactoryAsync(fileName, ImportFormats.SimpleValue, args);", + "\n var args = (await GetArgsFromMainAsync(dataSet)) with {ImportFormat = ImportFormats.SimpleValue};", + "\n await DataNodeFactoryAsync(dataSet, ImportFormats.SimpleValue, args);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", "\n Guid partitionId = new Guid();", - "\n var parsingLog = await ParseSimpleValueToWorkspaceAsync(fileName, args, partitionId);", - "\n if(parsingLog.Errors.Any()) return Activity.Finish().Merge(parsingLog);", "\n ", "\n Workspace.Query().Select(v => new {v.DataNode, v.AccidentYear}).Distinct();", "\n ", @@ -1106,7 +1104,7 @@ "\n snapshot : true,", "\n filter : x => Workspace.Query().Select(v => v.DataNode).Distinct().Contains(x.DataNode));", "\n ", - "\n return Activity.Finish().Merge(parsingLog);", + "\n return Activity.Finish();", "\n}" ] }, @@ -1119,25 +1117,21 @@ { "cell_type": "code", "source": [ - "// public async Task UploadOpeningAsync(string fileName)", - "\nImport.DefineFormat(ImportFormats.Opening, async (options, dataSet) => {", + "public async Task UploadOpeningAsync(IDataSet dataSet)", + "\n{", "\n Activity.Start();", "\n var args = (await GetArgsFromMainAsync(dataSet)) with {ImportFormat = ImportFormats.Opening};", - "\n await DataNodeFactoryAsync(fileName, ImportFormats.Opening, args);", + "\n await DataNodeFactoryAsync(dataSet, ImportFormats.Opening, args);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", - "\n Guid partitionId = new Guid();", - "\n var parsingLog = await ParseSimpleValueToWorkspaceAsync(fileName, args, partitionId);", - "\n if(parsingLog.Errors.Any()) return Activity.Finish().Merge(parsingLog);", - "\n ", "\n var storage = new ImportStorage(args, DataSource, Workspace);", "\n await storage.InitializeAsync();", - "\n if(Activity.HasErrors()) return Activity.Finish().Merge(parsingLog);", + "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", "\n var universe = Scopes.ForStorage(storage).ToScope();", "\n var identities = universe.GetScopes(storage.DataNodesByImportScope[ImportScope.Primary]).SelectMany(s => s.Identities);", "\n var ivs = universe.GetScopes(identities).SelectMany(x => x.CalculatedIfrsVariables);", - "\n if(Activity.HasErrors()) return Activity.Finish().Merge(parsingLog);", + "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", "\n await Workspace.UpdateAsync(ivs);", "\n await CommitToDatabase(storage.TargetPartition, ", @@ -1145,7 +1139,7 @@ "\n filter : x => storage.EstimateTypesByImportFormat[ImportFormats.Opening].Contains(x.EstimateType) && ", "\n storage.DataNodesByImportScope[ImportScope.Primary].Contains(x.DataNode ));", "\n", - "\n return Activity.Finish().Merge(parsingLog);", + "\n return Activity.Finish();", "\n}" ] }, From 602cc7332c3dfd4013dedc30b4d4de724aaabe45 Mon Sep 17 00:00:00 2001 From: nnikolopoulos Date: Wed, 2 Nov 2022 14:56:18 +0100 Subject: [PATCH 4/7] Correct Import statements --- ifrs17/Constants/Consts.ipynb | 4 +- ifrs17/Import/Importers.ipynb | 171 ++++++++++++++++++---------------- 2 files changed, 91 insertions(+), 84 deletions(-) diff --git a/ifrs17/Constants/Consts.ipynb b/ifrs17/Constants/Consts.ipynb index 19c0eec8..d5dafed5 100644 --- a/ifrs17/Constants/Consts.ipynb +++ b/ifrs17/Constants/Consts.ipynb @@ -138,12 +138,12 @@ "public static class ImportFormats{", "\n public const string Cashflow = nameof(Cashflow);", "\n public const string Actual = nameof(Actual);", + "\n public const string DataNode = nameof(DataNode);", "\n public const string DataNodeParameter = nameof(DataNodeParameter);", + "\n public const string DataNodeState = nameof(DataNodeState);", "\n public const string SimpleValue = nameof(SimpleValue);", "\n public const string Opening = nameof(Opening);", "\n public const string AocConfiguration = nameof(AocConfiguration);", - "\n public const string DataNode = nameof(DataNode);", - "\n public const string DataNodeState = nameof(DataNodeState);", "\n}" ] }, diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index 3aa01054..9f3f027a 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -467,8 +467,7 @@ { "cell_type": "markdown", "source": [ - "", - "\n## Parse and Upload: Data Nodes" + "## Parse and Upload: Data Nodes" ] }, { @@ -480,10 +479,10 @@ { "cell_type": "code", "source": [ - "Import.DefineFormat(ImportFormats.DataNode, async (options, dataSet) => {", - "\n var workspace = Workspace.CreateNew();", - "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(DataSource)", + "public async Task UploadDataNodesToWorkspaceAsync(IDataSet dataSet)", + "\n{", + "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n Workspace.Initialize(x => x.FromSource(DataSource)", "\n .DisableInitialization()", "\n .DisableInitialization()", "\n .DisableInitialization()", @@ -519,7 +518,7 @@ "\n ValuationApproach = datarow.Field(nameof(DataNode.ValuationApproach)),", "\n OciType = datarow.Field(nameof(DataNode.OciType))", "\n })", - "\n .WithTarget(workspace)", + "\n .WithTarget(Workspace)", "\n .ExecuteAsync();", "\n ", "\n var portfolios = await Workspace.Query().ToDictionaryAsync(x => x.SystemName);", @@ -573,24 +572,25 @@ "\n };", "\n return ExtendGroupOfContract(gric, datarow);", "\n })", - "\n .WithTarget(workspace)", + "\n .WithTarget(Workspace)", "\n .ExecuteAsync();", "\n ", "\n return Activity.Finish().Merge(importLogPortfolios).Merge(importLogGroupOfContracts);", - "\n});" + "\n}" ] }, { "cell_type": "code", "source": [ - "public async Task UploadDataNodesAsync()", - "\n{", + "Import.DefineFormat(ImportFormats.DataNode, async (options, dataSet) => {", + "\n var log = await UploadDataNodesToWorkspaceAsync(dataSet);", "\n var partition = (Guid)Workspace.Partition.GetCurrent(nameof(PartitionByReportingNode));", "\n await CommitToDatabase(partition);", "\n await CommitToDatabase(partition);", "\n await CommitToDatabase(partition);", "\n await CommitToDatabase(partition);", - "\n}" + "\n return log;", + "\n});" ] }, { @@ -623,15 +623,15 @@ { "cell_type": "code", "source": [ - "Import.DefineFormat(ImportFormats.DataNodeState, async (options, dataSet) => {", - "\n var workspace = Workspace.CreateNew();", - "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(DataSource)", + "public async Task UploadDataNodeStateToWorkspaceAsync(IDataSet dataSet)", + "\n{", + "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n Workspace.Initialize(x => x.FromSource(DataSource)", "\n .DisableInitialization()", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n ", - "\n await workspace.DeleteAsync(await workspace.Query().ToArrayAsync() );", + "\n await Workspace.DeleteAsync(await Workspace.Query().ToArrayAsync() );", "\n ", "\n Activity.Start();", "\n var args = await GetArgsFromMainAsync(dataSet);", @@ -649,20 +649,21 @@ "\n Month = args.Month,", "\n Partition = storage.TargetPartitionByReportingNode.Id", "\n }", - "\n ).WithTarget(workspace).ExecuteAsync();", + "\n ).WithTarget(Workspace).ExecuteAsync();", "\n", "\n await ValidateDataNodeStatesAsync(storage.DataNodeDataBySystemName);", "\n return Activity.Finish().Merge(importLog);", - "\n});" + "\n}" ] }, { "cell_type": "code", "source": [ - "public async Task UploadDataNodeStateAsync()", - "\n{", - "\n await CommitToDatabase((Guid)Workspace.Partition.GetCurrent(nameof(PartitionByReportingNode)), snapshot: false);", - "\n}" + "Import.DefineFormat(ImportFormats.DataNodeState, async (options, dataSet) => {", + "\n var log = await UploadDataNodeStateToWorkspaceAsync(dataSet);", + "\n await CommitToDatabase((Guid)Workspace.Partition.GetCurrent(nameof(PartitionByReportingNode)), snapshot: false); ", + "\n return log;", + "\n});" ] }, { @@ -674,15 +675,15 @@ { "cell_type": "code", "source": [ - "Import.DefineFormat(ImportFormats.DataNodeParameter, async (options, dataSet) => {", - "\n var workspace = Workspace.CreateNew();", - "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(DataSource)", + "public async Task UploadDataNodeParameterToWorkspaceAsync(IDataSet dataSet, Guid targetPartitionByReportingNodeId)", + "\n{", + "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n Workspace.Initialize(x => x.FromSource(DataSource)", "\n .DisableInitialization()", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n ", - "\n await workspace.DeleteAsync(await workspace.Query().ToArrayAsync() );", + "\n await Workspace.DeleteAsync(await Workspace.Query().ToArrayAsync() );", "\n ", "\n Activity.Start();", "\n var args = await GetArgsFromMainAsync(dataSet) with {ImportFormat = ImportFormats.DataNodeParameter};", @@ -751,24 +752,26 @@ "\n ReinsuranceCoverage = reinsCov,", "\n };", "\n })", - "\n .WithTarget(workspace)", + "\n .WithTarget(Workspace)", "\n .ExecuteAsync();", "\n ", - "\n Guid targetPartitionByReportingNodeId = storage.TargetPartitionByReportingNode.Id;", + "\n targetPartitionByReportingNodeId = storage.TargetPartitionByReportingNode.Id;", "\n return Activity.Finish().Merge(importLog);", - "\n});" + "\n}" ] }, { "cell_type": "code", "source": [ - "public async Task UploadDataNodeParameterAsync()", - "\n{", + "Import.DefineFormat(ImportFormats.DataNodeParameter, async (options, dataSet) => {", "\n Guid partitionId = new Guid();", + "\n var log = await UploadDataNodeParameterToWorkspaceAsync(dataSet, partitionId);", "\n ", "\n await CommitToDatabase(partitionId, snapshot: false);", - "\n await CommitToDatabase(partitionId, snapshot: false);", - "\n}" + "\n await CommitToDatabase(partitionId, snapshot: false); ", + "\n ", + "\n return log;", + "\n});" ] }, { @@ -803,15 +806,14 @@ { "cell_type": "code", "source": [ - "Import.DefineFormat(ImportFormats.Cashflow, async (options, dataSet) => {", - "\n var workspace = Workspace.CreateNew();", - "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(DataSource)", + "public async Task ParseCashflowsToWorkspaceAsync(IDataSet dataSet, ImportArgs args)", + "\n{", + "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n Workspace.Initialize(x => x.FromSource(DataSource)", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n ", "\n Activity.Start();", - "\n var args = (await GetArgsFromMainAsync(dataSet)) with {ImportFormat = ImportFormats.Cashflow};", "\n var parsingStorage = new ParsingStorage(args, DataSource, Workspace);", "\n await parsingStorage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish();", @@ -862,31 +864,33 @@ "\n };", "\n return item;", "\n }, ImportFormats.Cashflow", - "\n ).WithTarget(workspace).ExecuteAsync();", + "\n ).WithTarget(Workspace).ExecuteAsync();", "\n ", "\n await ValidateForDataNodeStateActiveAsync(parsingStorage.DataNodeDataBySystemName);", "\n return Activity.Finish().Merge(importLog);", - "\n});" + "\n}" ] }, { "cell_type": "code", "source": [ - "public async Task UploadCashflowsAsync(IDataSet dataSet)", - "\n{", + "Import.DefineFormat(ImportFormats.Cashflow, async (options, dataSet) => {", "\n Activity.Start();", "\n var args = (await GetArgsFromMainAsync(dataSet)) with {ImportFormat = ImportFormats.Cashflow};", "\n await DataNodeFactoryAsync(dataSet, ImportFormats.Cashflow, args);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", + "\n var parsingLog = await ParseCashflowsToWorkspaceAsync(dataSet, args);", + "\n if(parsingLog.Errors.Any()) return Activity.Finish().Merge(parsingLog);", + "\n ", "\n var storage = new ImportStorage(args, DataSource, Workspace);", "\n await storage.InitializeAsync();", - "\n if(Activity.HasErrors()) return Activity.Finish();", + "\n if(Activity.HasErrors()) return Activity.Finish().Merge(parsingLog);", "\n ", "\n var universe = Scopes.ForStorage(storage).ToScope();", "\n var identities = universe.GetScopes(storage.DataNodesByImportScope[ImportScope.Primary]).SelectMany(s => s.Identities);", "\n var ivs = universe.GetScopes(identities).SelectMany(x => x.CalculatedIfrsVariables);", - "\n if(Activity.HasErrors()) return Activity.Finish();", + "\n if(Activity.HasErrors()) return Activity.Finish().Merge(parsingLog);", "\n ", "\n await Workspace.UpdateAsync(ivs);", "\n await CommitToDatabase(storage.TargetPartition, ", @@ -897,8 +901,8 @@ "\n snapshot : true,", "\n filter : x => storage.DataNodesByImportScope[ImportScope.Primary].Contains(x.DataNode));", "\n", - "\n return Activity.Finish(); ", - "\n}" + "\n return Activity.Finish().Merge(parsingLog); ", + "\n});" ] }, { @@ -910,16 +914,15 @@ { "cell_type": "code", "source": [ - "Import.DefineFormat(ImportFormats.Actual, async (options, dataSet) => {", - "\n var workspace = Workspace.CreateNew();", - "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(DataSource)", + "public async Task ParseActualsToWorkspaceAsync(IDataSet dataSet, ImportArgs args)", + "\n{", + "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n Workspace.Initialize(x => x.FromSource(DataSource)", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n ", "\n Activity.Start();", - "\n var args = (await GetArgsFromMainAsync(dataSet)) with {ImportFormat = ImportFormats.Actual};", - "\n var parsingStorage = new ParsingStorage(args, DataSource, workspace);", + "\n var parsingStorage = new ParsingStorage(args, DataSource, Workspace);", "\n await parsingStorage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", @@ -964,31 +967,33 @@ "\n };", "\n return item;", "\n }, ImportFormats.Actual", - "\n ).WithTarget(workspace).ExecuteAsync();", + "\n ).WithTarget(Workspace).ExecuteAsync();", "\n ", "\n await ValidateForDataNodeStateActiveAsync(parsingStorage.DataNodeDataBySystemName);", "\n return Activity.Finish().Merge(importLog);", - "\n});" + "\n}" ] }, { "cell_type": "code", "source": [ - "public async Task UploadActualAsync(IDataSet dataSet)", - "\n{", + "Import.DefineFormat(ImportFormats.Actual, async (options, dataSet) => {", "\n Activity.Start();", "\n var args = (await GetArgsFromMainAsync(dataSet)) with {ImportFormat = ImportFormats.Actual};", "\n await DataNodeFactoryAsync(dataSet, ImportFormats.Actual, args);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", + "\n var parsingLog = await ParseActualsToWorkspaceAsync(dataSet, args);", + "\n if(parsingLog.Errors.Any()) return Activity.Finish().Merge(parsingLog);", + "\n", "\n var storage = new ImportStorage(args, DataSource, Workspace);", "\n await storage.InitializeAsync();", - "\n if(Activity.HasErrors()) return Activity.Finish();", + "\n if(Activity.HasErrors()) return Activity.Finish().Merge(parsingLog);", "\n", "\n var universe = Scopes.ForStorage(storage).ToScope();", "\n var identities = universe.GetScopes(storage.DataNodesByImportScope[ImportScope.Primary]).SelectMany(s => s.Identities);", "\n var ivs = universe.GetScopes(identities).SelectMany(x => x.CalculatedIfrsVariables);", - "\n if(Activity.HasErrors()) return Activity.Finish();", + "\n if(Activity.HasErrors()) return Activity.Finish().Merge(parsingLog);", "\n", "\n await Workspace.UpdateAsync(ivs);", "\n await CommitToDatabase(storage.TargetPartition, ", @@ -996,8 +1001,8 @@ "\n filter : x => storage.EstimateTypesByImportFormat[ImportFormats.Actual].Contains(x.EstimateType) && ", "\n storage.DataNodesByImportScope[ImportScope.Primary].Contains(x.DataNode));", "\n", - "\n return Activity.Finish();", - "\n}" + "\n return Activity.Finish().Merge(parsingLog);", + "\n});" ] }, { @@ -1015,17 +1020,16 @@ { "cell_type": "code", "source": [ - "Import.DefineFormat(ImportFormats.SimpleValue, async (options, dataSet) => {", - "\n var workspace = Workspace.CreateNew(); ", - "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n workspace.Initialize(x => x.FromSource(DataSource)", + "public async Task ParseSimpleValueToWorkspaceAsync(IDataSet dataSet, ImportArgs args, Guid targetPartitionByReportingNodeAndPeriodId)", + "\n{", + "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n Workspace.Initialize(x => x.FromSource(DataSource)", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n ", "\n Activity.Start();", - "\n var args = (await GetArgsFromMainAsync(dataSet)) with {ImportFormat = ImportFormats.SimpleValue};", "\n var importFormat = args.ImportFormat;", - "\n var parsingStorage = new ParsingStorage(args, DataSource, workspace);", + "\n var parsingStorage = new ParsingStorage(args, DataSource, Workspace);", "\n await parsingStorage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish(); ", "\n", @@ -1059,10 +1063,10 @@ "\n };", "\n return iv;", "\n }, importFormat // This should indicate the table name, not the input format", - "\n ).WithTarget(workspace).ExecuteAsync();", + "\n ).WithTarget(Workspace).ExecuteAsync();", "\n ", "\n // Checking if there are inconsistencies in the TechnicalMarginEstimateTypes --> double entries in the steps where we expect to have unique values", - "\n var invalidVariables = await workspace.Query()", + "\n var invalidVariables = await Workspace.Query()", "\n .Where(iv => parsingStorage.TechnicalMarginEstimateTypes.Contains(iv.EstimateType))", "\n .Where(iv => iv.AocType == AocTypes.BOP || iv.AocType == AocTypes.EOP || iv.AocType == AocTypes.AM || iv.AocType == AocTypes.EA)", "\n .GroupBy(iv => new {iv.DataNode, iv.AocType, iv.Novelty})", @@ -1074,10 +1078,9 @@ "\n ApplicationMessage.Log(Error.MultipleTechnicalMarginOpening, $\"{iv.DataNode},{iv.AocType},{iv.Novelty}\");", "\n ", "\n await ValidateForDataNodeStateActiveAsync(parsingStorage.DataNodeDataBySystemName);", - "\n Guid targetPartitionByReportingNodeAndPeriodId = new Guid();", "\n targetPartitionByReportingNodeAndPeriodId = parsingStorage.TargetPartitionByReportingNodeAndPeriod.Id;", "\n return Activity.Finish().Merge(importLog);", - "\n});" + "\n}" ] }, { @@ -1089,14 +1092,15 @@ { "cell_type": "code", "source": [ - "public async Task UploadSimpleValueAsync(IDataSet dataSet)", - "\n{", + "Import.DefineFormat(ImportFormats.SimpleValue, async (options, dataSet) => {", "\n Activity.Start();", "\n var args = (await GetArgsFromMainAsync(dataSet)) with {ImportFormat = ImportFormats.SimpleValue};", "\n await DataNodeFactoryAsync(dataSet, ImportFormats.SimpleValue, args);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", "\n Guid partitionId = new Guid();", + "\n var parsingLog = await ParseSimpleValueToWorkspaceAsync(dataSet, args, partitionId);", + "\n if(parsingLog.Errors.Any()) return Activity.Finish().Merge(parsingLog);", "\n ", "\n Workspace.Query().Select(v => new {v.DataNode, v.AccidentYear}).Distinct();", "\n ", @@ -1104,8 +1108,8 @@ "\n snapshot : true,", "\n filter : x => Workspace.Query().Select(v => v.DataNode).Distinct().Contains(x.DataNode));", "\n ", - "\n return Activity.Finish();", - "\n}" + "\n return Activity.Finish().Merge(parsingLog);", + "\n});" ] }, { @@ -1117,21 +1121,24 @@ { "cell_type": "code", "source": [ - "public async Task UploadOpeningAsync(IDataSet dataSet)", - "\n{", + "Import.DefineFormat(ImportFormats.Opening, async (options, dataSet) => {", "\n Activity.Start();", "\n var args = (await GetArgsFromMainAsync(dataSet)) with {ImportFormat = ImportFormats.Opening};", "\n await DataNodeFactoryAsync(dataSet, ImportFormats.Opening, args);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", + "\n Guid partitionId = new Guid();", + "\n var parsingLog = await ParseSimpleValueToWorkspaceAsync(dataSet, args, partitionId);", + "\n if(parsingLog.Errors.Any()) return Activity.Finish().Merge(parsingLog);", + "\n ", "\n var storage = new ImportStorage(args, DataSource, Workspace);", "\n await storage.InitializeAsync();", - "\n if(Activity.HasErrors()) return Activity.Finish();", + "\n if(Activity.HasErrors()) return Activity.Finish().Merge(parsingLog);", "\n ", "\n var universe = Scopes.ForStorage(storage).ToScope();", "\n var identities = universe.GetScopes(storage.DataNodesByImportScope[ImportScope.Primary]).SelectMany(s => s.Identities);", "\n var ivs = universe.GetScopes(identities).SelectMany(x => x.CalculatedIfrsVariables);", - "\n if(Activity.HasErrors()) return Activity.Finish();", + "\n if(Activity.HasErrors()) return Activity.Finish().Merge(parsingLog);", "\n", "\n await Workspace.UpdateAsync(ivs);", "\n await CommitToDatabase(storage.TargetPartition, ", @@ -1139,8 +1146,8 @@ "\n filter : x => storage.EstimateTypesByImportFormat[ImportFormats.Opening].Contains(x.EstimateType) && ", "\n storage.DataNodesByImportScope[ImportScope.Primary].Contains(x.DataNode ));", "\n", - "\n return Activity.Finish();", - "\n}" + "\n return Activity.Finish().Merge(parsingLog);", + "\n});" ] }, { From ddac5537599a53fe6db4803f4e4191ea33ec37d6 Mon Sep 17 00:00:00 2001 From: nnikolopoulos Date: Wed, 2 Nov 2022 17:16:59 +0100 Subject: [PATCH 5/7] Replace abs link with var link --- full-ifrs17-template/Database/Configure.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/full-ifrs17-template/Database/Configure.ipynb b/full-ifrs17-template/Database/Configure.ipynb index b332a43c..cbe56388 100644 --- a/full-ifrs17-template/Database/Configure.ipynb +++ b/full-ifrs17-template/Database/Configure.ipynb @@ -27,7 +27,8 @@ { "cell_type": "code", "source": [ - "#!import \"//ifrs17ce/dev/CalculationEngine\"" + "#!import \"../Constants/Consts\"", + "\n#!eval calculationEngine" ] }, { From c9fab0c21daff6a2e267bfd9b0142663587b9efa Mon Sep 17 00:00:00 2001 From: nnikolopoulos Date: Wed, 2 Nov 2022 17:24:47 +0100 Subject: [PATCH 6/7] Adjust naming convention --- ifrs17/Import/Importers.ipynb | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index 9f3f027a..8a499aa5 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -266,14 +266,14 @@ { "cell_type": "code", "source": [ - "public async Task DataNodeFactoryAsync(IDataSet dataSet, string tab, ImportArgs args)", + "public async Task DataNodeFactoryAsync(IDataSet dataSet, string tableName, ImportArgs args)", "\n{", "\n var partition = (await DataSource.Query().Where(p => p.ReportingNode == args.ReportingNode && p.Scenario == null).ToArrayAsync()).SingleOrDefault();", "\n if(partition == null) { ApplicationMessage.Log(Error.ParsedPartitionNotFound); return; }", "\n", - "\n var target = dataSet.Tables[tab];", + "\n var table = dataSet.Tables[tableName];", "\n", - "\n var dataNodesImported = target.Rows.Select(x => x.Field(nameof(RawVariable.DataNode))).ToHashSet();", + "\n var dataNodesImported = table.Rows.Select(x => x.Field(nameof(RawVariable.DataNode))).ToHashSet();", "\n var dataNodesDefined = await DataSource.Query().Where(x => dataNodesImported.Contains(x.SystemName)).ToArrayAsync();", "\n var dataNodeStatesDefined = await DataSource.Query().Select(x => x.DataNode).ToArrayAsync();", "\n var dataNodeParametersDefined = await DataSource.Query().Select(x => x.DataNode).ToArrayAsync(); ", @@ -302,24 +302,7 @@ { "cell_type": "markdown", "source": [ - "## Parse and Upload: Dimensions" - ] - }, - { - "cell_type": "code", - "source": [ - "public async Task UploadDimensionsAsync (string fileName) where T: class ", - "\n{", - "\n await CleanDatabaseAsync();", - "\n var importLog = await Import.FromFile(fileName).WithType().WithTarget(DataSource).ExecuteAsync();", - "\n return importLog;", - "\n}" - ] - }, - { - "cell_type": "markdown", - "source": [ - "### Analysis of Change Configuration" + "## Analysis of Change Configuration" ] }, { From d021daaa13e1e733ba1abe625ece64a0f6b0e6d2 Mon Sep 17 00:00:00 2001 From: nnikolopoulos Date: Wed, 2 Nov 2022 17:41:04 +0100 Subject: [PATCH 7/7] Adjust new import format in full-template --- .../InitSystemorphBaseToMemory.ipynb | 14 +++++------ .../InitSystemorphToDatabase.ipynb | 24 +++++++++---------- .../InitSystemorphToMemory.ipynb | 10 ++++---- .../Test/MapTemplateAndImportTest.ipynb | 3 +-- .../Test/SpecificationsImportCashflows.ipynb | 2 +- 5 files changed, 26 insertions(+), 27 deletions(-) diff --git a/full-ifrs17-template/Initialization/InitSystemorphBaseToMemory.ipynb b/full-ifrs17-template/Initialization/InitSystemorphBaseToMemory.ipynb index 1dc82a4e..e910c089 100644 --- a/full-ifrs17-template/Initialization/InitSystemorphBaseToMemory.ipynb +++ b/full-ifrs17-template/Initialization/InitSystemorphBaseToMemory.ipynb @@ -128,25 +128,25 @@ { "cell_type": "code", "source": [ - "await UploadDimensionsAsync(\"../Files/800.Parameters/YieldCurve.csv\")" + "await Import.FromFile(\"../Files/800.Parameters/YieldCurve.csv\").WithType().SnapshotMode().WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadDimensionsAsync(\"../Files/800.Parameters/ExchangeRate.csv\")" + "await Import.FromFile(\"../Files/800.Parameters/ExchangeRate.csv\").WithType().SnapshotMode().WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadDimensionsAsync(\"../Files/800.Parameters/PartnerRating.csv\")" + "await Import.FromFile(\"../Files/800.Parameters/PartnerRating.csv\").WithType().SnapshotMode().WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadDimensionsAsync(\"../Files/800.Parameters/CreditDefaultRate.csv\")" + "await Import.FromFile(\"../Files/800.Parameters/CreditDefaultRate.csv\").WithType().SnapshotMode().WithTarget(DataSource).ExecuteAsync()" ] }, { @@ -158,19 +158,19 @@ { "cell_type": "code", "source": [ - "await UploadDataNodesAsync(\"../Files/700.DataNodes/DataNodes_CH.csv\")" + "await Import.FromFile(\"../Files/700.DataNodes/DataNodes_CH.csv\").WithFormat(ImportFormats.DataNode).WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadDataNodeStateAsync(\"../Files/700.DataNodes/DataNodeStates_CH_2020_12.csv\")" + "await Import.FromFile(\"../Files/700.DataNodes/DataNodeStates_CH_2020_12.csv\").WithFormat(ImportFormats.DataNodeState).WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadDataNodeParameterAsync(\"../Files/700.DataNodes/DataNodeParameters_CH_2020_12.csv\")" + "await Import.FromFile(\"../Files/700.DataNodes/DataNodeParameters_CH_2020_12.csv\").WithFormat(ImportFormats.DataNodeParameter).WithTarget(DataSource).ExecuteAsync()" ] }, { diff --git a/full-ifrs17-template/Initialization/InitSystemorphToDatabase.ipynb b/full-ifrs17-template/Initialization/InitSystemorphToDatabase.ipynb index 9b7b5ab5..cc45633d 100644 --- a/full-ifrs17-template/Initialization/InitSystemorphToDatabase.ipynb +++ b/full-ifrs17-template/Initialization/InitSystemorphToDatabase.ipynb @@ -106,25 +106,25 @@ { "cell_type": "code", "source": [ - "await UploadDimensionsAsync(\"../Files/800.Parameters/YieldCurve.csv\")" + "await Import.FromFile(\"../Files/800.Parameters/YieldCurve.csv\").WithType().SnapshotMode().WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadDimensionsAsync(\"../Files/800.Parameters/ExchangeRate.csv\")" + "await Import.FromFile(\"../Files/800.Parameters/ExchangeRate.csv\").WithType().SnapshotMode().WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadDimensionsAsync(\"../Files/800.Parameters/PartnerRating.csv\")" + "await Import.FromFile(\"../Files/800.Parameters/PartnerRating.csv\").WithType().SnapshotMode().WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadDimensionsAsync(\"../Files/800.Parameters/CreditDefaultRate.csv\")" + "await Import.FromFile(\"../Files/800.Parameters/CreditDefaultRate.csv\").WithType().SnapshotMode().WithTarget(DataSource).ExecuteAsync()" ] }, { @@ -136,19 +136,19 @@ { "cell_type": "code", "source": [ - "await UploadDataNodesAsync(\"../Files/700.DataNodes/DataNodes_CH.csv\")" + "await Import.FromFile(\"../Files/700.DataNodes/DataNodes_CH.csv\").WithFormat(ImportFormats.DataNode).WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadDataNodeStateAsync(\"../Files/700.DataNodes/DataNodeStates_CH_2020_12.csv\")" + "await Import.FromFile(\"../Files/700.DataNodes/DataNodeStates_CH_2020_12.csv\").WithFormat(ImportFormats.DataNodeState).WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadDataNodeParameterAsync(\"../Files/700.DataNodes/DataNodeParameters_CH_2020_12.csv\")" + "await Import.FromFile(\"../Files/700.DataNodes/DataNodeParameters_CH_2020_12.csv\").WithFormat(ImportFormats.DataNodeParameter).WithTarget(DataSource).ExecuteAsync()" ] }, { @@ -160,31 +160,31 @@ { "cell_type": "code", "source": [ - "await UploadOpeningAsync(\"../Files/900.TransactionalData/Openings_CH_2020_12.csv\")" + "await Import.FromFile(\"../Files/900.TransactionalData/Openings_CH_2020_12.csv\").WithFormat(ImportFormats.Opening).WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadCashflowsAsync(\"../Files/900.TransactionalData/NominalCashflows_CH_2020_12.csv\")" + "await Import.FromFile(\"../Files/900.TransactionalData/NominalCashflows_CH_2020_12.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadActualAsync(\"../Files/900.TransactionalData/Actuals_CH_2020_12.csv\")" + "await Import.FromFile(\"../Files/900.TransactionalData/Actuals_CH_2020_12.csv\").WithFormat(ImportFormats.Actual).WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadCashflowsAsync(\"../Files/900.TransactionalData/NominalCashflows_CH_2021_3.csv\")" + "await Import.FromFile(\"../Files/900.TransactionalData/NominalCashflows_CH_2021_3.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadActualAsync(\"../Files/900.TransactionalData/Actuals_CH_2021_3.csv\")" + "await Import.FromFile(\"../Files/900.TransactionalData/Actuals_CH_2021_3.csv\").WithFormat(ImportFormats.Actual).WithTarget(DataSource).ExecuteAsync()" ] }, { diff --git a/full-ifrs17-template/Initialization/InitSystemorphToMemory.ipynb b/full-ifrs17-template/Initialization/InitSystemorphToMemory.ipynb index c9eb277e..1685c887 100644 --- a/full-ifrs17-template/Initialization/InitSystemorphToMemory.ipynb +++ b/full-ifrs17-template/Initialization/InitSystemorphToMemory.ipynb @@ -52,31 +52,31 @@ { "cell_type": "code", "source": [ - "await UploadOpeningAsync(\"../Files/900.TransactionalData/Openings_CH_2020_12.csv\")" + "await Import.FromFile(\"../Files/900.TransactionalData/Openings_CH_2020_12.csv\").WithFormat(ImportFormats.Opening).WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadCashflowsAsync(\"../Files/900.TransactionalData/NominalCashflows_CH_2020_12.csv\")" + "await Import.FromFile(\"../Files/900.TransactionalData/NominalCashflows_CH_2020_12.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadActualAsync(\"../Files/900.TransactionalData/Actuals_CH_2020_12.csv\")" + "await Import.FromFile(\"../Files/900.TransactionalData/Actuals_CH_2020_12.csv\").WithFormat(ImportFormats.Actual).WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadCashflowsAsync(\"../Files/900.TransactionalData/NominalCashflows_CH_2021_3.csv\")" + "await Import.FromFile(\"../Files/900.TransactionalData/NominalCashflows_CH_2021_3.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(DataSource).ExecuteAsync()" ] }, { "cell_type": "code", "source": [ - "await UploadActualAsync(\"../Files/900.TransactionalData/Actuals_CH_2021_3.csv\")" + "await Import.FromFile(\"../Files/900.TransactionalData/Actuals_CH_2021_3.csv\").WithFormat(ImportFormats.Actual).WithTarget(DataSource).ExecuteAsync()" ] }, { diff --git a/full-ifrs17-template/Test/MapTemplateAndImportTest.ipynb b/full-ifrs17-template/Test/MapTemplateAndImportTest.ipynb index e0b58e97..69f8137a 100644 --- a/full-ifrs17-template/Test/MapTemplateAndImportTest.ipynb +++ b/full-ifrs17-template/Test/MapTemplateAndImportTest.ipynb @@ -443,8 +443,7 @@ "source": [ "public async Task CheckErrors(string inputFileName, List errorBms)", "\n{", - "\n var partitionId = new Guid();", - "\n var log = await UploadDataNodeParameterToWorkspaceAsync(inputFileName, partitionId);", + "\n var log = await Import.FromFile(inputFileName).WithFormat(ImportFormats.DataNodeParameter).WithTarget(Workspace).ExecuteAsync();", "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", "\n errorBms.Intersect(log.Errors.Select(x => x.ToString().Substring(0,x.ToString().Length-2).Substring(40)).ToArray()).Count().Should().Be(errorBms.Count());", "\n}" diff --git a/full-ifrs17-template/Test/SpecificationsImportCashflows.ipynb b/full-ifrs17-template/Test/SpecificationsImportCashflows.ipynb index 12b9fc30..d75b5206 100644 --- a/full-ifrs17-template/Test/SpecificationsImportCashflows.ipynb +++ b/full-ifrs17-template/Test/SpecificationsImportCashflows.ipynb @@ -116,7 +116,7 @@ "cell_type": "code", "source": [ "var groupOfContract = \"GricComplex\";", - "\nawait UploadCashflowsAsync(\"Data/CashflowComplex.csv\")" + "\nawait Import.FromFile(\"Data/CashflowComplex.csv\").WithFormat(ImportFormats.Cashflow).WithTarget(DataSource).ExecuteAsync()" ] }, {