diff --git a/ifrs17/Import/Importers.ipynb b/ifrs17/Import/Importers.ipynb index efc27cba..fd7524ee 100644 --- a/ifrs17/Import/Importers.ipynb +++ b/ifrs17/Import/Importers.ipynb @@ -295,6 +295,79 @@ "execution_count": 0, "outputs": [] }, + { + "cell_type": "markdown", + "source": [ + "## Parse Main Tab and return Args", + "\n", + "\nThe main table of our custom import formats contains the information which are required to identify the data partition. These information are temporarily stored in [Args](../DataModel/DataStructure#args) and used in the next methods. " + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "public async Task GetArgsFromMainAsync(IDataSet dataSet)", + "\n{", + "\n var mainTab = dataSet.Tables[Main];", + "\n if(mainTab == null) ApplicationMessage.Log(Error.NoMainTab);", + "\n if(mainTab.Rows.Count() == 0) ApplicationMessage.Log(Error.IncompleteMainTab);", + "\n if(mainTab.Columns.FirstOrDefault(x => x.ColumnName == nameof(ReportingNode)) == null) ApplicationMessage.Log(Error.ReportingNodeInMainNotFound);", + "\n if(ApplicationMessage.HasErrors()) return null;", + "\n ", + "\n var main = mainTab.Rows.First();", + "\n var reportingNode = (string)main[nameof(ReportingNode)];", + "\n var scenario = mainTab.Columns.Where(x => x.ColumnName == nameof(Scenario)).Count()>0? (string)main[nameof(Scenario)] : default(string);", + "\n", + "\n ImportArgs args;", + "\n switch(typeof(IPartition).Name) {", + "\n case nameof(PartitionByReportingNode) : {", + "\n args = new ImportArgs( reportingNode,", + "\n default(int),", + "\n default(int),", + "\n default(Periodicity),", + "\n scenario,", + "\n default(string));", + "\n await DataSource.UpdateAsync( new[] { new PartitionByReportingNode { ", + "\n Id = (Guid)(await DataSource.Partition.GetKeyForInstanceAsync(args)),", + "\n ReportingNode = reportingNode, ", + "\n Scenario = scenario } } );", + "\n break;", + "\n }", + "\n case nameof(PartitionByReportingNodeAndPeriod) : {", + "\n if(mainTab.Columns.Where(x => x.ColumnName == nameof(PartitionByReportingNodeAndPeriod.Year)).Count()!=1) ApplicationMessage.Log(Error.YearInMainNotFound);", + "\n if(mainTab.Columns.Where(x => x.ColumnName == nameof(PartitionByReportingNodeAndPeriod.Month)).Count()!=1) ApplicationMessage.Log(Error.MonthInMainNotFound);", + "\n if(ApplicationMessage.HasErrors()) return null;", + "\n args = new ImportArgs( reportingNode, ", + "\n (int)Convert.ChangeType(main[nameof(PartitionByReportingNodeAndPeriod.Year)], typeof(int)),", + "\n (int)Convert.ChangeType(main[nameof(PartitionByReportingNodeAndPeriod.Month)], typeof(int)),", + "\n default(Periodicity),", + "\n scenario,", + "\n default(string) );", + "\n await DataSource.UpdateAsync( new[]{ new PartitionByReportingNodeAndPeriod { ", + "\n Id = (Guid)(await DataSource.Partition.GetKeyForInstanceAsync(args)),", + "\n Year = args.Year,", + "\n Month = args.Month,", + "\n ReportingNode = reportingNode, ", + "\n Scenario = scenario } } );", + "\n break;", + "\n }", + "\n default : {", + "\n ApplicationMessage.Log(Error.PartitionTypeNotFound, typeof(IPartition).Name); ", + "\n return null;", + "\n }", + "\n }", + "\n ", + "\n await DataSource.CommitAsync();", + "\n return args;", + "\n}" + ], + "metadata": {}, + "execution_count": 0, + "outputs": [] + }, { "cell_type": "markdown", "source": [ @@ -459,79 +532,6 @@ "execution_count": 0, "outputs": [] }, - { - "cell_type": "markdown", - "source": [ - "## Parse Main Tab and return Args", - "\n", - "\nThe main table of our custom import formats contains the information which are required to identify the data partition. These information are temporarily stored in [Args](../DataModel/DataStructure#args) and used in the next methods. " - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "public async Task GetArgsFromMainAsync(IDataSet dataSet)", - "\n{", - "\n var mainTab = dataSet.Tables[Main];", - "\n if(mainTab == null) ApplicationMessage.Log(Error.NoMainTab);", - "\n if(mainTab.Rows.Count() == 0) ApplicationMessage.Log(Error.IncompleteMainTab);", - "\n if(mainTab.Columns.FirstOrDefault(x => x.ColumnName == nameof(ReportingNode)) == null) ApplicationMessage.Log(Error.ReportingNodeInMainNotFound);", - "\n if(ApplicationMessage.HasErrors()) return null;", - "\n ", - "\n var main = mainTab.Rows.First();", - "\n var reportingNode = (string)main[nameof(ReportingNode)];", - "\n var scenario = mainTab.Columns.Where(x => x.ColumnName == nameof(Scenario)).Count()>0? (string)main[nameof(Scenario)] : default(string);", - "\n", - "\n ImportArgs args;", - "\n switch(typeof(IPartition).Name) {", - "\n case nameof(PartitionByReportingNode) : {", - "\n args = new ImportArgs( reportingNode,", - "\n default(int),", - "\n default(int),", - "\n default(Periodicity),", - "\n scenario,", - "\n default(string));", - "\n await DataSource.UpdateAsync( new[] { new PartitionByReportingNode { ", - "\n Id = (Guid)(await DataSource.Partition.GetKeyForInstanceAsync(args)),", - "\n ReportingNode = reportingNode, ", - "\n Scenario = scenario } } );", - "\n break;", - "\n }", - "\n case nameof(PartitionByReportingNodeAndPeriod) : {", - "\n if(mainTab.Columns.Where(x => x.ColumnName == nameof(PartitionByReportingNodeAndPeriod.Year)).Count()!=1) ApplicationMessage.Log(Error.YearInMainNotFound);", - "\n if(mainTab.Columns.Where(x => x.ColumnName == nameof(PartitionByReportingNodeAndPeriod.Month)).Count()!=1) ApplicationMessage.Log(Error.MonthInMainNotFound);", - "\n if(ApplicationMessage.HasErrors()) return null;", - "\n args = new ImportArgs( reportingNode, ", - "\n (int)Convert.ChangeType(main[nameof(PartitionByReportingNodeAndPeriod.Year)], typeof(int)),", - "\n (int)Convert.ChangeType(main[nameof(PartitionByReportingNodeAndPeriod.Month)], typeof(int)),", - "\n default(Periodicity),", - "\n scenario,", - "\n default(string) );", - "\n await DataSource.UpdateAsync( new[]{ new PartitionByReportingNodeAndPeriod { ", - "\n Id = (Guid)(await DataSource.Partition.GetKeyForInstanceAsync(args)),", - "\n Year = args.Year,", - "\n Month = args.Month,", - "\n ReportingNode = reportingNode, ", - "\n Scenario = scenario } } );", - "\n break;", - "\n }", - "\n default : {", - "\n ApplicationMessage.Log(Error.PartitionTypeNotFound, typeof(IPartition).Name); ", - "\n return null;", - "\n }", - "\n }", - "\n ", - "\n await DataSource.CommitAsync();", - "\n return args;", - "\n}" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] - }, { "cell_type": "markdown", "source": [ @@ -688,9 +688,9 @@ { "cell_type": "code", "source": [ - "public async Task ValidateDataNodeStatesAsync(Dictionary persistentDataNodeByDataNode)", + "public async static Task ValidateDataNodeStatesAsync(this IWorkspace workspace, Dictionary persistentDataNodeByDataNode)", "\n{", - "\n foreach(var importedDataNodeState in await Workspace.Query().ToArrayAsync())", + "\n foreach(var importedDataNodeState in await workspace.Query().ToArrayAsync())", "\n {", "\n if(persistentDataNodeByDataNode.TryGetValue(importedDataNodeState.DataNode, out var currentPersistentDataNode))", "\n {", @@ -700,7 +700,7 @@ "\n importedDataNodeState.State.ToString());", "\n", "\n if(importedDataNodeState.State == currentPersistentDataNode.State)", - "\n await Workspace.DeleteAsync(importedDataNodeState);", + "\n await workspace.DeleteAsync(importedDataNodeState);", "\n }", "\n }", "\n}" @@ -712,21 +712,19 @@ { "cell_type": "code", "source": [ - "public async Task UploadDataNodeStateToWorkspaceAsync(IDataSet dataSet)", + "public async Task UploadDataNodeStateToWorkspaceAsync(IDataSet dataSet, IWorkspace workspace)", "\n{", - "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n Workspace.Initialize(x => x.FromSource(DataSource)", + "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n workspace.Initialize(x => x.FromSource(DataSource)", "\n .DisableInitialization()", "\n .DisableInitialization()", "\n .DisableInitialization());", - "\n ", - "\n await Workspace.DeleteAsync(await Workspace.Query().ToArrayAsync() );", - "\n ", + "\n ", "\n Activity.Start();", "\n var args = await GetArgsFromMainAsync(dataSet);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", - "\n var storage = new ParsingStorage(args, DataSource, Workspace);", + "\n var storage = new ParsingStorage(args, DataSource, workspace);", "\n await storage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", @@ -738,9 +736,9 @@ "\n Month = args.Month,", "\n Partition = storage.TargetPartitionByReportingNode.Id", "\n }", - "\n ).WithTarget(Workspace).ExecuteAsync();", + "\n ).WithTarget(workspace).ExecuteAsync();", "\n", - "\n await ValidateDataNodeStatesAsync(storage.DataNodeDataBySystemName);", + "\n await workspace.ValidateDataNodeStatesAsync(storage.DataNodeDataBySystemName);", "\n return Activity.Finish().Merge(importLog);", "\n}" ], @@ -752,8 +750,9 @@ "cell_type": "code", "source": [ "Import.DefineFormat(ImportFormats.DataNodeState, async (options, dataSet) => {", - "\n var log = await UploadDataNodeStateToWorkspaceAsync(dataSet);", - "\n await CommitToDatabase(Workspace, (Guid)Workspace.Partition.GetCurrent(nameof(PartitionByReportingNode)), snapshot: false); ", + "\n var workspace = Workspace.CreateNew();", + "\n var log = await UploadDataNodeStateToWorkspaceAsync(dataSet, workspace);", + "\n await CommitToDatabase(workspace, (Guid)workspace.Partition.GetCurrent(nameof(PartitionByReportingNode)), snapshot: false); ", "\n return log;", "\n});" ], @@ -773,21 +772,19 @@ { "cell_type": "code", "source": [ - "public async Task UploadDataNodeParameterToWorkspaceAsync(IDataSet dataSet, Guid targetPartitionByReportingNodeId)", + "public async Task UploadDataNodeParameterToWorkspaceAsync(IDataSet dataSet, Guid targetPartitionByReportingNodeId, IWorkspace workspace)", "\n{", - "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n Workspace.Initialize(x => x.FromSource(DataSource)", + "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n workspace.Initialize(x => x.FromSource(DataSource)", "\n .DisableInitialization()", "\n .DisableInitialization()", "\n .DisableInitialization());", - "\n ", - "\n await Workspace.DeleteAsync(await Workspace.Query().ToArrayAsync() );", - "\n ", + "\n ", "\n Activity.Start();", "\n var args = await GetArgsFromMainAsync(dataSet) with {ImportFormat = ImportFormats.DataNodeParameter};", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", - "\n var storage = new ParsingStorage(args, DataSource, Workspace);", + "\n var storage = new ParsingStorage(args, DataSource, workspace);", "\n await storage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", @@ -850,7 +847,7 @@ "\n ReinsuranceCoverage = reinsCov,", "\n };", "\n })", - "\n .WithTarget(Workspace)", + "\n .WithTarget(workspace)", "\n .ExecuteAsync();", "\n ", "\n targetPartitionByReportingNodeId = storage.TargetPartitionByReportingNode.Id;", @@ -866,11 +863,10 @@ "source": [ "Import.DefineFormat(ImportFormats.DataNodeParameter, async (options, dataSet) => {", "\n Guid partitionId = new Guid();", - "\n var log = await UploadDataNodeParameterToWorkspaceAsync(dataSet, partitionId);", - "\n ", - "\n await CommitToDatabase(Workspace, partitionId, snapshot: false);", - "\n await CommitToDatabase(Workspace, partitionId, snapshot: false); ", - "\n ", + "\n var workspace = Workspace.CreateNew();", + "\n var log = await UploadDataNodeParameterToWorkspaceAsync(dataSet, partitionId, workspace);", + "\n await CommitToDatabase(workspace, partitionId, snapshot: false);", + "\n await CommitToDatabase(workspace, partitionId, snapshot: false); ", "\n return log;", "\n});" ], @@ -1048,15 +1044,15 @@ { "cell_type": "code", "source": [ - "public async Task ParseActualsToWorkspaceAsync(IDataSet dataSet, ImportArgs args)", + "public async Task ParseActualsToWorkspaceAsync(IDataSet dataSet, ImportArgs args, IWorkspace workspace)", "\n{", - "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n Workspace.Initialize(x => x.FromSource(DataSource)", + "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n workspace.Initialize(x => x.FromSource(DataSource)", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n ", "\n Activity.Start();", - "\n var parsingStorage = new ParsingStorage(args, DataSource, Workspace);", + "\n var parsingStorage = new ParsingStorage(args, DataSource, workspace);", "\n await parsingStorage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", @@ -1100,9 +1096,9 @@ "\n };", "\n return item;", "\n }, ImportFormats.Actual", - "\n ).WithTarget(Workspace).ExecuteAsync();", + "\n ).WithTarget(workspace).ExecuteAsync();", "\n ", - "\n await ValidateForDataNodeStateActiveAsync(Workspace, parsingStorage.DataNodeDataBySystemName);", + "\n await ValidateForDataNodeStateActiveAsync(workspace, parsingStorage.DataNodeDataBySystemName);", "\n return Activity.Finish().Merge(importLog);", "\n}" ], @@ -1119,10 +1115,11 @@ "\n await DataNodeFactoryAsync(dataSet, ImportFormats.Actual, args);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", - "\n var parsingLog = await ParseActualsToWorkspaceAsync(dataSet, args);", + "\n var workspace = Workspace.CreateNew();", + "\n var parsingLog = await ParseActualsToWorkspaceAsync(dataSet, args, workspace);", "\n if(parsingLog.Errors.Any()) return Activity.Finish().Merge(parsingLog);", "\n", - "\n var storage = new ImportStorage(args, DataSource, Workspace);", + "\n var storage = new ImportStorage(args, DataSource, workspace);", "\n await storage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish().Merge(parsingLog);", "\n", @@ -1131,8 +1128,8 @@ "\n var ivs = universe.GetScopes(identities).SelectMany(x => x.CalculatedIfrsVariables);", "\n if(Activity.HasErrors()) return Activity.Finish().Merge(parsingLog);", "\n", - "\n await Workspace.UpdateAsync(ivs);", - "\n await CommitToDatabase(Workspace, ", + "\n await workspace.UpdateAsync(ivs);", + "\n await CommitToDatabase(workspace, ", "\n storage.TargetPartition, ", "\n snapshot : true, ", "\n filter : x => storage.EstimateTypesByImportFormat[ImportFormats.Actual].Contains(x.EstimateType) && ", @@ -1171,16 +1168,16 @@ { "cell_type": "code", "source": [ - "public async Task ParseSimpleValueToWorkspaceAsync(IDataSet dataSet, ImportArgs args, Guid targetPartitionByReportingNodeAndPeriodId)", + "public async Task ParseSimpleValueToWorkspaceAsync(IDataSet dataSet, ImportArgs args, Guid targetPartitionByReportingNodeAndPeriodId, IWorkspace workspace)", "\n{", - "\n Workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", - "\n Workspace.Initialize(x => x.FromSource(DataSource)", + "\n workspace.Reset(x => x.ResetInitializationRules().ResetCurrentPartitions());", + "\n workspace.Initialize(x => x.FromSource(DataSource)", "\n .DisableInitialization()", "\n .DisableInitialization());", "\n ", "\n Activity.Start();", "\n var importFormat = args.ImportFormat;", - "\n var parsingStorage = new ParsingStorage(args, DataSource, Workspace);", + "\n var parsingStorage = new ParsingStorage(args, DataSource, workspace);", "\n await parsingStorage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish(); ", "\n", @@ -1212,10 +1209,10 @@ "\n };", "\n return iv;", "\n }, importFormat // This should indicate the table name, not the input format", - "\n ).WithTarget(Workspace).ExecuteAsync();", + "\n ).WithTarget(workspace).ExecuteAsync();", "\n ", "\n // Checking if there are inconsistencies in the TechnicalMarginEstimateTypes --> double entries in the steps where we expect to have unique values", - "\n var invalidVariables = await Workspace.Query()", + "\n var invalidVariables = await workspace.Query()", "\n .Where(iv => parsingStorage.TechnicalMarginEstimateTypes.Contains(iv.EstimateType))", "\n .Where(iv => iv.AocType == AocTypes.BOP || iv.AocType == AocTypes.EOP || iv.AocType == AocTypes.AM || iv.AocType == AocTypes.EA)", "\n .GroupBy(iv => new {iv.DataNode, iv.AocType, iv.Novelty})", @@ -1226,7 +1223,7 @@ "\n foreach (var iv in invalidVariables)", "\n ApplicationMessage.Log(Error.MultipleTechnicalMarginOpening, $\"{iv.DataNode},{iv.AocType},{iv.Novelty}\");", "\n ", - "\n await ValidateForDataNodeStateActiveAsync(Workspace, parsingStorage.DataNodeDataBySystemName);", + "\n await ValidateForDataNodeStateActiveAsync(workspace, parsingStorage.DataNodeDataBySystemName);", "\n targetPartitionByReportingNodeAndPeriodId = parsingStorage.TargetPartitionByReportingNodeAndPeriod.Id;", "\n return Activity.Finish().Merge(importLog);", "\n}" @@ -1254,15 +1251,16 @@ "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", "\n Guid partitionId = new Guid();", - "\n var parsingLog = await ParseSimpleValueToWorkspaceAsync(dataSet, args, partitionId);", + "\n var workspace = Workspace.CreateNew();", + "\n var parsingLog = await ParseSimpleValueToWorkspaceAsync(dataSet, args, partitionId, workspace);", "\n if(parsingLog.Errors.Any()) return Activity.Finish().Merge(parsingLog);", "\n ", - "\n Workspace.Query().Select(v => new {v.DataNode, v.AccidentYear}).Distinct();", + "\n workspace.Query().Select(v => new {v.DataNode, v.AccidentYear}).Distinct();", "\n ", - "\n await CommitToDatabase(Workspace, ", + "\n await CommitToDatabase(workspace, ", "\n partitionId, ", "\n snapshot : true,", - "\n filter : x => Workspace.Query().Select(v => v.DataNode).Distinct().Contains(x.DataNode));", + "\n filter : x => workspace.Query().Select(v => v.DataNode).Distinct().Contains(x.DataNode));", "\n ", "\n return Activity.Finish().Merge(parsingLog);", "\n});" @@ -1290,10 +1288,11 @@ "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", "\n Guid partitionId = new Guid();", - "\n var parsingLog = await ParseSimpleValueToWorkspaceAsync(dataSet, args, partitionId);", + "\n var workspace = Workspace.CreateNew();", + "\n var parsingLog = await ParseSimpleValueToWorkspaceAsync(dataSet, args, partitionId, workspace);", "\n if(parsingLog.Errors.Any()) return Activity.Finish().Merge(parsingLog);", "\n ", - "\n var storage = new ImportStorage(args, DataSource, Workspace);", + "\n var storage = new ImportStorage(args, DataSource, workspace);", "\n await storage.InitializeAsync();", "\n if(Activity.HasErrors()) return Activity.Finish().Merge(parsingLog);", "\n ", @@ -1302,8 +1301,8 @@ "\n var ivs = universe.GetScopes(identities).SelectMany(x => x.CalculatedIfrsVariables);", "\n if(Activity.HasErrors()) return Activity.Finish().Merge(parsingLog);", "\n", - "\n await Workspace.UpdateAsync(ivs);", - "\n await CommitToDatabase(Workspace, ", + "\n await workspace.UpdateAsync(ivs);", + "\n await CommitToDatabase(workspace, ", "\n storage.TargetPartition, ", "\n snapshot : true,", "\n filter : x => storage.EstimateTypesByImportFormat[ImportFormats.Opening].Contains(x.EstimateType) && ", @@ -1315,15 +1314,6 @@ "metadata": {}, "execution_count": 0, "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "" - ], - "metadata": {}, - "execution_count": 0, - "outputs": [] } ] } \ No newline at end of file