From 56fded2053127272d6c839e075d56987b970dfe4 Mon Sep 17 00:00:00 2001 From: Andrea Muolo Date: Sat, 8 Oct 2022 11:58:41 +0200 Subject: [PATCH 1/5] new data node factory --- Import/Importers.ipynb | 71 +++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/Import/Importers.ipynb b/Import/Importers.ipynb index 808b4cbe..6e3e0a55 100644 --- a/Import/Importers.ipynb +++ b/Import/Importers.ipynb @@ -1,19 +1,19 @@ { "metadata": { - "authors": [], "kernelspec": { "display_name": "Formula Framework", "language": "C#", "name": "C#" }, + "toc-autonumbering": "True", + "toc-showcode": "False", "language_info": { "file_extension": ".cs", "mimetype": "text/plain", "name": "C#" }, - "toc-autonumbering": "True", - "toc-showcode": "False", - "toc-showmarkdowntxt": "False" + "toc-showmarkdowntxt": "False", + "authors": [] }, "nbformat": 4, "nbformat_minor": 5, @@ -266,36 +266,31 @@ { "cell_type": "code", "source": [ - "// TODO: think of a better Factory to instantiate DataNode/State/Parameter simultaneously while avoiding public contructors (see Factory Patterns)", - "\n// ATTENTION: this Factory method provides default states and parameters so that it is not mandatory to import them (=> simpler calculation engine). ", - "\n// This method weakness is that these defaults aren't always replaced with the imported values, e,g, when Month > DefaultDataNodeActivationMonth.", - "\n// Therefore they will remain in the DataSource (hopefully without producing any harm).", - "\npublic async Task DataNodeFactoryAsync ()", + "public async Task DataNodeFactoryAsync(string file, string tab, ImportArgs args)", "\n{", - "\n var partition = (Guid)Workspace.Partition.GetCurrent(nameof(PartitionByReportingNode));", - "\n var existingGroupOfContracts = await DataSource.Query().Where(x => x.Partition == partition).ToArrayAsync();", - "\n var importedGroupOfContracts = await Workspace.Query().ToArrayAsync();", - "\n var listOfPartitionsWithPeriod = new List();", - "\n ", - "\n foreach (var groupOfContract in importedGroupOfContracts)", - "\n {", - "\n var name = groupOfContract.SystemName;", - "\n if(!existingGroupOfContracts.Any(x => x.SystemName == name))", - "\n {", - "\n var state = new DataNodeState { DataNode = name, Year = groupOfContract.AnnualCohort, Partition = partition };", - "\n var parameter = new SingleDataNodeParameter { DataNode = name, Year = groupOfContract.AnnualCohort, Partition = partition };", - "\n await Workspace.UpdateAsync(state);", - "\n if(groupOfContract is GroupOfInsuranceContract) await Workspace.UpdateAsync(parameter);", - "\n }", - "\n }", - "\n ", - "\n await CommitToDatabase(partition);", - "\n await CommitToDatabase(partition);", - "\n await CommitToDatabase(partition);", - "\n await CommitToDatabase(partition);", - "\n //Hot fix: Disabling default as using Annual Cohort breakes Opening importer logic. ", - "\n //await CommitToDatabase(partition, snapshot: false);", - "\n //await CommitToDatabase(partition, snapshot: false);", + "\n var partition = (await DataSource.Query().Where(p => p.ReportingNode == args.ReportingNode).ToArrayAsync()).SingleOrDefault();", + "\n if(partition == null) { ApplicationMessage.Log(Error.ParsedPartitionNotFound); return; }", + "\n", + "\n var extension = System.IO.Path.GetExtension(file);", + "\n var stream = await Project.FileStorage.ReadAsync(file);", + "\n var target = (await DataSetReader.ReadFromStream(stream).WithContentType(extension).ExecuteAsync()).Tables[tab];", + "\n", + "\n var dataNodesImported = target.Rows.Select(x => x.Field(nameof(RawVariable.DataNode))).ToHashSet();", + "\n var dataNodesDefined = await DataSource.Query().Where(x => dataNodesImported.Contains(x.SystemName)).ToArrayAsync();", + "\n var dataNodeStatesDefined = await DataSource.Query().Select(x => x.DataNode).ToArrayAsync();", + "\n var dataNodeParametersDefined = await DataSource.Query().Select(x => x.DataNode).ToArrayAsync(); ", + "\n", + "\n var dataNodeStatesUndefined = dataNodesImported.Where(x => !dataNodeStatesDefined.Contains(x)).Where(x => x != null).ToHashSet();", + "\n var dataNodeSingleParametersUndefined = dataNodesImported.Where(x => !dataNodeParametersDefined.Contains(x)).Where(x => x != null)", + "\n .Where(x => dataNodesDefined.SingleOrDefault(y => y.SystemName == x) is GroupOfInsuranceContract).ToHashSet();", + "\n", + "\n await DataSource.UpdateAsync( dataNodeStatesUndefined.Select(x => ", + "\n new DataNodeState {DataNode = x, Year = args.Year, Month = DefaultDataNodeActivationMonth, State = State.Active, Partition = partition.Id}).ToArray() );", + "\n", + "\n await DataSource.UpdateAsync( dataNodeSingleParametersUndefined.Select(x => ", + "\n new SingleDataNodeParameter {DataNode = x, Year = args.Year, Month = DefaultDataNodeActivationMonth, PremiumAllocation = DefaultPremiumExperienceAdjustmentFactor, Partition = partition.Id}).ToArray() );", + "\n", + "\n await DataSource.CommitAsync();", "\n}" ] }, @@ -491,8 +486,10 @@ "\n{", "\n var log = await UploadDataNodesToWorkspaceAsync(fileName);", "\n var partition = (Guid)Workspace.Partition.GetCurrent(nameof(PartitionByReportingNode));", - "\n ", - "\n await DataNodeFactoryAsync(); ", + "\n await CommitToDatabase(partition);", + "\n await CommitToDatabase(partition);", + "\n await CommitToDatabase(partition);", + "\n await CommitToDatabase(partition);", "\n return log;", "\n}" ] @@ -784,6 +781,7 @@ "\n{", "\n Activity.Start();", "\n var args = (await GetArgsFromMainAsync(fileName)) with {ImportFormat = ImportFormats.Cashflow};", + "\n await DataNodeFactoryAsync(fileName, ImportFormats.Cashflow, args);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", "\n var parsingLog = await ParseCashflowsToWorkspaceAsync(fileName, args);", @@ -886,6 +884,7 @@ "\n{", "\n Activity.Start();", "\n var args = (await GetArgsFromMainAsync(fileName)) with {ImportFormat = ImportFormats.Actual};", + "\n await DataNodeFactoryAsync(fileName, ImportFormats.Actual, args);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n", "\n var parsingLog = await ParseActualsToWorkspaceAsync(fileName, args);", @@ -997,6 +996,7 @@ "\n{", "\n Activity.Start();", "\n var args = (await GetArgsFromMainAsync(fileName)) with {ImportFormat = ImportFormats.SimpleValue};", + "\n await DataNodeFactoryAsync(fileName, ImportFormats.SimpleValue, args);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", "\n Guid partitionId = new Guid();", @@ -1026,6 +1026,7 @@ "\n{", "\n Activity.Start();", "\n var args = (await GetArgsFromMainAsync(fileName)) with {ImportFormat = ImportFormats.Opening};", + "\n await DataNodeFactoryAsync(fileName, ImportFormats.Opening, args);", "\n if(Activity.HasErrors()) return Activity.Finish();", "\n ", "\n Guid partitionId = new Guid();", From 74a381bb0a486a8becd62169680358d8137933ab Mon Sep 17 00:00:00 2001 From: Andrea Muolo Date: Mon, 10 Oct 2022 10:14:56 +0200 Subject: [PATCH 2/5] metadata --- Import/Importers.ipynb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Import/Importers.ipynb b/Import/Importers.ipynb index cfde246c..399ef397 100644 --- a/Import/Importers.ipynb +++ b/Import/Importers.ipynb @@ -1,19 +1,21 @@ { "metadata": { + "authors": [], "kernelspec": { "display_name": "Formula Framework", "language": "C#", "name": "C#" }, - "toc-autonumbering": "True", - "toc-showcode": "False", + + "language_info": { "file_extension": ".cs", "mimetype": "text/plain", "name": "C#" }, - "toc-showmarkdowntxt": "False", - "authors": [] + "toc-autonumbering": "True", + "toc-showcode": "False", + "toc-showmarkdowntxt": "False" }, "nbformat": 4, "nbformat_minor": 5, @@ -1059,4 +1061,4 @@ ] } ] -} \ No newline at end of file +} From 10ef85f5ea57b05083ac3ff65a75ba90d71e81ab Mon Sep 17 00:00:00 2001 From: Andrea Muolo Date: Mon, 10 Oct 2022 10:16:11 +0200 Subject: [PATCH 3/5] metadata x2 --- Import/Importers.ipynb | 2 -- 1 file changed, 2 deletions(-) diff --git a/Import/Importers.ipynb b/Import/Importers.ipynb index 399ef397..b9c2ad24 100644 --- a/Import/Importers.ipynb +++ b/Import/Importers.ipynb @@ -6,8 +6,6 @@ "language": "C#", "name": "C#" }, - - "language_info": { "file_extension": ".cs", "mimetype": "text/plain", From 28fd4f99c22626244f0a8c7c334dce0550f94a95 Mon Sep 17 00:00:00 2001 From: Andrea Muolo Date: Mon, 10 Oct 2022 10:18:01 +0200 Subject: [PATCH 4/5] metadata x3 --- Import/Importers.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Import/Importers.ipynb b/Import/Importers.ipynb index b9c2ad24..db76e980 100644 --- a/Import/Importers.ipynb +++ b/Import/Importers.ipynb @@ -1,6 +1,6 @@ { "metadata": { - "authors": [], + "authors": [], "kernelspec": { "display_name": "Formula Framework", "language": "C#", @@ -11,7 +11,7 @@ "mimetype": "text/plain", "name": "C#" }, - "toc-autonumbering": "True", + "toc-autonumbering": "True", "toc-showcode": "False", "toc-showmarkdowntxt": "False" }, From 4a6c9462807bc13b16bd25585bdc14e43d4b731a Mon Sep 17 00:00:00 2001 From: Andrea Muolo Date: Mon, 10 Oct 2022 13:54:20 +0200 Subject: [PATCH 5/5] implement feedback --- Import/Importers.ipynb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Import/Importers.ipynb b/Import/Importers.ipynb index db76e980..cbf48bab 100644 --- a/Import/Importers.ipynb +++ b/Import/Importers.ipynb @@ -268,7 +268,7 @@ "source": [ "public async Task DataNodeFactoryAsync(string file, string tab, ImportArgs args)", "\n{", - "\n var partition = (await DataSource.Query().Where(p => p.ReportingNode == args.ReportingNode).ToArrayAsync()).SingleOrDefault();", + "\n var partition = (await DataSource.Query().Where(p => p.ReportingNode == args.ReportingNode && p.Scenario == null).ToArrayAsync()).SingleOrDefault();", "\n if(partition == null) { ApplicationMessage.Log(Error.ParsedPartitionNotFound); return; }", "\n", "\n var extension = System.IO.Path.GetExtension(file);", @@ -280,9 +280,10 @@ "\n var dataNodeStatesDefined = await DataSource.Query().Select(x => x.DataNode).ToArrayAsync();", "\n var dataNodeParametersDefined = await DataSource.Query().Select(x => x.DataNode).ToArrayAsync(); ", "\n", - "\n var dataNodeStatesUndefined = dataNodesImported.Where(x => !dataNodeStatesDefined.Contains(x)).Where(x => x != null).ToHashSet();", - "\n var dataNodeSingleParametersUndefined = dataNodesImported.Where(x => !dataNodeParametersDefined.Contains(x)).Where(x => x != null)", - "\n .Where(x => dataNodesDefined.SingleOrDefault(y => y.SystemName == x) is GroupOfInsuranceContract).ToHashSet();", + "\n var dataNodeStatesUndefined = dataNodesImported.Where(x => x != null && !dataNodeStatesDefined.Contains(x)).ToHashSet();", + "\n var dataNodeSingleParametersUndefined = dataNodesImported.Where(x => x != null &&", + "\n !dataNodeParametersDefined.Contains(x) && ", + "\n dataNodesDefined.SingleOrDefault(y => y.SystemName == x) is GroupOfInsuranceContract).ToHashSet();", "\n", "\n await DataSource.UpdateAsync( dataNodeStatesUndefined.Select(x => ", "\n new DataNodeState {DataNode = x, Year = args.Year, Month = DefaultDataNodeActivationMonth, State = State.Active, Partition = partition.Id}).ToArray() );", @@ -1059,4 +1060,4 @@ ] } ] -} +} \ No newline at end of file